1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include "rte_swx_pipeline_internal.h"
12 #define CHECK(condition, err_code) \
18 #define CHECK_NAME(name, err_code) \
21 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
24 #define CHECK_INSTRUCTION(instr, err_code) \
27 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
28 RTE_SWX_INSTRUCTION_SIZE), \
34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
36 #include <rte_malloc.h>
39 env_malloc(size_t size, size_t alignment, int numa_node)
41 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
45 env_free(void *start, size_t size __rte_unused)
55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
59 if (numa_available() == -1)
62 start = numa_alloc_onnode(size, numa_node);
66 memset(start, 0, size);
71 env_free(void *start, size_t size)
73 if (numa_available() == -1)
76 numa_free(start, size);
84 static struct struct_type *
85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
87 struct struct_type *elem;
89 TAILQ_FOREACH(elem, &p->struct_types, node)
90 if (strcmp(elem->name, name) == 0)
97 struct_type_field_find(struct struct_type *st, const char *name)
101 for (i = 0; i < st->n_fields; i++) {
102 struct field *f = &st->fields[i];
104 if (strcmp(f->name, name) == 0)
112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
114 struct rte_swx_field_params *fields,
116 int last_field_has_variable_size)
118 struct struct_type *st;
122 CHECK_NAME(name, EINVAL);
123 CHECK(fields, EINVAL);
124 CHECK(n_fields, EINVAL);
126 for (i = 0; i < n_fields; i++) {
127 struct rte_swx_field_params *f = &fields[i];
128 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
131 CHECK_NAME(f->name, EINVAL);
132 CHECK(f->n_bits, EINVAL);
133 CHECK((f->n_bits <= 64) || var_size, EINVAL);
134 CHECK((f->n_bits & 7) == 0, EINVAL);
136 for (j = 0; j < i; j++) {
137 struct rte_swx_field_params *f_prev = &fields[j];
139 CHECK(strcmp(f->name, f_prev->name), EINVAL);
143 CHECK(!struct_type_find(p, name), EEXIST);
145 /* Node allocation. */
146 st = calloc(1, sizeof(struct struct_type));
149 st->fields = calloc(n_fields, sizeof(struct field));
155 /* Node initialization. */
156 strcpy(st->name, name);
157 for (i = 0; i < n_fields; i++) {
158 struct field *dst = &st->fields[i];
159 struct rte_swx_field_params *src = &fields[i];
160 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
162 strcpy(dst->name, src->name);
163 dst->n_bits = src->n_bits;
164 dst->offset = st->n_bits;
165 dst->var_size = var_size;
167 st->n_bits += src->n_bits;
168 st->n_bits_min += var_size ? 0 : src->n_bits;
170 st->n_fields = n_fields;
171 st->var_size = last_field_has_variable_size;
173 /* Node add to tailq. */
174 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
180 struct_build(struct rte_swx_pipeline *p)
184 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
185 struct thread *t = &p->threads[i];
187 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
188 CHECK(t->structs, ENOMEM);
195 struct_build_free(struct rte_swx_pipeline *p)
199 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
200 struct thread *t = &p->threads[i];
208 struct_free(struct rte_swx_pipeline *p)
210 struct_build_free(p);
214 struct struct_type *elem;
216 elem = TAILQ_FIRST(&p->struct_types);
220 TAILQ_REMOVE(&p->struct_types, elem, node);
229 static struct port_in_type *
230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
232 struct port_in_type *elem;
237 TAILQ_FOREACH(elem, &p->port_in_types, node)
238 if (strcmp(elem->name, name) == 0)
245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
247 struct rte_swx_port_in_ops *ops)
249 struct port_in_type *elem;
252 CHECK_NAME(name, EINVAL);
254 CHECK(ops->create, EINVAL);
255 CHECK(ops->free, EINVAL);
256 CHECK(ops->pkt_rx, EINVAL);
257 CHECK(ops->stats_read, EINVAL);
259 CHECK(!port_in_type_find(p, name), EEXIST);
261 /* Node allocation. */
262 elem = calloc(1, sizeof(struct port_in_type));
265 /* Node initialization. */
266 strcpy(elem->name, name);
267 memcpy(&elem->ops, ops, sizeof(*ops));
269 /* Node add to tailq. */
270 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
275 static struct port_in *
276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
278 struct port_in *port;
280 TAILQ_FOREACH(port, &p->ports_in, node)
281 if (port->id == port_id)
288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
290 const char *port_type_name,
293 struct port_in_type *type = NULL;
294 struct port_in *port = NULL;
299 CHECK(!port_in_find(p, port_id), EINVAL);
301 CHECK_NAME(port_type_name, EINVAL);
302 type = port_in_type_find(p, port_type_name);
305 obj = type->ops.create(args);
308 /* Node allocation. */
309 port = calloc(1, sizeof(struct port_in));
312 /* Node initialization. */
317 /* Node add to tailq. */
318 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
319 if (p->n_ports_in < port_id + 1)
320 p->n_ports_in = port_id + 1;
326 port_in_build(struct rte_swx_pipeline *p)
328 struct port_in *port;
331 CHECK(p->n_ports_in, EINVAL);
332 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
334 for (i = 0; i < p->n_ports_in; i++)
335 CHECK(port_in_find(p, i), EINVAL);
337 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
338 CHECK(p->in, ENOMEM);
340 TAILQ_FOREACH(port, &p->ports_in, node) {
341 struct port_in_runtime *in = &p->in[port->id];
343 in->pkt_rx = port->type->ops.pkt_rx;
351 port_in_build_free(struct rte_swx_pipeline *p)
358 port_in_free(struct rte_swx_pipeline *p)
360 port_in_build_free(p);
364 struct port_in *port;
366 port = TAILQ_FIRST(&p->ports_in);
370 TAILQ_REMOVE(&p->ports_in, port, node);
371 port->type->ops.free(port->obj);
375 /* Input port types. */
377 struct port_in_type *elem;
379 elem = TAILQ_FIRST(&p->port_in_types);
383 TAILQ_REMOVE(&p->port_in_types, elem, node);
391 static struct port_out_type *
392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
394 struct port_out_type *elem;
399 TAILQ_FOREACH(elem, &p->port_out_types, node)
400 if (!strcmp(elem->name, name))
407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
409 struct rte_swx_port_out_ops *ops)
411 struct port_out_type *elem;
414 CHECK_NAME(name, EINVAL);
416 CHECK(ops->create, EINVAL);
417 CHECK(ops->free, EINVAL);
418 CHECK(ops->pkt_tx, EINVAL);
419 CHECK(ops->stats_read, EINVAL);
421 CHECK(!port_out_type_find(p, name), EEXIST);
423 /* Node allocation. */
424 elem = calloc(1, sizeof(struct port_out_type));
427 /* Node initialization. */
428 strcpy(elem->name, name);
429 memcpy(&elem->ops, ops, sizeof(*ops));
431 /* Node add to tailq. */
432 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
437 static struct port_out *
438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
440 struct port_out *port;
442 TAILQ_FOREACH(port, &p->ports_out, node)
443 if (port->id == port_id)
450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
452 const char *port_type_name,
455 struct port_out_type *type = NULL;
456 struct port_out *port = NULL;
461 CHECK(!port_out_find(p, port_id), EINVAL);
463 CHECK_NAME(port_type_name, EINVAL);
464 type = port_out_type_find(p, port_type_name);
467 obj = type->ops.create(args);
470 /* Node allocation. */
471 port = calloc(1, sizeof(struct port_out));
474 /* Node initialization. */
479 /* Node add to tailq. */
480 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
481 if (p->n_ports_out < port_id + 1)
482 p->n_ports_out = port_id + 1;
488 port_out_build(struct rte_swx_pipeline *p)
490 struct port_out *port;
493 CHECK(p->n_ports_out, EINVAL);
495 for (i = 0; i < p->n_ports_out; i++)
496 CHECK(port_out_find(p, i), EINVAL);
498 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
499 CHECK(p->out, ENOMEM);
501 TAILQ_FOREACH(port, &p->ports_out, node) {
502 struct port_out_runtime *out = &p->out[port->id];
504 out->pkt_tx = port->type->ops.pkt_tx;
505 out->flush = port->type->ops.flush;
506 out->obj = port->obj;
513 port_out_build_free(struct rte_swx_pipeline *p)
520 port_out_free(struct rte_swx_pipeline *p)
522 port_out_build_free(p);
526 struct port_out *port;
528 port = TAILQ_FIRST(&p->ports_out);
532 TAILQ_REMOVE(&p->ports_out, port, node);
533 port->type->ops.free(port->obj);
537 /* Output port types. */
539 struct port_out_type *elem;
541 elem = TAILQ_FIRST(&p->port_out_types);
545 TAILQ_REMOVE(&p->port_out_types, elem, node);
553 static struct extern_type *
554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
556 struct extern_type *elem;
558 TAILQ_FOREACH(elem, &p->extern_types, node)
559 if (strcmp(elem->name, name) == 0)
565 static struct extern_type_member_func *
566 extern_type_member_func_find(struct extern_type *type, const char *name)
568 struct extern_type_member_func *elem;
570 TAILQ_FOREACH(elem, &type->funcs, node)
571 if (strcmp(elem->name, name) == 0)
577 static struct extern_obj *
578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
580 struct extern_obj *elem;
582 TAILQ_FOREACH(elem, &p->extern_objs, node)
583 if (strcmp(elem->name, name) == 0)
589 static struct extern_type_member_func *
590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
592 struct extern_obj **obj)
594 struct extern_obj *object;
595 struct extern_type_member_func *func;
596 char *object_name, *func_name;
598 if (name[0] != 'e' || name[1] != '.')
601 object_name = strdup(&name[2]);
605 func_name = strchr(object_name, '.');
614 object = extern_obj_find(p, object_name);
620 func = extern_type_member_func_find(object->type, func_name);
633 static struct field *
634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
636 struct extern_obj **object)
638 struct extern_obj *obj;
640 char *obj_name, *field_name;
642 if ((name[0] != 'e') || (name[1] != '.'))
645 obj_name = strdup(&name[2]);
649 field_name = strchr(obj_name, '.');
658 obj = extern_obj_find(p, obj_name);
664 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
680 const char *mailbox_struct_type_name,
681 rte_swx_extern_type_constructor_t constructor,
682 rte_swx_extern_type_destructor_t destructor)
684 struct extern_type *elem;
685 struct struct_type *mailbox_struct_type;
689 CHECK_NAME(name, EINVAL);
690 CHECK(!extern_type_find(p, name), EEXIST);
692 CHECK_NAME(mailbox_struct_type_name, EINVAL);
693 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
694 CHECK(mailbox_struct_type, EINVAL);
695 CHECK(!mailbox_struct_type->var_size, EINVAL);
697 CHECK(constructor, EINVAL);
698 CHECK(destructor, EINVAL);
700 /* Node allocation. */
701 elem = calloc(1, sizeof(struct extern_type));
704 /* Node initialization. */
705 strcpy(elem->name, name);
706 elem->mailbox_struct_type = mailbox_struct_type;
707 elem->constructor = constructor;
708 elem->destructor = destructor;
709 TAILQ_INIT(&elem->funcs);
711 /* Node add to tailq. */
712 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
719 const char *extern_type_name,
721 rte_swx_extern_type_member_func_t member_func)
723 struct extern_type *type;
724 struct extern_type_member_func *type_member;
728 CHECK_NAME(extern_type_name, EINVAL);
729 type = extern_type_find(p, extern_type_name);
731 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
733 CHECK_NAME(name, EINVAL);
734 CHECK(!extern_type_member_func_find(type, name), EEXIST);
736 CHECK(member_func, EINVAL);
738 /* Node allocation. */
739 type_member = calloc(1, sizeof(struct extern_type_member_func));
740 CHECK(type_member, ENOMEM);
742 /* Node initialization. */
743 strcpy(type_member->name, name);
744 type_member->func = member_func;
745 type_member->id = type->n_funcs;
747 /* Node add to tailq. */
748 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
756 const char *extern_type_name,
760 struct extern_type *type;
761 struct extern_obj *obj;
766 CHECK_NAME(extern_type_name, EINVAL);
767 type = extern_type_find(p, extern_type_name);
770 CHECK_NAME(name, EINVAL);
771 CHECK(!extern_obj_find(p, name), EEXIST);
773 /* Node allocation. */
774 obj = calloc(1, sizeof(struct extern_obj));
777 /* Object construction. */
778 obj_handle = type->constructor(args);
784 /* Node initialization. */
785 strcpy(obj->name, name);
787 obj->obj = obj_handle;
788 obj->struct_id = p->n_structs;
789 obj->id = p->n_extern_objs;
791 /* Node add to tailq. */
792 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
800 extern_obj_build(struct rte_swx_pipeline *p)
804 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
805 struct thread *t = &p->threads[i];
806 struct extern_obj *obj;
808 t->extern_objs = calloc(p->n_extern_objs,
809 sizeof(struct extern_obj_runtime));
810 CHECK(t->extern_objs, ENOMEM);
812 TAILQ_FOREACH(obj, &p->extern_objs, node) {
813 struct extern_obj_runtime *r =
814 &t->extern_objs[obj->id];
815 struct extern_type_member_func *func;
816 uint32_t mailbox_size =
817 obj->type->mailbox_struct_type->n_bits / 8;
821 r->mailbox = calloc(1, mailbox_size);
822 CHECK(r->mailbox, ENOMEM);
824 TAILQ_FOREACH(func, &obj->type->funcs, node)
825 r->funcs[func->id] = func->func;
827 t->structs[obj->struct_id] = r->mailbox;
835 extern_obj_build_free(struct rte_swx_pipeline *p)
839 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
840 struct thread *t = &p->threads[i];
846 for (j = 0; j < p->n_extern_objs; j++) {
847 struct extern_obj_runtime *r = &t->extern_objs[j];
852 free(t->extern_objs);
853 t->extern_objs = NULL;
858 extern_obj_free(struct rte_swx_pipeline *p)
860 extern_obj_build_free(p);
862 /* Extern objects. */
864 struct extern_obj *elem;
866 elem = TAILQ_FIRST(&p->extern_objs);
870 TAILQ_REMOVE(&p->extern_objs, elem, node);
872 elem->type->destructor(elem->obj);
878 struct extern_type *elem;
880 elem = TAILQ_FIRST(&p->extern_types);
884 TAILQ_REMOVE(&p->extern_types, elem, node);
887 struct extern_type_member_func *func;
889 func = TAILQ_FIRST(&elem->funcs);
893 TAILQ_REMOVE(&elem->funcs, func, node);
904 static struct extern_func *
905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
907 struct extern_func *elem;
909 TAILQ_FOREACH(elem, &p->extern_funcs, node)
910 if (strcmp(elem->name, name) == 0)
916 static struct extern_func *
917 extern_func_parse(struct rte_swx_pipeline *p,
920 if (name[0] != 'f' || name[1] != '.')
923 return extern_func_find(p, &name[2]);
926 static struct field *
927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
929 struct extern_func **function)
931 struct extern_func *func;
933 char *func_name, *field_name;
935 if ((name[0] != 'f') || (name[1] != '.'))
938 func_name = strdup(&name[2]);
942 field_name = strchr(func_name, '.');
951 func = extern_func_find(p, func_name);
957 f = struct_type_field_find(func->mailbox_struct_type, field_name);
971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
973 const char *mailbox_struct_type_name,
974 rte_swx_extern_func_t func)
976 struct extern_func *f;
977 struct struct_type *mailbox_struct_type;
981 CHECK_NAME(name, EINVAL);
982 CHECK(!extern_func_find(p, name), EEXIST);
984 CHECK_NAME(mailbox_struct_type_name, EINVAL);
985 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
986 CHECK(mailbox_struct_type, EINVAL);
987 CHECK(!mailbox_struct_type->var_size, EINVAL);
991 /* Node allocation. */
992 f = calloc(1, sizeof(struct extern_func));
995 /* Node initialization. */
996 strcpy(f->name, name);
997 f->mailbox_struct_type = mailbox_struct_type;
999 f->struct_id = p->n_structs;
1000 f->id = p->n_extern_funcs;
1002 /* Node add to tailq. */
1003 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004 p->n_extern_funcs++;
1011 extern_func_build(struct rte_swx_pipeline *p)
1015 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016 struct thread *t = &p->threads[i];
1017 struct extern_func *func;
1019 /* Memory allocation. */
1020 t->extern_funcs = calloc(p->n_extern_funcs,
1021 sizeof(struct extern_func_runtime));
1022 CHECK(t->extern_funcs, ENOMEM);
1024 /* Extern function. */
1025 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026 struct extern_func_runtime *r =
1027 &t->extern_funcs[func->id];
1028 uint32_t mailbox_size =
1029 func->mailbox_struct_type->n_bits / 8;
1031 r->func = func->func;
1033 r->mailbox = calloc(1, mailbox_size);
1034 CHECK(r->mailbox, ENOMEM);
1036 t->structs[func->struct_id] = r->mailbox;
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1048 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049 struct thread *t = &p->threads[i];
1052 if (!t->extern_funcs)
1055 for (j = 0; j < p->n_extern_funcs; j++) {
1056 struct extern_func_runtime *r = &t->extern_funcs[j];
1061 free(t->extern_funcs);
1062 t->extern_funcs = NULL;
1067 extern_func_free(struct rte_swx_pipeline *p)
1069 extern_func_build_free(p);
1072 struct extern_func *elem;
1074 elem = TAILQ_FIRST(&p->extern_funcs);
1078 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1089 struct header *elem;
1091 TAILQ_FOREACH(elem, &p->headers, node)
1092 if (strcmp(elem->name, name) == 0)
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1101 struct header *elem;
1103 TAILQ_FOREACH(elem, &p->headers, node)
1104 if (elem->struct_id == struct_id)
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1114 if (name[0] != 'h' || name[1] != '.')
1117 return header_find(p, &name[2]);
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1123 struct header **header)
1127 char *header_name, *field_name;
1129 if ((name[0] != 'h') || (name[1] != '.'))
1132 header_name = strdup(&name[2]);
1136 field_name = strchr(header_name, '.');
1145 h = header_find(p, header_name);
1151 f = struct_type_field_find(h->st, field_name);
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1167 const char *struct_type_name)
1169 struct struct_type *st;
1171 size_t n_headers_max;
1174 CHECK_NAME(name, EINVAL);
1175 CHECK_NAME(struct_type_name, EINVAL);
1177 CHECK(!header_find(p, name), EEXIST);
1179 st = struct_type_find(p, struct_type_name);
1182 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183 CHECK(p->n_headers < n_headers_max, ENOSPC);
1185 /* Node allocation. */
1186 h = calloc(1, sizeof(struct header));
1189 /* Node initialization. */
1190 strcpy(h->name, name);
1192 h->struct_id = p->n_structs;
1193 h->id = p->n_headers;
1195 /* Node add to tailq. */
1196 TAILQ_INSERT_TAIL(&p->headers, h, node);
1204 header_build(struct rte_swx_pipeline *p)
1207 uint32_t n_bytes = 0, i;
1209 TAILQ_FOREACH(h, &p->headers, node) {
1210 n_bytes += h->st->n_bits / 8;
1213 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214 struct thread *t = &p->threads[i];
1215 uint32_t offset = 0;
1217 t->headers = calloc(p->n_headers,
1218 sizeof(struct header_runtime));
1219 CHECK(t->headers, ENOMEM);
1221 t->headers_out = calloc(p->n_headers,
1222 sizeof(struct header_out_runtime));
1223 CHECK(t->headers_out, ENOMEM);
1225 t->header_storage = calloc(1, n_bytes);
1226 CHECK(t->header_storage, ENOMEM);
1228 t->header_out_storage = calloc(1, n_bytes);
1229 CHECK(t->header_out_storage, ENOMEM);
1231 TAILQ_FOREACH(h, &p->headers, node) {
1232 uint8_t *header_storage;
1233 uint32_t n_bytes = h->st->n_bits / 8;
1235 header_storage = &t->header_storage[offset];
1238 t->headers[h->id].ptr0 = header_storage;
1239 t->headers[h->id].n_bytes = n_bytes;
1241 t->structs[h->struct_id] = header_storage;
1249 header_build_free(struct rte_swx_pipeline *p)
1253 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254 struct thread *t = &p->threads[i];
1256 free(t->headers_out);
1257 t->headers_out = NULL;
1262 free(t->header_out_storage);
1263 t->header_out_storage = NULL;
1265 free(t->header_storage);
1266 t->header_storage = NULL;
1271 header_free(struct rte_swx_pipeline *p)
1273 header_build_free(p);
1276 struct header *elem;
1278 elem = TAILQ_FIRST(&p->headers);
1282 TAILQ_REMOVE(&p->headers, elem, node);
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1293 if (!p->metadata_st)
1296 if (name[0] != 'm' || name[1] != '.')
1299 return struct_type_field_find(p->metadata_st, &name[2]);
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304 const char *struct_type_name)
1306 struct struct_type *st = NULL;
1310 CHECK_NAME(struct_type_name, EINVAL);
1311 st = struct_type_find(p, struct_type_name);
1313 CHECK(!st->var_size, EINVAL);
1314 CHECK(!p->metadata_st, EINVAL);
1316 p->metadata_st = st;
1317 p->metadata_struct_id = p->n_structs;
1325 metadata_build(struct rte_swx_pipeline *p)
1327 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1330 /* Thread-level initialization. */
1331 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332 struct thread *t = &p->threads[i];
1335 metadata = calloc(1, n_bytes);
1336 CHECK(metadata, ENOMEM);
1338 t->metadata = metadata;
1339 t->structs[p->metadata_struct_id] = metadata;
1346 metadata_build_free(struct rte_swx_pipeline *p)
1350 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351 struct thread *t = &p->threads[i];
1359 metadata_free(struct rte_swx_pipeline *p)
1361 metadata_build_free(p);
1368 instruction_is_tx(enum instruction_type type)
1381 instruction_does_tx(struct instruction *instr)
1383 switch (instr->type) {
1386 case INSTR_HDR_EMIT_TX:
1387 case INSTR_HDR_EMIT2_TX:
1388 case INSTR_HDR_EMIT3_TX:
1389 case INSTR_HDR_EMIT4_TX:
1390 case INSTR_HDR_EMIT5_TX:
1391 case INSTR_HDR_EMIT6_TX:
1392 case INSTR_HDR_EMIT7_TX:
1393 case INSTR_HDR_EMIT8_TX:
1401 instruction_is_jmp(struct instruction *instr)
1403 switch (instr->type) {
1405 case INSTR_JMP_VALID:
1406 case INSTR_JMP_INVALID:
1408 case INSTR_JMP_MISS:
1409 case INSTR_JMP_ACTION_HIT:
1410 case INSTR_JMP_ACTION_MISS:
1412 case INSTR_JMP_EQ_MH:
1413 case INSTR_JMP_EQ_HM:
1414 case INSTR_JMP_EQ_HH:
1415 case INSTR_JMP_EQ_I:
1417 case INSTR_JMP_NEQ_MH:
1418 case INSTR_JMP_NEQ_HM:
1419 case INSTR_JMP_NEQ_HH:
1420 case INSTR_JMP_NEQ_I:
1422 case INSTR_JMP_LT_MH:
1423 case INSTR_JMP_LT_HM:
1424 case INSTR_JMP_LT_HH:
1425 case INSTR_JMP_LT_MI:
1426 case INSTR_JMP_LT_HI:
1428 case INSTR_JMP_GT_MH:
1429 case INSTR_JMP_GT_HM:
1430 case INSTR_JMP_GT_HH:
1431 case INSTR_JMP_GT_MI:
1432 case INSTR_JMP_GT_HI:
1441 instruction_does_thread_yield(struct instruction *instr)
1443 switch (instr->type) {
1446 case INSTR_TABLE_AF:
1447 case INSTR_SELECTOR:
1449 case INSTR_LEARNER_AF:
1450 case INSTR_EXTERN_OBJ:
1451 case INSTR_EXTERN_FUNC:
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463 struct action *action,
1465 uint32_t *struct_id)
1472 struct header *header;
1474 f = header_field_parse(p, name, &header);
1478 *struct_id = header->struct_id;
1484 f = metadata_field_parse(p, name);
1488 *struct_id = p->metadata_struct_id;
1497 f = action_field_parse(action, name);
1507 struct extern_obj *obj;
1509 f = extern_obj_mailbox_field_parse(p, name, &obj);
1513 *struct_id = obj->struct_id;
1519 struct extern_func *func;
1521 f = extern_func_mailbox_field_parse(p, name, &func);
1525 *struct_id = func->struct_id;
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539 struct action *action,
1542 struct instruction *instr,
1543 struct instruction_data *data __rte_unused)
1547 CHECK(!action, EINVAL);
1548 CHECK(n_tokens == 2, EINVAL);
1550 f = metadata_field_parse(p, tokens[1]);
1553 instr->type = INSTR_RX;
1554 instr->io.io.offset = f->offset / 8;
1555 instr->io.io.n_bits = f->n_bits;
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564 struct action *action __rte_unused,
1567 struct instruction *instr,
1568 struct instruction_data *data __rte_unused)
1570 char *port = tokens[1];
1574 CHECK(n_tokens == 2, EINVAL);
1576 f = metadata_field_parse(p, port);
1578 instr->type = INSTR_TX;
1579 instr->io.io.offset = f->offset / 8;
1580 instr->io.io.n_bits = f->n_bits;
1585 port_val = strtoul(port, &port, 0);
1586 CHECK(!port[0], EINVAL);
1588 instr->type = INSTR_TX_I;
1589 instr->io.io.val = port_val;
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595 struct action *action __rte_unused,
1596 char **tokens __rte_unused,
1598 struct instruction *instr,
1599 struct instruction_data *data __rte_unused)
1601 CHECK(n_tokens == 1, EINVAL);
1604 instr->type = INSTR_TX_I;
1605 instr->io.io.val = p->n_ports_out - 1;
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1612 struct thread *t = &p->threads[p->thread_id];
1613 struct instruction *ip = t->ip;
1615 __instr_tx_exec(p, t, ip);
1618 thread_ip_reset(p, t);
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1625 struct thread *t = &p->threads[p->thread_id];
1626 struct instruction *ip = t->ip;
1628 __instr_tx_i_exec(p, t, ip);
1631 thread_ip_reset(p, t);
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640 struct action *action,
1643 struct instruction *instr,
1644 struct instruction_data *data __rte_unused)
1648 CHECK(!action, EINVAL);
1649 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1651 h = header_parse(p, tokens[1]);
1654 if (n_tokens == 2) {
1655 CHECK(!h->st->var_size, EINVAL);
1657 instr->type = INSTR_HDR_EXTRACT;
1658 instr->io.hdr.header_id[0] = h->id;
1659 instr->io.hdr.struct_id[0] = h->struct_id;
1660 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1664 CHECK(h->st->var_size, EINVAL);
1666 mf = metadata_field_parse(p, tokens[2]);
1668 CHECK(!mf->var_size, EINVAL);
1670 instr->type = INSTR_HDR_EXTRACT_M;
1671 instr->io.io.offset = mf->offset / 8;
1672 instr->io.io.n_bits = mf->n_bits;
1673 instr->io.hdr.header_id[0] = h->id;
1674 instr->io.hdr.struct_id[0] = h->struct_id;
1675 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683 struct action *action,
1686 struct instruction *instr,
1687 struct instruction_data *data __rte_unused)
1691 CHECK(!action, EINVAL);
1692 CHECK(n_tokens == 2, EINVAL);
1694 h = header_parse(p, tokens[1]);
1696 CHECK(!h->st->var_size, EINVAL);
1698 instr->type = INSTR_HDR_LOOKAHEAD;
1699 instr->io.hdr.header_id[0] = h->id;
1700 instr->io.hdr.struct_id[0] = h->struct_id;
1701 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1709 struct thread *t = &p->threads[p->thread_id];
1710 struct instruction *ip = t->ip;
1712 __instr_hdr_extract_exec(p, t, ip);
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1721 struct thread *t = &p->threads[p->thread_id];
1722 struct instruction *ip = t->ip;
1724 __instr_hdr_extract2_exec(p, t, ip);
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1733 struct thread *t = &p->threads[p->thread_id];
1734 struct instruction *ip = t->ip;
1736 __instr_hdr_extract3_exec(p, t, ip);
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1745 struct thread *t = &p->threads[p->thread_id];
1746 struct instruction *ip = t->ip;
1748 __instr_hdr_extract4_exec(p, t, ip);
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1757 struct thread *t = &p->threads[p->thread_id];
1758 struct instruction *ip = t->ip;
1760 __instr_hdr_extract5_exec(p, t, ip);
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1769 struct thread *t = &p->threads[p->thread_id];
1770 struct instruction *ip = t->ip;
1772 __instr_hdr_extract6_exec(p, t, ip);
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1781 struct thread *t = &p->threads[p->thread_id];
1782 struct instruction *ip = t->ip;
1784 __instr_hdr_extract7_exec(p, t, ip);
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1793 struct thread *t = &p->threads[p->thread_id];
1794 struct instruction *ip = t->ip;
1796 __instr_hdr_extract8_exec(p, t, ip);
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1805 struct thread *t = &p->threads[p->thread_id];
1806 struct instruction *ip = t->ip;
1808 __instr_hdr_extract_m_exec(p, t, ip);
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1817 struct thread *t = &p->threads[p->thread_id];
1818 struct instruction *ip = t->ip;
1820 __instr_hdr_lookahead_exec(p, t, ip);
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831 struct action *action __rte_unused,
1834 struct instruction *instr,
1835 struct instruction_data *data __rte_unused)
1839 CHECK(n_tokens == 2, EINVAL);
1841 h = header_parse(p, tokens[1]);
1844 instr->type = INSTR_HDR_EMIT;
1845 instr->io.hdr.header_id[0] = h->id;
1846 instr->io.hdr.struct_id[0] = h->struct_id;
1847 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1854 struct thread *t = &p->threads[p->thread_id];
1855 struct instruction *ip = t->ip;
1857 __instr_hdr_emit_exec(p, t, ip);
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1866 struct thread *t = &p->threads[p->thread_id];
1867 struct instruction *ip = t->ip;
1869 __instr_hdr_emit_tx_exec(p, t, ip);
1872 thread_ip_reset(p, t);
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1879 struct thread *t = &p->threads[p->thread_id];
1880 struct instruction *ip = t->ip;
1882 __instr_hdr_emit2_tx_exec(p, t, ip);
1885 thread_ip_reset(p, t);
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1892 struct thread *t = &p->threads[p->thread_id];
1893 struct instruction *ip = t->ip;
1895 __instr_hdr_emit3_tx_exec(p, t, ip);
1898 thread_ip_reset(p, t);
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1905 struct thread *t = &p->threads[p->thread_id];
1906 struct instruction *ip = t->ip;
1908 __instr_hdr_emit4_tx_exec(p, t, ip);
1911 thread_ip_reset(p, t);
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1918 struct thread *t = &p->threads[p->thread_id];
1919 struct instruction *ip = t->ip;
1921 __instr_hdr_emit5_tx_exec(p, t, ip);
1924 thread_ip_reset(p, t);
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1931 struct thread *t = &p->threads[p->thread_id];
1932 struct instruction *ip = t->ip;
1934 __instr_hdr_emit6_tx_exec(p, t, ip);
1937 thread_ip_reset(p, t);
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1944 struct thread *t = &p->threads[p->thread_id];
1945 struct instruction *ip = t->ip;
1947 __instr_hdr_emit7_tx_exec(p, t, ip);
1950 thread_ip_reset(p, t);
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1957 struct thread *t = &p->threads[p->thread_id];
1958 struct instruction *ip = t->ip;
1960 __instr_hdr_emit8_tx_exec(p, t, ip);
1963 thread_ip_reset(p, t);
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972 struct action *action __rte_unused,
1975 struct instruction *instr,
1976 struct instruction_data *data __rte_unused)
1980 CHECK(n_tokens == 2, EINVAL);
1982 h = header_parse(p, tokens[1]);
1985 instr->type = INSTR_HDR_VALIDATE;
1986 instr->valid.header_id = h->id;
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1993 struct thread *t = &p->threads[p->thread_id];
1994 struct instruction *ip = t->ip;
1996 __instr_hdr_validate_exec(p, t, ip);
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007 struct action *action __rte_unused,
2010 struct instruction *instr,
2011 struct instruction_data *data __rte_unused)
2015 CHECK(n_tokens == 2, EINVAL);
2017 h = header_parse(p, tokens[1]);
2020 instr->type = INSTR_HDR_INVALIDATE;
2021 instr->valid.header_id = h->id;
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2028 struct thread *t = &p->threads[p->thread_id];
2029 struct instruction *ip = t->ip;
2031 __instr_hdr_invalidate_exec(p, t, ip);
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051 struct action *action,
2054 struct instruction *instr,
2055 struct instruction_data *data __rte_unused)
2061 CHECK(!action, EINVAL);
2062 CHECK(n_tokens == 2, EINVAL);
2064 t = table_find(p, tokens[1]);
2066 instr->type = INSTR_TABLE;
2067 instr->table.table_id = t->id;
2071 s = selector_find(p, tokens[1]);
2073 instr->type = INSTR_SELECTOR;
2074 instr->table.table_id = s->id;
2078 l = learner_find(p, tokens[1]);
2080 instr->type = INSTR_LEARNER;
2081 instr->table.table_id = l->id;
2089 instr_table_exec(struct rte_swx_pipeline *p)
2091 struct thread *t = &p->threads[p->thread_id];
2092 struct instruction *ip = t->ip;
2093 uint32_t table_id = ip->table.table_id;
2094 struct rte_swx_table_state *ts = &t->table_state[table_id];
2095 struct table_runtime *table = &t->tables[table_id];
2096 struct table_statistics *stats = &p->table_stats[table_id];
2097 uint64_t action_id, n_pkts_hit, n_pkts_action;
2098 uint8_t *action_data;
2102 done = table->func(ts->obj,
2110 TRACE("[Thread %2u] table %u (not finalized)\n",
2118 action_id = hit ? action_id : ts->default_action_id;
2119 action_data = hit ? action_data : ts->default_action_data;
2120 n_pkts_hit = stats->n_pkts_hit[hit];
2121 n_pkts_action = stats->n_pkts_action[action_id];
2123 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2126 hit ? "hit" : "miss",
2127 (uint32_t)action_id);
2129 t->action_id = action_id;
2130 t->structs[0] = action_data;
2132 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2136 thread_ip_action_call(p, t, action_id);
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2142 struct thread *t = &p->threads[p->thread_id];
2143 struct instruction *ip = t->ip;
2144 uint32_t table_id = ip->table.table_id;
2145 struct rte_swx_table_state *ts = &t->table_state[table_id];
2146 struct table_runtime *table = &t->tables[table_id];
2147 struct table_statistics *stats = &p->table_stats[table_id];
2148 uint64_t action_id, n_pkts_hit, n_pkts_action;
2149 uint8_t *action_data;
2150 action_func_t action_func;
2154 done = table->func(ts->obj,
2162 TRACE("[Thread %2u] table %u (not finalized)\n",
2170 action_id = hit ? action_id : ts->default_action_id;
2171 action_data = hit ? action_data : ts->default_action_data;
2172 action_func = p->action_funcs[action_id];
2173 n_pkts_hit = stats->n_pkts_hit[hit];
2174 n_pkts_action = stats->n_pkts_action[action_id];
2176 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2179 hit ? "hit" : "miss",
2180 (uint32_t)action_id);
2182 t->action_id = action_id;
2183 t->structs[0] = action_data;
2185 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2198 struct thread *t = &p->threads[p->thread_id];
2199 struct instruction *ip = t->ip;
2200 uint32_t selector_id = ip->table.table_id;
2201 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202 struct selector_runtime *selector = &t->selectors[selector_id];
2203 struct selector_statistics *stats = &p->selector_stats[selector_id];
2204 uint64_t n_pkts = stats->n_pkts;
2208 done = rte_swx_table_selector_select(ts->obj,
2210 selector->group_id_buffer,
2211 selector->selector_buffer,
2212 selector->member_id_buffer);
2215 TRACE("[Thread %2u] selector %u (not finalized)\n",
2224 TRACE("[Thread %2u] selector %u\n",
2228 stats->n_pkts = n_pkts + 1;
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2237 struct thread *t = &p->threads[p->thread_id];
2238 struct instruction *ip = t->ip;
2239 uint32_t learner_id = ip->table.table_id;
2240 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241 p->n_selectors + learner_id];
2242 struct learner_runtime *l = &t->learners[learner_id];
2243 struct learner_statistics *stats = &p->learner_stats[learner_id];
2244 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245 uint8_t *action_data;
2249 time = rte_get_tsc_cycles();
2251 done = rte_swx_table_learner_lookup(ts->obj,
2260 TRACE("[Thread %2u] learner %u (not finalized)\n",
2268 action_id = hit ? action_id : ts->default_action_id;
2269 action_data = hit ? action_data : ts->default_action_data;
2270 n_pkts_hit = stats->n_pkts_hit[hit];
2271 n_pkts_action = stats->n_pkts_action[action_id];
2273 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2276 hit ? "hit" : "miss",
2277 (uint32_t)action_id);
2279 t->action_id = action_id;
2280 t->structs[0] = action_data;
2282 t->learner_id = learner_id;
2284 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2288 thread_ip_action_call(p, t, action_id);
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2294 struct thread *t = &p->threads[p->thread_id];
2295 struct instruction *ip = t->ip;
2296 uint32_t learner_id = ip->table.table_id;
2297 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298 p->n_selectors + learner_id];
2299 struct learner_runtime *l = &t->learners[learner_id];
2300 struct learner_statistics *stats = &p->learner_stats[learner_id];
2301 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302 uint8_t *action_data;
2303 action_func_t action_func;
2307 time = rte_get_tsc_cycles();
2309 done = rte_swx_table_learner_lookup(ts->obj,
2318 TRACE("[Thread %2u] learner %u (not finalized)\n",
2326 action_id = hit ? action_id : ts->default_action_id;
2327 action_data = hit ? action_data : ts->default_action_data;
2328 action_func = p->action_funcs[action_id];
2329 n_pkts_hit = stats->n_pkts_hit[hit];
2330 n_pkts_action = stats->n_pkts_action[action_id];
2332 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2335 hit ? "hit" : "miss",
2336 (uint32_t)action_id);
2338 t->action_id = action_id;
2339 t->structs[0] = action_data;
2341 t->learner_id = learner_id;
2343 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2347 thread_ip_action_call(p, t, action_id);
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2360 action_has_nbo_args(struct action *a);
2363 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2366 instr_learn_translate(struct rte_swx_pipeline *p,
2367 struct action *action,
2370 struct instruction *instr,
2371 struct instruction_data *data __rte_unused)
2374 const char *mf_name;
2375 uint32_t mf_offset = 0;
2377 CHECK(action, EINVAL);
2378 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2380 a = action_find(p, tokens[1]);
2382 CHECK(!action_has_nbo_args(a), EINVAL);
2384 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2385 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2390 mf = metadata_field_parse(p, mf_name);
2393 mf_offset = mf->offset / 8;
2396 instr->type = INSTR_LEARNER_LEARN;
2397 instr->learn.action_id = a->id;
2398 instr->learn.mf_offset = mf_offset;
2404 instr_learn_exec(struct rte_swx_pipeline *p)
2406 struct thread *t = &p->threads[p->thread_id];
2407 struct instruction *ip = t->ip;
2409 __instr_learn_exec(p, t, ip);
2419 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2420 struct action *action,
2421 char **tokens __rte_unused,
2423 struct instruction *instr,
2424 struct instruction_data *data __rte_unused)
2426 CHECK(action, EINVAL);
2427 CHECK(n_tokens == 1, EINVAL);
2429 instr->type = INSTR_LEARNER_FORGET;
2435 instr_forget_exec(struct rte_swx_pipeline *p)
2437 struct thread *t = &p->threads[p->thread_id];
2438 struct instruction *ip = t->ip;
2440 __instr_forget_exec(p, t, ip);
2450 instr_extern_translate(struct rte_swx_pipeline *p,
2451 struct action *action __rte_unused,
2454 struct instruction *instr,
2455 struct instruction_data *data __rte_unused)
2457 char *token = tokens[1];
2459 CHECK(n_tokens == 2, EINVAL);
2461 if (token[0] == 'e') {
2462 struct extern_obj *obj;
2463 struct extern_type_member_func *func;
2465 func = extern_obj_member_func_parse(p, token, &obj);
2466 CHECK(func, EINVAL);
2468 instr->type = INSTR_EXTERN_OBJ;
2469 instr->ext_obj.ext_obj_id = obj->id;
2470 instr->ext_obj.func_id = func->id;
2475 if (token[0] == 'f') {
2476 struct extern_func *func;
2478 func = extern_func_parse(p, token);
2479 CHECK(func, EINVAL);
2481 instr->type = INSTR_EXTERN_FUNC;
2482 instr->ext_func.ext_func_id = func->id;
2491 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2493 struct thread *t = &p->threads[p->thread_id];
2494 struct instruction *ip = t->ip;
2497 /* Extern object member function execute. */
2498 done = __instr_extern_obj_exec(p, t, ip);
2501 thread_ip_inc_cond(t, done);
2502 thread_yield_cond(p, done ^ 1);
2506 instr_extern_func_exec(struct rte_swx_pipeline *p)
2508 struct thread *t = &p->threads[p->thread_id];
2509 struct instruction *ip = t->ip;
2512 /* Extern function execute. */
2513 done = __instr_extern_func_exec(p, t, ip);
2516 thread_ip_inc_cond(t, done);
2517 thread_yield_cond(p, done ^ 1);
2524 instr_mov_translate(struct rte_swx_pipeline *p,
2525 struct action *action,
2528 struct instruction *instr,
2529 struct instruction_data *data __rte_unused)
2531 char *dst = tokens[1], *src = tokens[2];
2532 struct field *fdst, *fsrc;
2534 uint32_t dst_struct_id = 0, src_struct_id = 0;
2536 CHECK(n_tokens == 3, EINVAL);
2538 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2539 CHECK(fdst, EINVAL);
2540 CHECK(!fdst->var_size, EINVAL);
2542 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2543 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2545 CHECK(!fsrc->var_size, EINVAL);
2547 instr->type = INSTR_MOV;
2548 if (dst[0] != 'h' && src[0] == 'h')
2549 instr->type = INSTR_MOV_MH;
2550 if (dst[0] == 'h' && src[0] != 'h')
2551 instr->type = INSTR_MOV_HM;
2552 if (dst[0] == 'h' && src[0] == 'h')
2553 instr->type = INSTR_MOV_HH;
2555 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2556 instr->mov.dst.n_bits = fdst->n_bits;
2557 instr->mov.dst.offset = fdst->offset / 8;
2558 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2559 instr->mov.src.n_bits = fsrc->n_bits;
2560 instr->mov.src.offset = fsrc->offset / 8;
2565 src_val = strtoull(src, &src, 0);
2566 CHECK(!src[0], EINVAL);
2569 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2571 instr->type = INSTR_MOV_I;
2572 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2573 instr->mov.dst.n_bits = fdst->n_bits;
2574 instr->mov.dst.offset = fdst->offset / 8;
2575 instr->mov.src_val = src_val;
2580 instr_mov_exec(struct rte_swx_pipeline *p)
2582 struct thread *t = &p->threads[p->thread_id];
2583 struct instruction *ip = t->ip;
2585 __instr_mov_exec(p, t, ip);
2592 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2594 struct thread *t = &p->threads[p->thread_id];
2595 struct instruction *ip = t->ip;
2597 __instr_mov_mh_exec(p, t, ip);
2604 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2606 struct thread *t = &p->threads[p->thread_id];
2607 struct instruction *ip = t->ip;
2609 __instr_mov_hm_exec(p, t, ip);
2616 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2618 struct thread *t = &p->threads[p->thread_id];
2619 struct instruction *ip = t->ip;
2621 __instr_mov_hh_exec(p, t, ip);
2628 instr_mov_i_exec(struct rte_swx_pipeline *p)
2630 struct thread *t = &p->threads[p->thread_id];
2631 struct instruction *ip = t->ip;
2633 __instr_mov_i_exec(p, t, ip);
2643 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2645 struct thread *t = &p->threads[p->thread_id];
2646 struct instruction *ip = t->ip;
2648 __instr_dma_ht_exec(p, t, ip);
2655 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2657 struct thread *t = &p->threads[p->thread_id];
2658 struct instruction *ip = t->ip;
2660 __instr_dma_ht2_exec(p, t, ip);
2667 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2669 struct thread *t = &p->threads[p->thread_id];
2670 struct instruction *ip = t->ip;
2672 __instr_dma_ht3_exec(p, t, ip);
2679 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2681 struct thread *t = &p->threads[p->thread_id];
2682 struct instruction *ip = t->ip;
2684 __instr_dma_ht4_exec(p, t, ip);
2691 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2693 struct thread *t = &p->threads[p->thread_id];
2694 struct instruction *ip = t->ip;
2696 __instr_dma_ht5_exec(p, t, ip);
2703 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2705 struct thread *t = &p->threads[p->thread_id];
2706 struct instruction *ip = t->ip;
2708 __instr_dma_ht6_exec(p, t, ip);
2715 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2717 struct thread *t = &p->threads[p->thread_id];
2718 struct instruction *ip = t->ip;
2720 __instr_dma_ht7_exec(p, t, ip);
2727 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2729 struct thread *t = &p->threads[p->thread_id];
2730 struct instruction *ip = t->ip;
2732 __instr_dma_ht8_exec(p, t, ip);
2742 instr_alu_add_translate(struct rte_swx_pipeline *p,
2743 struct action *action,
2746 struct instruction *instr,
2747 struct instruction_data *data __rte_unused)
2749 char *dst = tokens[1], *src = tokens[2];
2750 struct field *fdst, *fsrc;
2752 uint32_t dst_struct_id = 0, src_struct_id = 0;
2754 CHECK(n_tokens == 3, EINVAL);
2756 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2757 CHECK(fdst, EINVAL);
2758 CHECK(!fdst->var_size, EINVAL);
2760 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2761 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2763 CHECK(!fsrc->var_size, EINVAL);
2765 instr->type = INSTR_ALU_ADD;
2766 if (dst[0] == 'h' && src[0] != 'h')
2767 instr->type = INSTR_ALU_ADD_HM;
2768 if (dst[0] != 'h' && src[0] == 'h')
2769 instr->type = INSTR_ALU_ADD_MH;
2770 if (dst[0] == 'h' && src[0] == 'h')
2771 instr->type = INSTR_ALU_ADD_HH;
2773 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774 instr->alu.dst.n_bits = fdst->n_bits;
2775 instr->alu.dst.offset = fdst->offset / 8;
2776 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2777 instr->alu.src.n_bits = fsrc->n_bits;
2778 instr->alu.src.offset = fsrc->offset / 8;
2782 /* ADD_MI, ADD_HI. */
2783 src_val = strtoull(src, &src, 0);
2784 CHECK(!src[0], EINVAL);
2786 instr->type = INSTR_ALU_ADD_MI;
2788 instr->type = INSTR_ALU_ADD_HI;
2790 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2791 instr->alu.dst.n_bits = fdst->n_bits;
2792 instr->alu.dst.offset = fdst->offset / 8;
2793 instr->alu.src_val = src_val;
2798 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2799 struct action *action,
2802 struct instruction *instr,
2803 struct instruction_data *data __rte_unused)
2805 char *dst = tokens[1], *src = tokens[2];
2806 struct field *fdst, *fsrc;
2808 uint32_t dst_struct_id = 0, src_struct_id = 0;
2810 CHECK(n_tokens == 3, EINVAL);
2812 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2813 CHECK(fdst, EINVAL);
2814 CHECK(!fdst->var_size, EINVAL);
2816 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2817 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2819 CHECK(!fsrc->var_size, EINVAL);
2821 instr->type = INSTR_ALU_SUB;
2822 if (dst[0] == 'h' && src[0] != 'h')
2823 instr->type = INSTR_ALU_SUB_HM;
2824 if (dst[0] != 'h' && src[0] == 'h')
2825 instr->type = INSTR_ALU_SUB_MH;
2826 if (dst[0] == 'h' && src[0] == 'h')
2827 instr->type = INSTR_ALU_SUB_HH;
2829 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2830 instr->alu.dst.n_bits = fdst->n_bits;
2831 instr->alu.dst.offset = fdst->offset / 8;
2832 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2833 instr->alu.src.n_bits = fsrc->n_bits;
2834 instr->alu.src.offset = fsrc->offset / 8;
2838 /* SUB_MI, SUB_HI. */
2839 src_val = strtoull(src, &src, 0);
2840 CHECK(!src[0], EINVAL);
2842 instr->type = INSTR_ALU_SUB_MI;
2844 instr->type = INSTR_ALU_SUB_HI;
2846 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2847 instr->alu.dst.n_bits = fdst->n_bits;
2848 instr->alu.dst.offset = fdst->offset / 8;
2849 instr->alu.src_val = src_val;
2854 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2855 struct action *action __rte_unused,
2858 struct instruction *instr,
2859 struct instruction_data *data __rte_unused)
2861 char *dst = tokens[1], *src = tokens[2];
2862 struct header *hdst, *hsrc;
2863 struct field *fdst, *fsrc;
2865 CHECK(n_tokens == 3, EINVAL);
2867 fdst = header_field_parse(p, dst, &hdst);
2868 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2869 CHECK(!fdst->var_size, EINVAL);
2872 fsrc = header_field_parse(p, src, &hsrc);
2874 CHECK(!fsrc->var_size, EINVAL);
2876 instr->type = INSTR_ALU_CKADD_FIELD;
2877 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878 instr->alu.dst.n_bits = fdst->n_bits;
2879 instr->alu.dst.offset = fdst->offset / 8;
2880 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881 instr->alu.src.n_bits = fsrc->n_bits;
2882 instr->alu.src.offset = fsrc->offset / 8;
2886 /* CKADD_STRUCT, CKADD_STRUCT20. */
2887 hsrc = header_parse(p, src);
2888 CHECK(hsrc, EINVAL);
2889 CHECK(!hsrc->st->var_size, EINVAL);
2891 instr->type = INSTR_ALU_CKADD_STRUCT;
2892 if ((hsrc->st->n_bits / 8) == 20)
2893 instr->type = INSTR_ALU_CKADD_STRUCT20;
2895 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2896 instr->alu.dst.n_bits = fdst->n_bits;
2897 instr->alu.dst.offset = fdst->offset / 8;
2898 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2899 instr->alu.src.n_bits = hsrc->st->n_bits;
2900 instr->alu.src.offset = 0; /* Unused. */
2905 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2906 struct action *action __rte_unused,
2909 struct instruction *instr,
2910 struct instruction_data *data __rte_unused)
2912 char *dst = tokens[1], *src = tokens[2];
2913 struct header *hdst, *hsrc;
2914 struct field *fdst, *fsrc;
2916 CHECK(n_tokens == 3, EINVAL);
2918 fdst = header_field_parse(p, dst, &hdst);
2919 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2920 CHECK(!fdst->var_size, EINVAL);
2922 fsrc = header_field_parse(p, src, &hsrc);
2923 CHECK(fsrc, EINVAL);
2924 CHECK(!fsrc->var_size, EINVAL);
2926 instr->type = INSTR_ALU_CKSUB_FIELD;
2927 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2928 instr->alu.dst.n_bits = fdst->n_bits;
2929 instr->alu.dst.offset = fdst->offset / 8;
2930 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2931 instr->alu.src.n_bits = fsrc->n_bits;
2932 instr->alu.src.offset = fsrc->offset / 8;
2937 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2938 struct action *action,
2941 struct instruction *instr,
2942 struct instruction_data *data __rte_unused)
2944 char *dst = tokens[1], *src = tokens[2];
2945 struct field *fdst, *fsrc;
2947 uint32_t dst_struct_id = 0, src_struct_id = 0;
2949 CHECK(n_tokens == 3, EINVAL);
2951 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2952 CHECK(fdst, EINVAL);
2953 CHECK(!fdst->var_size, EINVAL);
2955 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2956 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2958 CHECK(!fsrc->var_size, EINVAL);
2960 instr->type = INSTR_ALU_SHL;
2961 if (dst[0] == 'h' && src[0] != 'h')
2962 instr->type = INSTR_ALU_SHL_HM;
2963 if (dst[0] != 'h' && src[0] == 'h')
2964 instr->type = INSTR_ALU_SHL_MH;
2965 if (dst[0] == 'h' && src[0] == 'h')
2966 instr->type = INSTR_ALU_SHL_HH;
2968 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969 instr->alu.dst.n_bits = fdst->n_bits;
2970 instr->alu.dst.offset = fdst->offset / 8;
2971 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2972 instr->alu.src.n_bits = fsrc->n_bits;
2973 instr->alu.src.offset = fsrc->offset / 8;
2977 /* SHL_MI, SHL_HI. */
2978 src_val = strtoull(src, &src, 0);
2979 CHECK(!src[0], EINVAL);
2981 instr->type = INSTR_ALU_SHL_MI;
2983 instr->type = INSTR_ALU_SHL_HI;
2985 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986 instr->alu.dst.n_bits = fdst->n_bits;
2987 instr->alu.dst.offset = fdst->offset / 8;
2988 instr->alu.src_val = src_val;
2993 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2994 struct action *action,
2997 struct instruction *instr,
2998 struct instruction_data *data __rte_unused)
3000 char *dst = tokens[1], *src = tokens[2];
3001 struct field *fdst, *fsrc;
3003 uint32_t dst_struct_id = 0, src_struct_id = 0;
3005 CHECK(n_tokens == 3, EINVAL);
3007 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3008 CHECK(fdst, EINVAL);
3009 CHECK(!fdst->var_size, EINVAL);
3011 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3012 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3014 CHECK(!fsrc->var_size, EINVAL);
3016 instr->type = INSTR_ALU_SHR;
3017 if (dst[0] == 'h' && src[0] != 'h')
3018 instr->type = INSTR_ALU_SHR_HM;
3019 if (dst[0] != 'h' && src[0] == 'h')
3020 instr->type = INSTR_ALU_SHR_MH;
3021 if (dst[0] == 'h' && src[0] == 'h')
3022 instr->type = INSTR_ALU_SHR_HH;
3024 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025 instr->alu.dst.n_bits = fdst->n_bits;
3026 instr->alu.dst.offset = fdst->offset / 8;
3027 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3028 instr->alu.src.n_bits = fsrc->n_bits;
3029 instr->alu.src.offset = fsrc->offset / 8;
3033 /* SHR_MI, SHR_HI. */
3034 src_val = strtoull(src, &src, 0);
3035 CHECK(!src[0], EINVAL);
3037 instr->type = INSTR_ALU_SHR_MI;
3039 instr->type = INSTR_ALU_SHR_HI;
3041 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042 instr->alu.dst.n_bits = fdst->n_bits;
3043 instr->alu.dst.offset = fdst->offset / 8;
3044 instr->alu.src_val = src_val;
3049 instr_alu_and_translate(struct rte_swx_pipeline *p,
3050 struct action *action,
3053 struct instruction *instr,
3054 struct instruction_data *data __rte_unused)
3056 char *dst = tokens[1], *src = tokens[2];
3057 struct field *fdst, *fsrc;
3059 uint32_t dst_struct_id = 0, src_struct_id = 0;
3061 CHECK(n_tokens == 3, EINVAL);
3063 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3064 CHECK(fdst, EINVAL);
3065 CHECK(!fdst->var_size, EINVAL);
3067 /* AND, AND_MH, AND_HM, AND_HH. */
3068 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3070 CHECK(!fsrc->var_size, EINVAL);
3072 instr->type = INSTR_ALU_AND;
3073 if (dst[0] != 'h' && src[0] == 'h')
3074 instr->type = INSTR_ALU_AND_MH;
3075 if (dst[0] == 'h' && src[0] != 'h')
3076 instr->type = INSTR_ALU_AND_HM;
3077 if (dst[0] == 'h' && src[0] == 'h')
3078 instr->type = INSTR_ALU_AND_HH;
3080 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081 instr->alu.dst.n_bits = fdst->n_bits;
3082 instr->alu.dst.offset = fdst->offset / 8;
3083 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3084 instr->alu.src.n_bits = fsrc->n_bits;
3085 instr->alu.src.offset = fsrc->offset / 8;
3090 src_val = strtoull(src, &src, 0);
3091 CHECK(!src[0], EINVAL);
3094 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3096 instr->type = INSTR_ALU_AND_I;
3097 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3098 instr->alu.dst.n_bits = fdst->n_bits;
3099 instr->alu.dst.offset = fdst->offset / 8;
3100 instr->alu.src_val = src_val;
3105 instr_alu_or_translate(struct rte_swx_pipeline *p,
3106 struct action *action,
3109 struct instruction *instr,
3110 struct instruction_data *data __rte_unused)
3112 char *dst = tokens[1], *src = tokens[2];
3113 struct field *fdst, *fsrc;
3115 uint32_t dst_struct_id = 0, src_struct_id = 0;
3117 CHECK(n_tokens == 3, EINVAL);
3119 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3120 CHECK(fdst, EINVAL);
3121 CHECK(!fdst->var_size, EINVAL);
3123 /* OR, OR_MH, OR_HM, OR_HH. */
3124 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3126 CHECK(!fsrc->var_size, EINVAL);
3128 instr->type = INSTR_ALU_OR;
3129 if (dst[0] != 'h' && src[0] == 'h')
3130 instr->type = INSTR_ALU_OR_MH;
3131 if (dst[0] == 'h' && src[0] != 'h')
3132 instr->type = INSTR_ALU_OR_HM;
3133 if (dst[0] == 'h' && src[0] == 'h')
3134 instr->type = INSTR_ALU_OR_HH;
3136 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137 instr->alu.dst.n_bits = fdst->n_bits;
3138 instr->alu.dst.offset = fdst->offset / 8;
3139 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140 instr->alu.src.n_bits = fsrc->n_bits;
3141 instr->alu.src.offset = fsrc->offset / 8;
3146 src_val = strtoull(src, &src, 0);
3147 CHECK(!src[0], EINVAL);
3150 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3152 instr->type = INSTR_ALU_OR_I;
3153 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154 instr->alu.dst.n_bits = fdst->n_bits;
3155 instr->alu.dst.offset = fdst->offset / 8;
3156 instr->alu.src_val = src_val;
3161 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3162 struct action *action,
3165 struct instruction *instr,
3166 struct instruction_data *data __rte_unused)
3168 char *dst = tokens[1], *src = tokens[2];
3169 struct field *fdst, *fsrc;
3171 uint32_t dst_struct_id = 0, src_struct_id = 0;
3173 CHECK(n_tokens == 3, EINVAL);
3175 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3176 CHECK(fdst, EINVAL);
3177 CHECK(!fdst->var_size, EINVAL);
3179 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3180 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3182 CHECK(!fsrc->var_size, EINVAL);
3184 instr->type = INSTR_ALU_XOR;
3185 if (dst[0] != 'h' && src[0] == 'h')
3186 instr->type = INSTR_ALU_XOR_MH;
3187 if (dst[0] == 'h' && src[0] != 'h')
3188 instr->type = INSTR_ALU_XOR_HM;
3189 if (dst[0] == 'h' && src[0] == 'h')
3190 instr->type = INSTR_ALU_XOR_HH;
3192 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3193 instr->alu.dst.n_bits = fdst->n_bits;
3194 instr->alu.dst.offset = fdst->offset / 8;
3195 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3196 instr->alu.src.n_bits = fsrc->n_bits;
3197 instr->alu.src.offset = fsrc->offset / 8;
3202 src_val = strtoull(src, &src, 0);
3203 CHECK(!src[0], EINVAL);
3206 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3208 instr->type = INSTR_ALU_XOR_I;
3209 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3210 instr->alu.dst.n_bits = fdst->n_bits;
3211 instr->alu.dst.offset = fdst->offset / 8;
3212 instr->alu.src_val = src_val;
3217 instr_alu_add_exec(struct rte_swx_pipeline *p)
3219 struct thread *t = &p->threads[p->thread_id];
3220 struct instruction *ip = t->ip;
3223 __instr_alu_add_exec(p, t, ip);
3230 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3232 struct thread *t = &p->threads[p->thread_id];
3233 struct instruction *ip = t->ip;
3236 __instr_alu_add_mh_exec(p, t, ip);
3243 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3245 struct thread *t = &p->threads[p->thread_id];
3246 struct instruction *ip = t->ip;
3249 __instr_alu_add_hm_exec(p, t, ip);
3256 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3258 struct thread *t = &p->threads[p->thread_id];
3259 struct instruction *ip = t->ip;
3262 __instr_alu_add_hh_exec(p, t, ip);
3269 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3271 struct thread *t = &p->threads[p->thread_id];
3272 struct instruction *ip = t->ip;
3275 __instr_alu_add_mi_exec(p, t, ip);
3282 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3284 struct thread *t = &p->threads[p->thread_id];
3285 struct instruction *ip = t->ip;
3288 __instr_alu_add_hi_exec(p, t, ip);
3295 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3297 struct thread *t = &p->threads[p->thread_id];
3298 struct instruction *ip = t->ip;
3301 __instr_alu_sub_exec(p, t, ip);
3308 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3310 struct thread *t = &p->threads[p->thread_id];
3311 struct instruction *ip = t->ip;
3314 __instr_alu_sub_mh_exec(p, t, ip);
3321 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3323 struct thread *t = &p->threads[p->thread_id];
3324 struct instruction *ip = t->ip;
3327 __instr_alu_sub_hm_exec(p, t, ip);
3334 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3336 struct thread *t = &p->threads[p->thread_id];
3337 struct instruction *ip = t->ip;
3340 __instr_alu_sub_hh_exec(p, t, ip);
3347 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3349 struct thread *t = &p->threads[p->thread_id];
3350 struct instruction *ip = t->ip;
3353 __instr_alu_sub_mi_exec(p, t, ip);
3360 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3362 struct thread *t = &p->threads[p->thread_id];
3363 struct instruction *ip = t->ip;
3366 __instr_alu_sub_hi_exec(p, t, ip);
3373 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3375 struct thread *t = &p->threads[p->thread_id];
3376 struct instruction *ip = t->ip;
3379 __instr_alu_shl_exec(p, t, ip);
3386 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3388 struct thread *t = &p->threads[p->thread_id];
3389 struct instruction *ip = t->ip;
3392 __instr_alu_shl_mh_exec(p, t, ip);
3399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3401 struct thread *t = &p->threads[p->thread_id];
3402 struct instruction *ip = t->ip;
3405 __instr_alu_shl_hm_exec(p, t, ip);
3412 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3414 struct thread *t = &p->threads[p->thread_id];
3415 struct instruction *ip = t->ip;
3418 __instr_alu_shl_hh_exec(p, t, ip);
3425 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3427 struct thread *t = &p->threads[p->thread_id];
3428 struct instruction *ip = t->ip;
3431 __instr_alu_shl_mi_exec(p, t, ip);
3438 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3440 struct thread *t = &p->threads[p->thread_id];
3441 struct instruction *ip = t->ip;
3444 __instr_alu_shl_hi_exec(p, t, ip);
3451 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3453 struct thread *t = &p->threads[p->thread_id];
3454 struct instruction *ip = t->ip;
3457 __instr_alu_shr_exec(p, t, ip);
3464 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3466 struct thread *t = &p->threads[p->thread_id];
3467 struct instruction *ip = t->ip;
3470 __instr_alu_shr_mh_exec(p, t, ip);
3477 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3479 struct thread *t = &p->threads[p->thread_id];
3480 struct instruction *ip = t->ip;
3483 __instr_alu_shr_hm_exec(p, t, ip);
3490 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3492 struct thread *t = &p->threads[p->thread_id];
3493 struct instruction *ip = t->ip;
3496 __instr_alu_shr_hh_exec(p, t, ip);
3503 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3505 struct thread *t = &p->threads[p->thread_id];
3506 struct instruction *ip = t->ip;
3509 __instr_alu_shr_mi_exec(p, t, ip);
3516 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3518 struct thread *t = &p->threads[p->thread_id];
3519 struct instruction *ip = t->ip;
3522 __instr_alu_shr_hi_exec(p, t, ip);
3529 instr_alu_and_exec(struct rte_swx_pipeline *p)
3531 struct thread *t = &p->threads[p->thread_id];
3532 struct instruction *ip = t->ip;
3535 __instr_alu_and_exec(p, t, ip);
3542 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3544 struct thread *t = &p->threads[p->thread_id];
3545 struct instruction *ip = t->ip;
3548 __instr_alu_and_mh_exec(p, t, ip);
3555 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3557 struct thread *t = &p->threads[p->thread_id];
3558 struct instruction *ip = t->ip;
3561 __instr_alu_and_hm_exec(p, t, ip);
3568 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3570 struct thread *t = &p->threads[p->thread_id];
3571 struct instruction *ip = t->ip;
3574 __instr_alu_and_hh_exec(p, t, ip);
3581 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3583 struct thread *t = &p->threads[p->thread_id];
3584 struct instruction *ip = t->ip;
3587 __instr_alu_and_i_exec(p, t, ip);
3594 instr_alu_or_exec(struct rte_swx_pipeline *p)
3596 struct thread *t = &p->threads[p->thread_id];
3597 struct instruction *ip = t->ip;
3600 __instr_alu_or_exec(p, t, ip);
3607 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3609 struct thread *t = &p->threads[p->thread_id];
3610 struct instruction *ip = t->ip;
3613 __instr_alu_or_mh_exec(p, t, ip);
3620 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3622 struct thread *t = &p->threads[p->thread_id];
3623 struct instruction *ip = t->ip;
3626 __instr_alu_or_hm_exec(p, t, ip);
3633 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3635 struct thread *t = &p->threads[p->thread_id];
3636 struct instruction *ip = t->ip;
3639 __instr_alu_or_hh_exec(p, t, ip);
3646 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3648 struct thread *t = &p->threads[p->thread_id];
3649 struct instruction *ip = t->ip;
3652 __instr_alu_or_i_exec(p, t, ip);
3659 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3661 struct thread *t = &p->threads[p->thread_id];
3662 struct instruction *ip = t->ip;
3665 __instr_alu_xor_exec(p, t, ip);
3672 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3674 struct thread *t = &p->threads[p->thread_id];
3675 struct instruction *ip = t->ip;
3678 __instr_alu_xor_mh_exec(p, t, ip);
3685 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3687 struct thread *t = &p->threads[p->thread_id];
3688 struct instruction *ip = t->ip;
3691 __instr_alu_xor_hm_exec(p, t, ip);
3698 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3700 struct thread *t = &p->threads[p->thread_id];
3701 struct instruction *ip = t->ip;
3704 __instr_alu_xor_hh_exec(p, t, ip);
3711 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3713 struct thread *t = &p->threads[p->thread_id];
3714 struct instruction *ip = t->ip;
3717 __instr_alu_xor_i_exec(p, t, ip);
3724 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3726 struct thread *t = &p->threads[p->thread_id];
3727 struct instruction *ip = t->ip;
3730 __instr_alu_ckadd_field_exec(p, t, ip);
3737 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3739 struct thread *t = &p->threads[p->thread_id];
3740 struct instruction *ip = t->ip;
3743 __instr_alu_cksub_field_exec(p, t, ip);
3750 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3752 struct thread *t = &p->threads[p->thread_id];
3753 struct instruction *ip = t->ip;
3756 __instr_alu_ckadd_struct20_exec(p, t, ip);
3763 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3765 struct thread *t = &p->threads[p->thread_id];
3766 struct instruction *ip = t->ip;
3769 __instr_alu_ckadd_struct_exec(p, t, ip);
3778 static struct regarray *
3779 regarray_find(struct rte_swx_pipeline *p, const char *name);
3782 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3783 struct action *action,
3786 struct instruction *instr,
3787 struct instruction_data *data __rte_unused)
3789 char *regarray = tokens[1], *idx = tokens[2];
3792 uint32_t idx_struct_id, idx_val;
3794 CHECK(n_tokens == 3, EINVAL);
3796 r = regarray_find(p, regarray);
3799 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3800 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3802 CHECK(!fidx->var_size, EINVAL);
3804 instr->type = INSTR_REGPREFETCH_RM;
3806 instr->type = INSTR_REGPREFETCH_RH;
3808 instr->regarray.regarray_id = r->id;
3809 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3810 instr->regarray.idx.n_bits = fidx->n_bits;
3811 instr->regarray.idx.offset = fidx->offset / 8;
3812 instr->regarray.dstsrc_val = 0; /* Unused. */
3816 /* REGPREFETCH_RI. */
3817 idx_val = strtoul(idx, &idx, 0);
3818 CHECK(!idx[0], EINVAL);
3820 instr->type = INSTR_REGPREFETCH_RI;
3821 instr->regarray.regarray_id = r->id;
3822 instr->regarray.idx_val = idx_val;
3823 instr->regarray.dstsrc_val = 0; /* Unused. */
3828 instr_regrd_translate(struct rte_swx_pipeline *p,
3829 struct action *action,
3832 struct instruction *instr,
3833 struct instruction_data *data __rte_unused)
3835 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3837 struct field *fdst, *fidx;
3838 uint32_t dst_struct_id, idx_struct_id, idx_val;
3840 CHECK(n_tokens == 4, EINVAL);
3842 r = regarray_find(p, regarray);
3845 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3846 CHECK(fdst, EINVAL);
3847 CHECK(!fdst->var_size, EINVAL);
3849 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3850 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3852 CHECK(!fidx->var_size, EINVAL);
3854 instr->type = INSTR_REGRD_MRM;
3855 if (dst[0] == 'h' && idx[0] != 'h')
3856 instr->type = INSTR_REGRD_HRM;
3857 if (dst[0] != 'h' && idx[0] == 'h')
3858 instr->type = INSTR_REGRD_MRH;
3859 if (dst[0] == 'h' && idx[0] == 'h')
3860 instr->type = INSTR_REGRD_HRH;
3862 instr->regarray.regarray_id = r->id;
3863 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3864 instr->regarray.idx.n_bits = fidx->n_bits;
3865 instr->regarray.idx.offset = fidx->offset / 8;
3866 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3867 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3868 instr->regarray.dstsrc.offset = fdst->offset / 8;
3872 /* REGRD_MRI, REGRD_HRI. */
3873 idx_val = strtoul(idx, &idx, 0);
3874 CHECK(!idx[0], EINVAL);
3876 instr->type = INSTR_REGRD_MRI;
3878 instr->type = INSTR_REGRD_HRI;
3880 instr->regarray.regarray_id = r->id;
3881 instr->regarray.idx_val = idx_val;
3882 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3883 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3884 instr->regarray.dstsrc.offset = fdst->offset / 8;
3889 instr_regwr_translate(struct rte_swx_pipeline *p,
3890 struct action *action,
3893 struct instruction *instr,
3894 struct instruction_data *data __rte_unused)
3896 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3898 struct field *fidx, *fsrc;
3900 uint32_t idx_struct_id, idx_val, src_struct_id;
3902 CHECK(n_tokens == 4, EINVAL);
3904 r = regarray_find(p, regarray);
3907 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3908 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3909 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3911 CHECK(!fidx->var_size, EINVAL);
3912 CHECK(!fsrc->var_size, EINVAL);
3914 instr->type = INSTR_REGWR_RMM;
3915 if (idx[0] == 'h' && src[0] != 'h')
3916 instr->type = INSTR_REGWR_RHM;
3917 if (idx[0] != 'h' && src[0] == 'h')
3918 instr->type = INSTR_REGWR_RMH;
3919 if (idx[0] == 'h' && src[0] == 'h')
3920 instr->type = INSTR_REGWR_RHH;
3922 instr->regarray.regarray_id = r->id;
3923 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3924 instr->regarray.idx.n_bits = fidx->n_bits;
3925 instr->regarray.idx.offset = fidx->offset / 8;
3926 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3927 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3928 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3932 /* REGWR_RHI, REGWR_RMI. */
3933 if (fidx && !fsrc) {
3934 CHECK(!fidx->var_size, EINVAL);
3936 src_val = strtoull(src, &src, 0);
3937 CHECK(!src[0], EINVAL);
3939 instr->type = INSTR_REGWR_RMI;
3941 instr->type = INSTR_REGWR_RHI;
3943 instr->regarray.regarray_id = r->id;
3944 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945 instr->regarray.idx.n_bits = fidx->n_bits;
3946 instr->regarray.idx.offset = fidx->offset / 8;
3947 instr->regarray.dstsrc_val = src_val;
3951 /* REGWR_RIH, REGWR_RIM. */
3952 if (!fidx && fsrc) {
3953 idx_val = strtoul(idx, &idx, 0);
3954 CHECK(!idx[0], EINVAL);
3956 CHECK(!fsrc->var_size, EINVAL);
3958 instr->type = INSTR_REGWR_RIM;
3960 instr->type = INSTR_REGWR_RIH;
3962 instr->regarray.regarray_id = r->id;
3963 instr->regarray.idx_val = idx_val;
3964 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3965 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3966 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3971 src_val = strtoull(src, &src, 0);
3972 CHECK(!src[0], EINVAL);
3974 idx_val = strtoul(idx, &idx, 0);
3975 CHECK(!idx[0], EINVAL);
3977 instr->type = INSTR_REGWR_RII;
3978 instr->regarray.idx_val = idx_val;
3979 instr->regarray.dstsrc_val = src_val;
3985 instr_regadd_translate(struct rte_swx_pipeline *p,
3986 struct action *action,
3989 struct instruction *instr,
3990 struct instruction_data *data __rte_unused)
3992 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3994 struct field *fidx, *fsrc;
3996 uint32_t idx_struct_id, idx_val, src_struct_id;
3998 CHECK(n_tokens == 4, EINVAL);
4000 r = regarray_find(p, regarray);
4003 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4004 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4005 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4007 CHECK(!fidx->var_size, EINVAL);
4008 CHECK(!fsrc->var_size, EINVAL);
4010 instr->type = INSTR_REGADD_RMM;
4011 if (idx[0] == 'h' && src[0] != 'h')
4012 instr->type = INSTR_REGADD_RHM;
4013 if (idx[0] != 'h' && src[0] == 'h')
4014 instr->type = INSTR_REGADD_RMH;
4015 if (idx[0] == 'h' && src[0] == 'h')
4016 instr->type = INSTR_REGADD_RHH;
4018 instr->regarray.regarray_id = r->id;
4019 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4020 instr->regarray.idx.n_bits = fidx->n_bits;
4021 instr->regarray.idx.offset = fidx->offset / 8;
4022 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4023 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4024 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4028 /* REGADD_RHI, REGADD_RMI. */
4029 if (fidx && !fsrc) {
4030 CHECK(!fidx->var_size, EINVAL);
4032 src_val = strtoull(src, &src, 0);
4033 CHECK(!src[0], EINVAL);
4035 instr->type = INSTR_REGADD_RMI;
4037 instr->type = INSTR_REGADD_RHI;
4039 instr->regarray.regarray_id = r->id;
4040 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4041 instr->regarray.idx.n_bits = fidx->n_bits;
4042 instr->regarray.idx.offset = fidx->offset / 8;
4043 instr->regarray.dstsrc_val = src_val;
4047 /* REGADD_RIH, REGADD_RIM. */
4048 if (!fidx && fsrc) {
4049 idx_val = strtoul(idx, &idx, 0);
4050 CHECK(!idx[0], EINVAL);
4052 CHECK(!fsrc->var_size, EINVAL);
4054 instr->type = INSTR_REGADD_RIM;
4056 instr->type = INSTR_REGADD_RIH;
4058 instr->regarray.regarray_id = r->id;
4059 instr->regarray.idx_val = idx_val;
4060 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4061 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4062 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4067 src_val = strtoull(src, &src, 0);
4068 CHECK(!src[0], EINVAL);
4070 idx_val = strtoul(idx, &idx, 0);
4071 CHECK(!idx[0], EINVAL);
4073 instr->type = INSTR_REGADD_RII;
4074 instr->regarray.idx_val = idx_val;
4075 instr->regarray.dstsrc_val = src_val;
4080 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4082 struct thread *t = &p->threads[p->thread_id];
4083 struct instruction *ip = t->ip;
4086 __instr_regprefetch_rh_exec(p, t, ip);
4093 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4095 struct thread *t = &p->threads[p->thread_id];
4096 struct instruction *ip = t->ip;
4099 __instr_regprefetch_rm_exec(p, t, ip);
4106 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4108 struct thread *t = &p->threads[p->thread_id];
4109 struct instruction *ip = t->ip;
4112 __instr_regprefetch_ri_exec(p, t, ip);
4119 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4121 struct thread *t = &p->threads[p->thread_id];
4122 struct instruction *ip = t->ip;
4125 __instr_regrd_hrh_exec(p, t, ip);
4132 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4134 struct thread *t = &p->threads[p->thread_id];
4135 struct instruction *ip = t->ip;
4138 __instr_regrd_hrm_exec(p, t, ip);
4145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4147 struct thread *t = &p->threads[p->thread_id];
4148 struct instruction *ip = t->ip;
4151 __instr_regrd_mrh_exec(p, t, ip);
4158 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4160 struct thread *t = &p->threads[p->thread_id];
4161 struct instruction *ip = t->ip;
4164 __instr_regrd_mrm_exec(p, t, ip);
4171 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4173 struct thread *t = &p->threads[p->thread_id];
4174 struct instruction *ip = t->ip;
4177 __instr_regrd_hri_exec(p, t, ip);
4184 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4186 struct thread *t = &p->threads[p->thread_id];
4187 struct instruction *ip = t->ip;
4190 __instr_regrd_mri_exec(p, t, ip);
4197 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4199 struct thread *t = &p->threads[p->thread_id];
4200 struct instruction *ip = t->ip;
4203 __instr_regwr_rhh_exec(p, t, ip);
4210 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4212 struct thread *t = &p->threads[p->thread_id];
4213 struct instruction *ip = t->ip;
4216 __instr_regwr_rhm_exec(p, t, ip);
4223 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4225 struct thread *t = &p->threads[p->thread_id];
4226 struct instruction *ip = t->ip;
4229 __instr_regwr_rmh_exec(p, t, ip);
4236 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4238 struct thread *t = &p->threads[p->thread_id];
4239 struct instruction *ip = t->ip;
4242 __instr_regwr_rmm_exec(p, t, ip);
4249 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4251 struct thread *t = &p->threads[p->thread_id];
4252 struct instruction *ip = t->ip;
4255 __instr_regwr_rhi_exec(p, t, ip);
4262 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4264 struct thread *t = &p->threads[p->thread_id];
4265 struct instruction *ip = t->ip;
4268 __instr_regwr_rmi_exec(p, t, ip);
4275 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4277 struct thread *t = &p->threads[p->thread_id];
4278 struct instruction *ip = t->ip;
4281 __instr_regwr_rih_exec(p, t, ip);
4288 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4290 struct thread *t = &p->threads[p->thread_id];
4291 struct instruction *ip = t->ip;
4294 __instr_regwr_rim_exec(p, t, ip);
4301 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4303 struct thread *t = &p->threads[p->thread_id];
4304 struct instruction *ip = t->ip;
4307 __instr_regwr_rii_exec(p, t, ip);
4314 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4316 struct thread *t = &p->threads[p->thread_id];
4317 struct instruction *ip = t->ip;
4320 __instr_regadd_rhh_exec(p, t, ip);
4327 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4329 struct thread *t = &p->threads[p->thread_id];
4330 struct instruction *ip = t->ip;
4333 __instr_regadd_rhm_exec(p, t, ip);
4340 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4342 struct thread *t = &p->threads[p->thread_id];
4343 struct instruction *ip = t->ip;
4346 __instr_regadd_rmh_exec(p, t, ip);
4353 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4355 struct thread *t = &p->threads[p->thread_id];
4356 struct instruction *ip = t->ip;
4359 __instr_regadd_rmm_exec(p, t, ip);
4366 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4368 struct thread *t = &p->threads[p->thread_id];
4369 struct instruction *ip = t->ip;
4372 __instr_regadd_rhi_exec(p, t, ip);
4379 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4381 struct thread *t = &p->threads[p->thread_id];
4382 struct instruction *ip = t->ip;
4385 __instr_regadd_rmi_exec(p, t, ip);
4392 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4394 struct thread *t = &p->threads[p->thread_id];
4395 struct instruction *ip = t->ip;
4398 __instr_regadd_rih_exec(p, t, ip);
4405 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4407 struct thread *t = &p->threads[p->thread_id];
4408 struct instruction *ip = t->ip;
4411 __instr_regadd_rim_exec(p, t, ip);
4418 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4420 struct thread *t = &p->threads[p->thread_id];
4421 struct instruction *ip = t->ip;
4424 __instr_regadd_rii_exec(p, t, ip);
4433 static struct metarray *
4434 metarray_find(struct rte_swx_pipeline *p, const char *name);
4437 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4438 struct action *action,
4441 struct instruction *instr,
4442 struct instruction_data *data __rte_unused)
4444 char *metarray = tokens[1], *idx = tokens[2];
4447 uint32_t idx_struct_id, idx_val;
4449 CHECK(n_tokens == 3, EINVAL);
4451 m = metarray_find(p, metarray);
4454 /* METPREFETCH_H, METPREFETCH_M. */
4455 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4457 CHECK(!fidx->var_size, EINVAL);
4459 instr->type = INSTR_METPREFETCH_M;
4461 instr->type = INSTR_METPREFETCH_H;
4463 instr->meter.metarray_id = m->id;
4464 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4465 instr->meter.idx.n_bits = fidx->n_bits;
4466 instr->meter.idx.offset = fidx->offset / 8;
4470 /* METPREFETCH_I. */
4471 idx_val = strtoul(idx, &idx, 0);
4472 CHECK(!idx[0], EINVAL);
4474 instr->type = INSTR_METPREFETCH_I;
4475 instr->meter.metarray_id = m->id;
4476 instr->meter.idx_val = idx_val;
4481 instr_meter_translate(struct rte_swx_pipeline *p,
4482 struct action *action,
4485 struct instruction *instr,
4486 struct instruction_data *data __rte_unused)
4488 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4489 char *color_in = tokens[4], *color_out = tokens[5];
4491 struct field *fidx, *flength, *fcin, *fcout;
4492 uint32_t idx_struct_id, length_struct_id;
4493 uint32_t color_in_struct_id, color_out_struct_id;
4495 CHECK(n_tokens == 6, EINVAL);
4497 m = metarray_find(p, metarray);
4500 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4502 flength = struct_field_parse(p, action, length, &length_struct_id);
4503 CHECK(flength, EINVAL);
4504 CHECK(!flength->var_size, EINVAL);
4506 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4508 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4509 CHECK(fcout, EINVAL);
4510 CHECK(!fcout->var_size, EINVAL);
4512 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4514 CHECK(!fidx->var_size, EINVAL);
4515 CHECK(!fcin->var_size, EINVAL);
4517 instr->type = INSTR_METER_MMM;
4518 if (idx[0] == 'h' && length[0] == 'h')
4519 instr->type = INSTR_METER_HHM;
4520 if (idx[0] == 'h' && length[0] != 'h')
4521 instr->type = INSTR_METER_HMM;
4522 if (idx[0] != 'h' && length[0] == 'h')
4523 instr->type = INSTR_METER_MHM;
4525 instr->meter.metarray_id = m->id;
4527 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4528 instr->meter.idx.n_bits = fidx->n_bits;
4529 instr->meter.idx.offset = fidx->offset / 8;
4531 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4532 instr->meter.length.n_bits = flength->n_bits;
4533 instr->meter.length.offset = flength->offset / 8;
4535 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4536 instr->meter.color_in.n_bits = fcin->n_bits;
4537 instr->meter.color_in.offset = fcin->offset / 8;
4539 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4540 instr->meter.color_out.n_bits = fcout->n_bits;
4541 instr->meter.color_out.offset = fcout->offset / 8;
4544 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4545 if (fidx && !fcin) {
4546 uint32_t color_in_val;
4548 CHECK(!fidx->var_size, EINVAL);
4550 color_in_val = strtoul(color_in, &color_in, 0);
4551 CHECK(!color_in[0], EINVAL);
4553 instr->type = INSTR_METER_MMI;
4554 if (idx[0] == 'h' && length[0] == 'h')
4555 instr->type = INSTR_METER_HHI;
4556 if (idx[0] == 'h' && length[0] != 'h')
4557 instr->type = INSTR_METER_HMI;
4558 if (idx[0] != 'h' && length[0] == 'h')
4559 instr->type = INSTR_METER_MHI;
4561 instr->meter.metarray_id = m->id;
4563 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4564 instr->meter.idx.n_bits = fidx->n_bits;
4565 instr->meter.idx.offset = fidx->offset / 8;
4567 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4568 instr->meter.length.n_bits = flength->n_bits;
4569 instr->meter.length.offset = flength->offset / 8;
4571 instr->meter.color_in_val = color_in_val;
4573 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4574 instr->meter.color_out.n_bits = fcout->n_bits;
4575 instr->meter.color_out.offset = fcout->offset / 8;
4578 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4579 if (!fidx && fcin) {
4582 idx_val = strtoul(idx, &idx, 0);
4583 CHECK(!idx[0], EINVAL);
4585 CHECK(!fcin->var_size, EINVAL);
4587 instr->type = INSTR_METER_IMM;
4588 if (length[0] == 'h')
4589 instr->type = INSTR_METER_IHM;
4591 instr->meter.metarray_id = m->id;
4593 instr->meter.idx_val = idx_val;
4595 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4596 instr->meter.length.n_bits = flength->n_bits;
4597 instr->meter.length.offset = flength->offset / 8;
4599 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4600 instr->meter.color_in.n_bits = fcin->n_bits;
4601 instr->meter.color_in.offset = fcin->offset / 8;
4603 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4604 instr->meter.color_out.n_bits = fcout->n_bits;
4605 instr->meter.color_out.offset = fcout->offset / 8;
4608 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4609 if (!fidx && !fcin) {
4610 uint32_t idx_val, color_in_val;
4612 idx_val = strtoul(idx, &idx, 0);
4613 CHECK(!idx[0], EINVAL);
4615 color_in_val = strtoul(color_in, &color_in, 0);
4616 CHECK(!color_in[0], EINVAL);
4618 instr->type = INSTR_METER_IMI;
4619 if (length[0] == 'h')
4620 instr->type = INSTR_METER_IHI;
4622 instr->meter.metarray_id = m->id;
4624 instr->meter.idx_val = idx_val;
4626 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4627 instr->meter.length.n_bits = flength->n_bits;
4628 instr->meter.length.offset = flength->offset / 8;
4630 instr->meter.color_in_val = color_in_val;
4632 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4633 instr->meter.color_out.n_bits = fcout->n_bits;
4634 instr->meter.color_out.offset = fcout->offset / 8;
4641 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4643 struct thread *t = &p->threads[p->thread_id];
4644 struct instruction *ip = t->ip;
4647 __instr_metprefetch_h_exec(p, t, ip);
4654 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4656 struct thread *t = &p->threads[p->thread_id];
4657 struct instruction *ip = t->ip;
4660 __instr_metprefetch_m_exec(p, t, ip);
4667 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4669 struct thread *t = &p->threads[p->thread_id];
4670 struct instruction *ip = t->ip;
4673 __instr_metprefetch_i_exec(p, t, ip);
4680 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4682 struct thread *t = &p->threads[p->thread_id];
4683 struct instruction *ip = t->ip;
4686 __instr_meter_hhm_exec(p, t, ip);
4693 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4695 struct thread *t = &p->threads[p->thread_id];
4696 struct instruction *ip = t->ip;
4699 __instr_meter_hhi_exec(p, t, ip);
4706 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4708 struct thread *t = &p->threads[p->thread_id];
4709 struct instruction *ip = t->ip;
4712 __instr_meter_hmm_exec(p, t, ip);
4719 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4721 struct thread *t = &p->threads[p->thread_id];
4722 struct instruction *ip = t->ip;
4725 __instr_meter_hmi_exec(p, t, ip);
4732 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4734 struct thread *t = &p->threads[p->thread_id];
4735 struct instruction *ip = t->ip;
4738 __instr_meter_mhm_exec(p, t, ip);
4745 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4747 struct thread *t = &p->threads[p->thread_id];
4748 struct instruction *ip = t->ip;
4751 __instr_meter_mhi_exec(p, t, ip);
4758 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4760 struct thread *t = &p->threads[p->thread_id];
4761 struct instruction *ip = t->ip;
4764 __instr_meter_mmm_exec(p, t, ip);
4771 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4773 struct thread *t = &p->threads[p->thread_id];
4774 struct instruction *ip = t->ip;
4777 __instr_meter_mmi_exec(p, t, ip);
4784 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4786 struct thread *t = &p->threads[p->thread_id];
4787 struct instruction *ip = t->ip;
4790 __instr_meter_ihm_exec(p, t, ip);
4797 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4799 struct thread *t = &p->threads[p->thread_id];
4800 struct instruction *ip = t->ip;
4803 __instr_meter_ihi_exec(p, t, ip);
4810 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4812 struct thread *t = &p->threads[p->thread_id];
4813 struct instruction *ip = t->ip;
4816 __instr_meter_imm_exec(p, t, ip);
4823 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4825 struct thread *t = &p->threads[p->thread_id];
4826 struct instruction *ip = t->ip;
4829 __instr_meter_imi_exec(p, t, ip);
4839 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4840 struct action *action __rte_unused,
4843 struct instruction *instr,
4844 struct instruction_data *data)
4846 CHECK(n_tokens == 2, EINVAL);
4848 strcpy(data->jmp_label, tokens[1]);
4850 instr->type = INSTR_JMP;
4851 instr->jmp.ip = NULL; /* Resolved later. */
4856 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4857 struct action *action __rte_unused,
4860 struct instruction *instr,
4861 struct instruction_data *data)
4865 CHECK(n_tokens == 3, EINVAL);
4867 strcpy(data->jmp_label, tokens[1]);
4869 h = header_parse(p, tokens[2]);
4872 instr->type = INSTR_JMP_VALID;
4873 instr->jmp.ip = NULL; /* Resolved later. */
4874 instr->jmp.header_id = h->id;
4879 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4880 struct action *action __rte_unused,
4883 struct instruction *instr,
4884 struct instruction_data *data)
4888 CHECK(n_tokens == 3, EINVAL);
4890 strcpy(data->jmp_label, tokens[1]);
4892 h = header_parse(p, tokens[2]);
4895 instr->type = INSTR_JMP_INVALID;
4896 instr->jmp.ip = NULL; /* Resolved later. */
4897 instr->jmp.header_id = h->id;
4902 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4903 struct action *action,
4906 struct instruction *instr,
4907 struct instruction_data *data)
4909 CHECK(!action, EINVAL);
4910 CHECK(n_tokens == 2, EINVAL);
4912 strcpy(data->jmp_label, tokens[1]);
4914 instr->type = INSTR_JMP_HIT;
4915 instr->jmp.ip = NULL; /* Resolved later. */
4920 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4921 struct action *action,
4924 struct instruction *instr,
4925 struct instruction_data *data)
4927 CHECK(!action, EINVAL);
4928 CHECK(n_tokens == 2, EINVAL);
4930 strcpy(data->jmp_label, tokens[1]);
4932 instr->type = INSTR_JMP_MISS;
4933 instr->jmp.ip = NULL; /* Resolved later. */
4938 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4939 struct action *action,
4942 struct instruction *instr,
4943 struct instruction_data *data)
4947 CHECK(!action, EINVAL);
4948 CHECK(n_tokens == 3, EINVAL);
4950 strcpy(data->jmp_label, tokens[1]);
4952 a = action_find(p, tokens[2]);
4955 instr->type = INSTR_JMP_ACTION_HIT;
4956 instr->jmp.ip = NULL; /* Resolved later. */
4957 instr->jmp.action_id = a->id;
4962 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4963 struct action *action,
4966 struct instruction *instr,
4967 struct instruction_data *data)
4971 CHECK(!action, EINVAL);
4972 CHECK(n_tokens == 3, EINVAL);
4974 strcpy(data->jmp_label, tokens[1]);
4976 a = action_find(p, tokens[2]);
4979 instr->type = INSTR_JMP_ACTION_MISS;
4980 instr->jmp.ip = NULL; /* Resolved later. */
4981 instr->jmp.action_id = a->id;
4986 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4987 struct action *action,
4990 struct instruction *instr,
4991 struct instruction_data *data)
4993 char *a = tokens[2], *b = tokens[3];
4994 struct field *fa, *fb;
4996 uint32_t a_struct_id, b_struct_id;
4998 CHECK(n_tokens == 4, EINVAL);
5000 strcpy(data->jmp_label, tokens[1]);
5002 fa = struct_field_parse(p, action, a, &a_struct_id);
5004 CHECK(!fa->var_size, EINVAL);
5006 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5007 fb = struct_field_parse(p, action, b, &b_struct_id);
5009 CHECK(!fb->var_size, EINVAL);
5011 instr->type = INSTR_JMP_EQ;
5012 if (a[0] != 'h' && b[0] == 'h')
5013 instr->type = INSTR_JMP_EQ_MH;
5014 if (a[0] == 'h' && b[0] != 'h')
5015 instr->type = INSTR_JMP_EQ_HM;
5016 if (a[0] == 'h' && b[0] == 'h')
5017 instr->type = INSTR_JMP_EQ_HH;
5018 instr->jmp.ip = NULL; /* Resolved later. */
5020 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5021 instr->jmp.a.n_bits = fa->n_bits;
5022 instr->jmp.a.offset = fa->offset / 8;
5023 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5024 instr->jmp.b.n_bits = fb->n_bits;
5025 instr->jmp.b.offset = fb->offset / 8;
5030 b_val = strtoull(b, &b, 0);
5031 CHECK(!b[0], EINVAL);
5034 b_val = hton64(b_val) >> (64 - fa->n_bits);
5036 instr->type = INSTR_JMP_EQ_I;
5037 instr->jmp.ip = NULL; /* Resolved later. */
5038 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5039 instr->jmp.a.n_bits = fa->n_bits;
5040 instr->jmp.a.offset = fa->offset / 8;
5041 instr->jmp.b_val = b_val;
5046 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5047 struct action *action,
5050 struct instruction *instr,
5051 struct instruction_data *data)
5053 char *a = tokens[2], *b = tokens[3];
5054 struct field *fa, *fb;
5056 uint32_t a_struct_id, b_struct_id;
5058 CHECK(n_tokens == 4, EINVAL);
5060 strcpy(data->jmp_label, tokens[1]);
5062 fa = struct_field_parse(p, action, a, &a_struct_id);
5064 CHECK(!fa->var_size, EINVAL);
5066 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5067 fb = struct_field_parse(p, action, b, &b_struct_id);
5069 CHECK(!fb->var_size, EINVAL);
5071 instr->type = INSTR_JMP_NEQ;
5072 if (a[0] != 'h' && b[0] == 'h')
5073 instr->type = INSTR_JMP_NEQ_MH;
5074 if (a[0] == 'h' && b[0] != 'h')
5075 instr->type = INSTR_JMP_NEQ_HM;
5076 if (a[0] == 'h' && b[0] == 'h')
5077 instr->type = INSTR_JMP_NEQ_HH;
5078 instr->jmp.ip = NULL; /* Resolved later. */
5080 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5081 instr->jmp.a.n_bits = fa->n_bits;
5082 instr->jmp.a.offset = fa->offset / 8;
5083 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5084 instr->jmp.b.n_bits = fb->n_bits;
5085 instr->jmp.b.offset = fb->offset / 8;
5090 b_val = strtoull(b, &b, 0);
5091 CHECK(!b[0], EINVAL);
5094 b_val = hton64(b_val) >> (64 - fa->n_bits);
5096 instr->type = INSTR_JMP_NEQ_I;
5097 instr->jmp.ip = NULL; /* Resolved later. */
5098 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5099 instr->jmp.a.n_bits = fa->n_bits;
5100 instr->jmp.a.offset = fa->offset / 8;
5101 instr->jmp.b_val = b_val;
5106 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5107 struct action *action,
5110 struct instruction *instr,
5111 struct instruction_data *data)
5113 char *a = tokens[2], *b = tokens[3];
5114 struct field *fa, *fb;
5116 uint32_t a_struct_id, b_struct_id;
5118 CHECK(n_tokens == 4, EINVAL);
5120 strcpy(data->jmp_label, tokens[1]);
5122 fa = struct_field_parse(p, action, a, &a_struct_id);
5124 CHECK(!fa->var_size, EINVAL);
5126 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5127 fb = struct_field_parse(p, action, b, &b_struct_id);
5129 CHECK(!fb->var_size, EINVAL);
5131 instr->type = INSTR_JMP_LT;
5132 if (a[0] == 'h' && b[0] != 'h')
5133 instr->type = INSTR_JMP_LT_HM;
5134 if (a[0] != 'h' && b[0] == 'h')
5135 instr->type = INSTR_JMP_LT_MH;
5136 if (a[0] == 'h' && b[0] == 'h')
5137 instr->type = INSTR_JMP_LT_HH;
5138 instr->jmp.ip = NULL; /* Resolved later. */
5140 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5141 instr->jmp.a.n_bits = fa->n_bits;
5142 instr->jmp.a.offset = fa->offset / 8;
5143 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5144 instr->jmp.b.n_bits = fb->n_bits;
5145 instr->jmp.b.offset = fb->offset / 8;
5149 /* JMP_LT_MI, JMP_LT_HI. */
5150 b_val = strtoull(b, &b, 0);
5151 CHECK(!b[0], EINVAL);
5153 instr->type = INSTR_JMP_LT_MI;
5155 instr->type = INSTR_JMP_LT_HI;
5156 instr->jmp.ip = NULL; /* Resolved later. */
5158 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5159 instr->jmp.a.n_bits = fa->n_bits;
5160 instr->jmp.a.offset = fa->offset / 8;
5161 instr->jmp.b_val = b_val;
5166 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5167 struct action *action,
5170 struct instruction *instr,
5171 struct instruction_data *data)
5173 char *a = tokens[2], *b = tokens[3];
5174 struct field *fa, *fb;
5176 uint32_t a_struct_id, b_struct_id;
5178 CHECK(n_tokens == 4, EINVAL);
5180 strcpy(data->jmp_label, tokens[1]);
5182 fa = struct_field_parse(p, action, a, &a_struct_id);
5184 CHECK(!fa->var_size, EINVAL);
5186 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5187 fb = struct_field_parse(p, action, b, &b_struct_id);
5189 CHECK(!fb->var_size, EINVAL);
5191 instr->type = INSTR_JMP_GT;
5192 if (a[0] == 'h' && b[0] != 'h')
5193 instr->type = INSTR_JMP_GT_HM;
5194 if (a[0] != 'h' && b[0] == 'h')
5195 instr->type = INSTR_JMP_GT_MH;
5196 if (a[0] == 'h' && b[0] == 'h')
5197 instr->type = INSTR_JMP_GT_HH;
5198 instr->jmp.ip = NULL; /* Resolved later. */
5200 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5201 instr->jmp.a.n_bits = fa->n_bits;
5202 instr->jmp.a.offset = fa->offset / 8;
5203 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5204 instr->jmp.b.n_bits = fb->n_bits;
5205 instr->jmp.b.offset = fb->offset / 8;
5209 /* JMP_GT_MI, JMP_GT_HI. */
5210 b_val = strtoull(b, &b, 0);
5211 CHECK(!b[0], EINVAL);
5213 instr->type = INSTR_JMP_GT_MI;
5215 instr->type = INSTR_JMP_GT_HI;
5216 instr->jmp.ip = NULL; /* Resolved later. */
5218 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5219 instr->jmp.a.n_bits = fa->n_bits;
5220 instr->jmp.a.offset = fa->offset / 8;
5221 instr->jmp.b_val = b_val;
5226 instr_jmp_exec(struct rte_swx_pipeline *p)
5228 struct thread *t = &p->threads[p->thread_id];
5229 struct instruction *ip = t->ip;
5231 TRACE("[Thread %2u] jmp\n", p->thread_id);
5233 thread_ip_set(t, ip->jmp.ip);
5237 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5239 struct thread *t = &p->threads[p->thread_id];
5240 struct instruction *ip = t->ip;
5241 uint32_t header_id = ip->jmp.header_id;
5243 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5245 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5249 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5251 struct thread *t = &p->threads[p->thread_id];
5252 struct instruction *ip = t->ip;
5253 uint32_t header_id = ip->jmp.header_id;
5255 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5257 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5261 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5263 struct thread *t = &p->threads[p->thread_id];
5264 struct instruction *ip = t->ip;
5265 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5267 TRACE("[Thread %2u] jmph\n", p->thread_id);
5269 t->ip = ip_next[t->hit];
5273 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5275 struct thread *t = &p->threads[p->thread_id];
5276 struct instruction *ip = t->ip;
5277 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5279 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5281 t->ip = ip_next[t->hit];
5285 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5287 struct thread *t = &p->threads[p->thread_id];
5288 struct instruction *ip = t->ip;
5290 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5292 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5296 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5298 struct thread *t = &p->threads[p->thread_id];
5299 struct instruction *ip = t->ip;
5301 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5303 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5307 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5309 struct thread *t = &p->threads[p->thread_id];
5310 struct instruction *ip = t->ip;
5312 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5318 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5320 struct thread *t = &p->threads[p->thread_id];
5321 struct instruction *ip = t->ip;
5323 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5325 JMP_CMP_MH(t, ip, ==);
5329 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5331 struct thread *t = &p->threads[p->thread_id];
5332 struct instruction *ip = t->ip;
5334 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5336 JMP_CMP_HM(t, ip, ==);
5340 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5342 struct thread *t = &p->threads[p->thread_id];
5343 struct instruction *ip = t->ip;
5345 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5347 JMP_CMP_HH_FAST(t, ip, ==);
5351 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5353 struct thread *t = &p->threads[p->thread_id];
5354 struct instruction *ip = t->ip;
5356 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5358 JMP_CMP_I(t, ip, ==);
5362 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5364 struct thread *t = &p->threads[p->thread_id];
5365 struct instruction *ip = t->ip;
5367 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5373 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5375 struct thread *t = &p->threads[p->thread_id];
5376 struct instruction *ip = t->ip;
5378 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5380 JMP_CMP_MH(t, ip, !=);
5384 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5386 struct thread *t = &p->threads[p->thread_id];
5387 struct instruction *ip = t->ip;
5389 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5391 JMP_CMP_HM(t, ip, !=);
5395 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5397 struct thread *t = &p->threads[p->thread_id];
5398 struct instruction *ip = t->ip;
5400 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5402 JMP_CMP_HH_FAST(t, ip, !=);
5406 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5408 struct thread *t = &p->threads[p->thread_id];
5409 struct instruction *ip = t->ip;
5411 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5413 JMP_CMP_I(t, ip, !=);
5417 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5419 struct thread *t = &p->threads[p->thread_id];
5420 struct instruction *ip = t->ip;
5422 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5428 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5430 struct thread *t = &p->threads[p->thread_id];
5431 struct instruction *ip = t->ip;
5433 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5435 JMP_CMP_MH(t, ip, <);
5439 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5441 struct thread *t = &p->threads[p->thread_id];
5442 struct instruction *ip = t->ip;
5444 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5446 JMP_CMP_HM(t, ip, <);
5450 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5452 struct thread *t = &p->threads[p->thread_id];
5453 struct instruction *ip = t->ip;
5455 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5457 JMP_CMP_HH(t, ip, <);
5461 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5463 struct thread *t = &p->threads[p->thread_id];
5464 struct instruction *ip = t->ip;
5466 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5468 JMP_CMP_MI(t, ip, <);
5472 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5474 struct thread *t = &p->threads[p->thread_id];
5475 struct instruction *ip = t->ip;
5477 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5479 JMP_CMP_HI(t, ip, <);
5483 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5485 struct thread *t = &p->threads[p->thread_id];
5486 struct instruction *ip = t->ip;
5488 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5494 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5496 struct thread *t = &p->threads[p->thread_id];
5497 struct instruction *ip = t->ip;
5499 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5501 JMP_CMP_MH(t, ip, >);
5505 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5507 struct thread *t = &p->threads[p->thread_id];
5508 struct instruction *ip = t->ip;
5510 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5512 JMP_CMP_HM(t, ip, >);
5516 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5518 struct thread *t = &p->threads[p->thread_id];
5519 struct instruction *ip = t->ip;
5521 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5523 JMP_CMP_HH(t, ip, >);
5527 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5529 struct thread *t = &p->threads[p->thread_id];
5530 struct instruction *ip = t->ip;
5532 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5534 JMP_CMP_MI(t, ip, >);
5538 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5540 struct thread *t = &p->threads[p->thread_id];
5541 struct instruction *ip = t->ip;
5543 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5545 JMP_CMP_HI(t, ip, >);
5552 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5553 struct action *action,
5554 char **tokens __rte_unused,
5556 struct instruction *instr,
5557 struct instruction_data *data __rte_unused)
5559 CHECK(action, EINVAL);
5560 CHECK(n_tokens == 1, EINVAL);
5562 instr->type = INSTR_RETURN;
5567 instr_return_exec(struct rte_swx_pipeline *p)
5569 struct thread *t = &p->threads[p->thread_id];
5571 TRACE("[Thread %2u] return\n", p->thread_id);
5577 instr_translate(struct rte_swx_pipeline *p,
5578 struct action *action,
5580 struct instruction *instr,
5581 struct instruction_data *data)
5583 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5584 int n_tokens = 0, tpos = 0;
5586 /* Parse the instruction string into tokens. */
5590 token = strtok_r(string, " \t\v", &string);
5594 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5595 CHECK_NAME(token, EINVAL);
5597 tokens[n_tokens] = token;
5601 CHECK(n_tokens, EINVAL);
5603 /* Handle the optional instruction label. */
5604 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5605 strcpy(data->label, tokens[0]);
5608 CHECK(n_tokens - tpos, EINVAL);
5611 /* Identify the instruction type. */
5612 if (!strcmp(tokens[tpos], "rx"))
5613 return instr_rx_translate(p,
5620 if (!strcmp(tokens[tpos], "tx"))
5621 return instr_tx_translate(p,
5628 if (!strcmp(tokens[tpos], "drop"))
5629 return instr_drop_translate(p,
5636 if (!strcmp(tokens[tpos], "extract"))
5637 return instr_hdr_extract_translate(p,
5644 if (!strcmp(tokens[tpos], "lookahead"))
5645 return instr_hdr_lookahead_translate(p,
5652 if (!strcmp(tokens[tpos], "emit"))
5653 return instr_hdr_emit_translate(p,
5660 if (!strcmp(tokens[tpos], "validate"))
5661 return instr_hdr_validate_translate(p,
5668 if (!strcmp(tokens[tpos], "invalidate"))
5669 return instr_hdr_invalidate_translate(p,
5676 if (!strcmp(tokens[tpos], "mov"))
5677 return instr_mov_translate(p,
5684 if (!strcmp(tokens[tpos], "add"))
5685 return instr_alu_add_translate(p,
5692 if (!strcmp(tokens[tpos], "sub"))
5693 return instr_alu_sub_translate(p,
5700 if (!strcmp(tokens[tpos], "ckadd"))
5701 return instr_alu_ckadd_translate(p,
5708 if (!strcmp(tokens[tpos], "cksub"))
5709 return instr_alu_cksub_translate(p,
5716 if (!strcmp(tokens[tpos], "and"))
5717 return instr_alu_and_translate(p,
5724 if (!strcmp(tokens[tpos], "or"))
5725 return instr_alu_or_translate(p,
5732 if (!strcmp(tokens[tpos], "xor"))
5733 return instr_alu_xor_translate(p,
5740 if (!strcmp(tokens[tpos], "shl"))
5741 return instr_alu_shl_translate(p,
5748 if (!strcmp(tokens[tpos], "shr"))
5749 return instr_alu_shr_translate(p,
5756 if (!strcmp(tokens[tpos], "regprefetch"))
5757 return instr_regprefetch_translate(p,
5764 if (!strcmp(tokens[tpos], "regrd"))
5765 return instr_regrd_translate(p,
5772 if (!strcmp(tokens[tpos], "regwr"))
5773 return instr_regwr_translate(p,
5780 if (!strcmp(tokens[tpos], "regadd"))
5781 return instr_regadd_translate(p,
5788 if (!strcmp(tokens[tpos], "metprefetch"))
5789 return instr_metprefetch_translate(p,
5796 if (!strcmp(tokens[tpos], "meter"))
5797 return instr_meter_translate(p,
5804 if (!strcmp(tokens[tpos], "table"))
5805 return instr_table_translate(p,
5812 if (!strcmp(tokens[tpos], "learn"))
5813 return instr_learn_translate(p,
5820 if (!strcmp(tokens[tpos], "forget"))
5821 return instr_forget_translate(p,
5828 if (!strcmp(tokens[tpos], "extern"))
5829 return instr_extern_translate(p,
5836 if (!strcmp(tokens[tpos], "jmp"))
5837 return instr_jmp_translate(p,
5844 if (!strcmp(tokens[tpos], "jmpv"))
5845 return instr_jmp_valid_translate(p,
5852 if (!strcmp(tokens[tpos], "jmpnv"))
5853 return instr_jmp_invalid_translate(p,
5860 if (!strcmp(tokens[tpos], "jmph"))
5861 return instr_jmp_hit_translate(p,
5868 if (!strcmp(tokens[tpos], "jmpnh"))
5869 return instr_jmp_miss_translate(p,
5876 if (!strcmp(tokens[tpos], "jmpa"))
5877 return instr_jmp_action_hit_translate(p,
5884 if (!strcmp(tokens[tpos], "jmpna"))
5885 return instr_jmp_action_miss_translate(p,
5892 if (!strcmp(tokens[tpos], "jmpeq"))
5893 return instr_jmp_eq_translate(p,
5900 if (!strcmp(tokens[tpos], "jmpneq"))
5901 return instr_jmp_neq_translate(p,
5908 if (!strcmp(tokens[tpos], "jmplt"))
5909 return instr_jmp_lt_translate(p,
5916 if (!strcmp(tokens[tpos], "jmpgt"))
5917 return instr_jmp_gt_translate(p,
5924 if (!strcmp(tokens[tpos], "return"))
5925 return instr_return_translate(p,
5935 static struct instruction_data *
5936 label_find(struct instruction_data *data, uint32_t n, const char *label)
5940 for (i = 0; i < n; i++)
5941 if (!strcmp(label, data[i].label))
5948 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5950 uint32_t count = 0, i;
5955 for (i = 0; i < n; i++)
5956 if (!strcmp(label, data[i].jmp_label))
5963 instr_label_check(struct instruction_data *instruction_data,
5964 uint32_t n_instructions)
5968 /* Check that all instruction labels are unique. */
5969 for (i = 0; i < n_instructions; i++) {
5970 struct instruction_data *data = &instruction_data[i];
5971 char *label = data->label;
5977 for (j = i + 1; j < n_instructions; j++)
5978 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
5981 /* Get users for each instruction label. */
5982 for (i = 0; i < n_instructions; i++) {
5983 struct instruction_data *data = &instruction_data[i];
5984 char *label = data->label;
5986 data->n_users = label_is_used(instruction_data,
5995 instr_jmp_resolve(struct instruction *instructions,
5996 struct instruction_data *instruction_data,
5997 uint32_t n_instructions)
6001 for (i = 0; i < n_instructions; i++) {
6002 struct instruction *instr = &instructions[i];
6003 struct instruction_data *data = &instruction_data[i];
6004 struct instruction_data *found;
6006 if (!instruction_is_jmp(instr))
6009 found = label_find(instruction_data,
6012 CHECK(found, EINVAL);
6014 instr->jmp.ip = &instructions[found - instruction_data];
6021 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6023 struct instruction *instr,
6024 struct instruction_data *data __rte_unused,
6025 uint32_t n_instructions)
6028 enum instruction_type type;
6031 /* Check that the first instruction is rx. */
6032 CHECK(instr[0].type == INSTR_RX, EINVAL);
6034 /* Check that there is at least one tx instruction. */
6035 for (i = 0; i < n_instructions; i++) {
6036 type = instr[i].type;
6038 if (instruction_is_tx(type))
6041 CHECK(i < n_instructions, EINVAL);
6043 /* Check that the last instruction is either tx or unconditional
6046 type = instr[n_instructions - 1].type;
6047 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6051 enum instruction_type type;
6054 /* Check that there is at least one return or tx instruction. */
6055 for (i = 0; i < n_instructions; i++) {
6056 type = instr[i].type;
6058 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6061 CHECK(i < n_instructions, EINVAL);
6068 instr_compact(struct instruction *instructions,
6069 struct instruction_data *instruction_data,
6070 uint32_t n_instructions)
6072 uint32_t i, pos = 0;
6074 /* Eliminate the invalid instructions that have been optimized out. */
6075 for (i = 0; i < n_instructions; i++) {
6076 struct instruction *instr = &instructions[i];
6077 struct instruction_data *data = &instruction_data[i];
6083 memcpy(&instructions[pos], instr, sizeof(*instr));
6084 memcpy(&instruction_data[pos], data, sizeof(*data));
6094 instr_pattern_extract_many_search(struct instruction *instr,
6095 struct instruction_data *data,
6097 uint32_t *n_pattern_instr)
6101 for (i = 0; i < n_instr; i++) {
6102 if (data[i].invalid)
6105 if (instr[i].type != INSTR_HDR_EXTRACT)
6108 if (i == RTE_DIM(instr->io.hdr.header_id))
6111 if (i && data[i].n_users)
6118 *n_pattern_instr = i;
6123 instr_pattern_extract_many_replace(struct instruction *instr,
6124 struct instruction_data *data,
6129 for (i = 1; i < n_instr; i++) {
6131 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6132 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6133 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6135 data[i].invalid = 1;
6140 instr_pattern_extract_many_optimize(struct instruction *instructions,
6141 struct instruction_data *instruction_data,
6142 uint32_t n_instructions)
6146 for (i = 0; i < n_instructions; ) {
6147 struct instruction *instr = &instructions[i];
6148 struct instruction_data *data = &instruction_data[i];
6149 uint32_t n_instr = 0;
6153 detected = instr_pattern_extract_many_search(instr,
6158 instr_pattern_extract_many_replace(instr,
6165 /* No pattern starting at the current instruction. */
6169 /* Eliminate the invalid instructions that have been optimized out. */
6170 n_instructions = instr_compact(instructions,
6174 return n_instructions;
6178 instr_pattern_emit_many_tx_search(struct instruction *instr,
6179 struct instruction_data *data,
6181 uint32_t *n_pattern_instr)
6185 for (i = 0; i < n_instr; i++) {
6186 if (data[i].invalid)
6189 if (instr[i].type != INSTR_HDR_EMIT)
6192 if (i == RTE_DIM(instr->io.hdr.header_id))
6195 if (i && data[i].n_users)
6202 if (!instruction_is_tx(instr[i].type))
6205 if (data[i].n_users)
6210 *n_pattern_instr = i;
6215 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6216 struct instruction_data *data,
6221 /* Any emit instruction in addition to the first one. */
6222 for (i = 1; i < n_instr - 1; i++) {
6224 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6225 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6226 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6228 data[i].invalid = 1;
6231 /* The TX instruction is the last one in the pattern. */
6233 instr[0].io.io.offset = instr[i].io.io.offset;
6234 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6235 data[i].invalid = 1;
6239 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6240 struct instruction_data *instruction_data,
6241 uint32_t n_instructions)
6245 for (i = 0; i < n_instructions; ) {
6246 struct instruction *instr = &instructions[i];
6247 struct instruction_data *data = &instruction_data[i];
6248 uint32_t n_instr = 0;
6251 /* Emit many + TX. */
6252 detected = instr_pattern_emit_many_tx_search(instr,
6257 instr_pattern_emit_many_tx_replace(instr,
6264 /* No pattern starting at the current instruction. */
6268 /* Eliminate the invalid instructions that have been optimized out. */
6269 n_instructions = instr_compact(instructions,
6273 return n_instructions;
6277 action_arg_src_mov_count(struct action *a,
6279 struct instruction *instructions,
6280 struct instruction_data *instruction_data,
6281 uint32_t n_instructions);
6284 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6286 struct instruction *instr,
6287 struct instruction_data *data,
6289 struct instruction *instructions,
6290 struct instruction_data *instruction_data,
6291 uint32_t n_instructions,
6292 uint32_t *n_pattern_instr)
6295 uint32_t src_field_id, i, j;
6297 /* Prerequisites. */
6301 /* First instruction: MOV_HM. */
6302 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6305 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6306 if (!h || h->st->var_size)
6309 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6310 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6313 if (src_field_id == a->st->n_fields)
6316 if (instr[0].mov.dst.offset ||
6317 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6318 instr[0].mov.src.struct_id ||
6319 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6320 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6323 if ((n_instr < h->st->n_fields + 1) ||
6324 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6327 /* Subsequent instructions: MOV_HM. */
6328 for (i = 1; i < h->st->n_fields; i++)
6329 if (data[i].invalid ||
6331 (instr[i].type != INSTR_MOV_HM) ||
6332 (instr[i].mov.dst.struct_id != h->struct_id) ||
6333 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6334 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6335 instr[i].mov.src.struct_id ||
6336 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6337 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6338 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6341 /* Last instruction: HDR_VALIDATE. */
6342 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6343 (instr[i].valid.header_id != h->id))
6346 /* Check that none of the action args that are used as source for this
6347 * DMA transfer are not used as source in any other mov instruction.
6349 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6352 n_users = action_arg_src_mov_count(a,
6361 *n_pattern_instr = 1 + i;
6366 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6368 struct instruction *instr,
6369 struct instruction_data *data,
6373 uint32_t src_field_id, src_offset, i;
6375 /* Read from the instructions before they are modified. */
6376 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6380 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6381 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6384 if (src_field_id == a->st->n_fields)
6387 src_offset = instr[0].mov.src.offset;
6389 /* Modify the instructions. */
6390 instr[0].type = INSTR_DMA_HT;
6391 instr[0].dma.dst.header_id[0] = h->id;
6392 instr[0].dma.dst.struct_id[0] = h->struct_id;
6393 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6394 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6396 for (i = 1; i < n_instr; i++)
6397 data[i].invalid = 1;
6399 /* Update the endianness of the action arguments to header endianness. */
6400 for (i = 0; i < h->st->n_fields; i++)
6401 a->args_endianness[src_field_id + i] = 1;
6405 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6407 struct instruction *instructions,
6408 struct instruction_data *instruction_data,
6409 uint32_t n_instructions)
6414 return n_instructions;
6416 for (i = 0; i < n_instructions; ) {
6417 struct instruction *instr = &instructions[i];
6418 struct instruction_data *data = &instruction_data[i];
6419 uint32_t n_instr = 0;
6422 /* Mov all + validate. */
6423 detected = instr_pattern_mov_all_validate_search(p,
6433 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6438 /* No pattern starting at the current instruction. */
6442 /* Eliminate the invalid instructions that have been optimized out. */
6443 n_instructions = instr_compact(instructions,
6447 return n_instructions;
6451 instr_pattern_dma_many_search(struct instruction *instr,
6452 struct instruction_data *data,
6454 uint32_t *n_pattern_instr)
6458 for (i = 0; i < n_instr; i++) {
6459 if (data[i].invalid)
6462 if (instr[i].type != INSTR_DMA_HT)
6465 if (i == RTE_DIM(instr->dma.dst.header_id))
6468 if (i && data[i].n_users)
6475 *n_pattern_instr = i;
6480 instr_pattern_dma_many_replace(struct instruction *instr,
6481 struct instruction_data *data,
6486 for (i = 1; i < n_instr; i++) {
6488 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6489 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6490 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6491 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6493 data[i].invalid = 1;
6498 instr_pattern_dma_many_optimize(struct instruction *instructions,
6499 struct instruction_data *instruction_data,
6500 uint32_t n_instructions)
6504 for (i = 0; i < n_instructions; ) {
6505 struct instruction *instr = &instructions[i];
6506 struct instruction_data *data = &instruction_data[i];
6507 uint32_t n_instr = 0;
6511 detected = instr_pattern_dma_many_search(instr,
6516 instr_pattern_dma_many_replace(instr, data, n_instr);
6521 /* No pattern starting at the current instruction. */
6525 /* Eliminate the invalid instructions that have been optimized out. */
6526 n_instructions = instr_compact(instructions,
6530 return n_instructions;
6534 instr_optimize(struct rte_swx_pipeline *p,
6536 struct instruction *instructions,
6537 struct instruction_data *instruction_data,
6538 uint32_t n_instructions)
6541 n_instructions = instr_pattern_extract_many_optimize(instructions,
6545 /* Emit many + TX. */
6546 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6550 /* Mov all + validate. */
6551 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6558 n_instructions = instr_pattern_dma_many_optimize(instructions,
6562 return n_instructions;
6566 instruction_config(struct rte_swx_pipeline *p,
6568 const char **instructions,
6569 uint32_t n_instructions)
6571 struct instruction *instr = NULL;
6572 struct instruction_data *data = NULL;
6576 CHECK(n_instructions, EINVAL);
6577 CHECK(instructions, EINVAL);
6578 for (i = 0; i < n_instructions; i++)
6579 CHECK_INSTRUCTION(instructions[i], EINVAL);
6581 /* Memory allocation. */
6582 instr = calloc(n_instructions, sizeof(struct instruction));
6588 data = calloc(n_instructions, sizeof(struct instruction_data));
6594 for (i = 0; i < n_instructions; i++) {
6595 char *string = strdup(instructions[i]);
6601 err = instr_translate(p, a, string, &instr[i], &data[i]);
6610 err = instr_label_check(data, n_instructions);
6614 err = instr_verify(p, a, instr, data, n_instructions);
6618 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6620 err = instr_jmp_resolve(instr, data, n_instructions);
6625 a->instructions = instr;
6626 a->instruction_data = data;
6627 a->n_instructions = n_instructions;
6629 p->instructions = instr;
6630 p->instruction_data = data;
6631 p->n_instructions = n_instructions;
6642 static instr_exec_t instruction_table[] = {
6643 [INSTR_RX] = instr_rx_exec,
6644 [INSTR_TX] = instr_tx_exec,
6645 [INSTR_TX_I] = instr_tx_i_exec,
6647 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6648 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6649 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6650 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6651 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6652 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6653 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6654 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6655 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6656 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6658 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6659 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6660 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6661 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6662 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6663 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6664 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6665 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6666 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6668 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6669 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6671 [INSTR_MOV] = instr_mov_exec,
6672 [INSTR_MOV_MH] = instr_mov_mh_exec,
6673 [INSTR_MOV_HM] = instr_mov_hm_exec,
6674 [INSTR_MOV_HH] = instr_mov_hh_exec,
6675 [INSTR_MOV_I] = instr_mov_i_exec,
6677 [INSTR_DMA_HT] = instr_dma_ht_exec,
6678 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6679 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6680 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6681 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6682 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6683 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6684 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6686 [INSTR_ALU_ADD] = instr_alu_add_exec,
6687 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6688 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6689 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6690 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6691 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6693 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6694 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6695 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6696 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6697 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6698 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6700 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6701 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6702 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6703 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6705 [INSTR_ALU_AND] = instr_alu_and_exec,
6706 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6707 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6708 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6709 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6711 [INSTR_ALU_OR] = instr_alu_or_exec,
6712 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6713 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6714 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6715 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6717 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6718 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6719 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6720 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6721 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6723 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6724 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6725 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6726 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6727 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6728 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6730 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6731 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6732 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6733 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6734 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6735 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6737 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6738 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6739 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6741 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6742 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6743 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6744 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6745 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6746 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6748 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6749 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6750 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6751 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6752 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6753 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6754 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6755 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6756 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6758 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6759 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6760 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6761 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6762 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6763 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6764 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6765 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6766 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6768 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6769 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6770 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6772 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6773 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6774 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6775 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6776 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6777 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6778 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6779 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6780 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6781 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6782 [INSTR_METER_IMM] = instr_meter_imm_exec,
6783 [INSTR_METER_IMI] = instr_meter_imi_exec,
6785 [INSTR_TABLE] = instr_table_exec,
6786 [INSTR_TABLE_AF] = instr_table_af_exec,
6787 [INSTR_SELECTOR] = instr_selector_exec,
6788 [INSTR_LEARNER] = instr_learner_exec,
6789 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6790 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6791 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6792 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6793 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6795 [INSTR_JMP] = instr_jmp_exec,
6796 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6797 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6798 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6799 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6800 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6801 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6803 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6804 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6805 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6806 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6807 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6809 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6810 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6811 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6812 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6813 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6815 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6816 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6817 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6818 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6819 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6820 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6822 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6823 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6824 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6825 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6826 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6827 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6829 [INSTR_RETURN] = instr_return_exec,
6833 instruction_table_build(struct rte_swx_pipeline *p)
6835 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6836 sizeof(struct instr_exec_t *));
6837 if (!p->instruction_table)
6840 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6846 instruction_table_build_free(struct rte_swx_pipeline *p)
6848 if (!p->instruction_table)
6851 free(p->instruction_table);
6852 p->instruction_table = NULL;
6856 instruction_table_free(struct rte_swx_pipeline *p)
6858 instruction_table_build_free(p);
6862 instr_exec(struct rte_swx_pipeline *p)
6864 struct thread *t = &p->threads[p->thread_id];
6865 struct instruction *ip = t->ip;
6866 instr_exec_t instr = p->instruction_table[ip->type];
6874 static struct action *
6875 action_find(struct rte_swx_pipeline *p, const char *name)
6877 struct action *elem;
6882 TAILQ_FOREACH(elem, &p->actions, node)
6883 if (strcmp(elem->name, name) == 0)
6889 static struct action *
6890 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6892 struct action *action = NULL;
6894 TAILQ_FOREACH(action, &p->actions, node)
6895 if (action->id == id)
6901 static struct field *
6902 action_field_find(struct action *a, const char *name)
6904 return a->st ? struct_type_field_find(a->st, name) : NULL;
6907 static struct field *
6908 action_field_parse(struct action *action, const char *name)
6910 if (name[0] != 't' || name[1] != '.')
6913 return action_field_find(action, &name[2]);
6917 action_has_nbo_args(struct action *a)
6921 /* Return if the action does not have any args. */
6923 return 0; /* FALSE */
6925 for (i = 0; i < a->st->n_fields; i++)
6926 if (a->args_endianness[i])
6927 return 1; /* TRUE */
6929 return 0; /* FALSE */
6933 action_does_learning(struct action *a)
6937 for (i = 0; i < a->n_instructions; i++)
6938 switch (a->instructions[i].type) {
6939 case INSTR_LEARNER_LEARN:
6940 return 1; /* TRUE */
6942 case INSTR_LEARNER_FORGET:
6943 return 1; /* TRUE */
6949 return 0; /* FALSE */
6953 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6955 const char *args_struct_type_name,
6956 const char **instructions,
6957 uint32_t n_instructions)
6959 struct struct_type *args_struct_type = NULL;
6965 CHECK_NAME(name, EINVAL);
6966 CHECK(!action_find(p, name), EEXIST);
6968 if (args_struct_type_name) {
6969 CHECK_NAME(args_struct_type_name, EINVAL);
6970 args_struct_type = struct_type_find(p, args_struct_type_name);
6971 CHECK(args_struct_type, EINVAL);
6972 CHECK(!args_struct_type->var_size, EINVAL);
6975 /* Node allocation. */
6976 a = calloc(1, sizeof(struct action));
6978 if (args_struct_type) {
6979 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6980 if (!a->args_endianness) {
6986 /* Node initialization. */
6987 strcpy(a->name, name);
6988 a->st = args_struct_type;
6989 a->id = p->n_actions;
6991 /* Instruction translation. */
6992 err = instruction_config(p, a, instructions, n_instructions);
6994 free(a->args_endianness);
6999 /* Node add to tailq. */
7000 TAILQ_INSERT_TAIL(&p->actions, a, node);
7007 action_build(struct rte_swx_pipeline *p)
7009 struct action *action;
7011 /* p->action_instructions. */
7012 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7013 CHECK(p->action_instructions, ENOMEM);
7015 TAILQ_FOREACH(action, &p->actions, node)
7016 p->action_instructions[action->id] = action->instructions;
7018 /* p->action_funcs. */
7019 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7020 CHECK(p->action_funcs, ENOMEM);
7026 action_build_free(struct rte_swx_pipeline *p)
7028 free(p->action_funcs);
7029 p->action_funcs = NULL;
7031 free(p->action_instructions);
7032 p->action_instructions = NULL;
7036 action_free(struct rte_swx_pipeline *p)
7038 action_build_free(p);
7041 struct action *action;
7043 action = TAILQ_FIRST(&p->actions);
7047 TAILQ_REMOVE(&p->actions, action, node);
7048 free(action->instruction_data);
7049 free(action->instructions);
7055 action_arg_src_mov_count(struct action *a,
7057 struct instruction *instructions,
7058 struct instruction_data *instruction_data,
7059 uint32_t n_instructions)
7061 uint32_t offset, n_users = 0, i;
7064 (arg_id >= a->st->n_fields) ||
7066 !instruction_data ||
7070 offset = a->st->fields[arg_id].offset / 8;
7072 for (i = 0; i < n_instructions; i++) {
7073 struct instruction *instr = &instructions[i];
7074 struct instruction_data *data = &instruction_data[i];
7076 if (data->invalid ||
7077 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7078 instr->mov.src.struct_id ||
7079 (instr->mov.src.offset != offset))
7091 static struct table_type *
7092 table_type_find(struct rte_swx_pipeline *p, const char *name)
7094 struct table_type *elem;
7096 TAILQ_FOREACH(elem, &p->table_types, node)
7097 if (strcmp(elem->name, name) == 0)
7103 static struct table_type *
7104 table_type_resolve(struct rte_swx_pipeline *p,
7105 const char *recommended_type_name,
7106 enum rte_swx_table_match_type match_type)
7108 struct table_type *elem;
7110 /* Only consider the recommended type if the match type is correct. */
7111 if (recommended_type_name)
7112 TAILQ_FOREACH(elem, &p->table_types, node)
7113 if (!strcmp(elem->name, recommended_type_name) &&
7114 (elem->match_type == match_type))
7117 /* Ignore the recommended type and get the first element with this match
7120 TAILQ_FOREACH(elem, &p->table_types, node)
7121 if (elem->match_type == match_type)
7127 static struct table *
7128 table_find(struct rte_swx_pipeline *p, const char *name)
7132 TAILQ_FOREACH(elem, &p->tables, node)
7133 if (strcmp(elem->name, name) == 0)
7139 static struct table *
7140 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7142 struct table *table = NULL;
7144 TAILQ_FOREACH(table, &p->tables, node)
7145 if (table->id == id)
7152 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7154 enum rte_swx_table_match_type match_type,
7155 struct rte_swx_table_ops *ops)
7157 struct table_type *elem;
7161 CHECK_NAME(name, EINVAL);
7162 CHECK(!table_type_find(p, name), EEXIST);
7165 CHECK(ops->create, EINVAL);
7166 CHECK(ops->lkp, EINVAL);
7167 CHECK(ops->free, EINVAL);
7169 /* Node allocation. */
7170 elem = calloc(1, sizeof(struct table_type));
7171 CHECK(elem, ENOMEM);
7173 /* Node initialization. */
7174 strcpy(elem->name, name);
7175 elem->match_type = match_type;
7176 memcpy(&elem->ops, ops, sizeof(*ops));
7178 /* Node add to tailq. */
7179 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7185 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7187 enum rte_swx_table_match_type *match_type)
7189 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7191 for (i = 0; i < n_fields; i++) {
7192 struct rte_swx_match_field_params *f = &fields[i];
7194 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7197 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7201 if ((n_fields_lpm > 1) ||
7202 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7205 *match_type = (n_fields_em == n_fields) ?
7206 RTE_SWX_TABLE_MATCH_EXACT :
7207 RTE_SWX_TABLE_MATCH_WILDCARD;
7213 table_match_fields_check(struct rte_swx_pipeline *p,
7214 struct rte_swx_pipeline_table_params *params,
7215 struct header **header)
7217 struct header *h0 = NULL;
7218 struct field *hf, *mf;
7219 uint32_t *offset = NULL, i;
7222 /* Return if no match fields. */
7223 if (!params->n_fields) {
7224 if (params->fields) {
7235 /* Memory allocation. */
7236 offset = calloc(params->n_fields, sizeof(uint32_t));
7242 /* Check that all the match fields belong to either the same header or
7245 hf = header_field_parse(p, params->fields[0].name, &h0);
7246 mf = metadata_field_parse(p, params->fields[0].name);
7247 if ((!hf && !mf) || (hf && hf->var_size)) {
7252 offset[0] = h0 ? hf->offset : mf->offset;
7254 for (i = 1; i < params->n_fields; i++)
7258 hf = header_field_parse(p, params->fields[i].name, &h);
7259 if (!hf || (h->id != h0->id) || hf->var_size) {
7264 offset[i] = hf->offset;
7266 mf = metadata_field_parse(p, params->fields[i].name);
7272 offset[i] = mf->offset;
7275 /* Check that there are no duplicated match fields. */
7276 for (i = 0; i < params->n_fields; i++) {
7279 for (j = 0; j < i; j++)
7280 if (offset[j] == offset[i]) {
7296 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7298 struct rte_swx_pipeline_table_params *params,
7299 const char *recommended_table_type_name,
7303 struct table_type *type;
7304 struct table *t = NULL;
7305 struct action *default_action;
7306 struct header *header = NULL;
7307 uint32_t action_data_size_max = 0, i;
7312 CHECK_NAME(name, EINVAL);
7313 CHECK(!table_find(p, name), EEXIST);
7314 CHECK(!selector_find(p, name), EEXIST);
7315 CHECK(!learner_find(p, name), EEXIST);
7317 CHECK(params, EINVAL);
7320 status = table_match_fields_check(p, params, &header);
7324 /* Action checks. */
7325 CHECK(params->n_actions, EINVAL);
7326 CHECK(params->action_names, EINVAL);
7327 for (i = 0; i < params->n_actions; i++) {
7328 const char *action_name = params->action_names[i];
7330 uint32_t action_data_size;
7331 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7333 CHECK_NAME(action_name, EINVAL);
7335 a = action_find(p, action_name);
7337 CHECK(!action_does_learning(a), EINVAL);
7339 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7340 if (action_data_size > action_data_size_max)
7341 action_data_size_max = action_data_size;
7343 if (params->action_is_for_table_entries)
7344 action_is_for_table_entries = params->action_is_for_table_entries[i];
7345 if (params->action_is_for_default_entry)
7346 action_is_for_default_entry = params->action_is_for_default_entry[i];
7347 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7350 CHECK_NAME(params->default_action_name, EINVAL);
7351 for (i = 0; i < p->n_actions; i++)
7352 if (!strcmp(params->action_names[i],
7353 params->default_action_name))
7355 CHECK(i < params->n_actions, EINVAL);
7356 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7359 default_action = action_find(p, params->default_action_name);
7360 CHECK((default_action->st && params->default_action_data) ||
7361 !params->default_action_data, EINVAL);
7363 /* Table type checks. */
7364 if (recommended_table_type_name)
7365 CHECK_NAME(recommended_table_type_name, EINVAL);
7367 if (params->n_fields) {
7368 enum rte_swx_table_match_type match_type;
7370 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7374 type = table_type_resolve(p, recommended_table_type_name, match_type);
7375 CHECK(type, EINVAL);
7380 /* Memory allocation. */
7381 t = calloc(1, sizeof(struct table));
7385 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7389 t->actions = calloc(params->n_actions, sizeof(struct action *));
7393 if (action_data_size_max) {
7394 t->default_action_data = calloc(1, action_data_size_max);
7395 if (!t->default_action_data)
7399 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7400 if (!t->action_is_for_table_entries)
7403 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7404 if (!t->action_is_for_default_entry)
7407 /* Node initialization. */
7408 strcpy(t->name, name);
7409 if (args && args[0])
7410 strcpy(t->args, args);
7413 for (i = 0; i < params->n_fields; i++) {
7414 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7415 struct match_field *f = &t->fields[i];
7417 f->match_type = field->match_type;
7419 header_field_parse(p, field->name, NULL) :
7420 metadata_field_parse(p, field->name);
7422 t->n_fields = params->n_fields;
7425 for (i = 0; i < params->n_actions; i++) {
7426 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7428 if (params->action_is_for_table_entries)
7429 action_is_for_table_entries = params->action_is_for_table_entries[i];
7430 if (params->action_is_for_default_entry)
7431 action_is_for_default_entry = params->action_is_for_default_entry[i];
7433 t->actions[i] = action_find(p, params->action_names[i]);
7434 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7435 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7437 t->default_action = default_action;
7438 if (default_action->st)
7439 memcpy(t->default_action_data,
7440 params->default_action_data,
7441 default_action->st->n_bits / 8);
7442 t->n_actions = params->n_actions;
7443 t->default_action_is_const = params->default_action_is_const;
7444 t->action_data_size_max = action_data_size_max;
7447 t->id = p->n_tables;
7449 /* Node add to tailq. */
7450 TAILQ_INSERT_TAIL(&p->tables, t, node);
7459 free(t->action_is_for_default_entry);
7460 free(t->action_is_for_table_entries);
7461 free(t->default_action_data);
7469 static struct rte_swx_table_params *
7470 table_params_get(struct table *table)
7472 struct rte_swx_table_params *params;
7473 struct field *first, *last;
7475 uint32_t key_size, key_offset, action_data_size, i;
7477 /* Memory allocation. */
7478 params = calloc(1, sizeof(struct rte_swx_table_params));
7482 /* Find first (smallest offset) and last (biggest offset) match fields. */
7483 first = table->fields[0].field;
7484 last = table->fields[0].field;
7486 for (i = 0; i < table->n_fields; i++) {
7487 struct field *f = table->fields[i].field;
7489 if (f->offset < first->offset)
7492 if (f->offset > last->offset)
7496 /* Key offset and size. */
7497 key_offset = first->offset / 8;
7498 key_size = (last->offset + last->n_bits - first->offset) / 8;
7500 /* Memory allocation. */
7501 key_mask = calloc(1, key_size);
7508 for (i = 0; i < table->n_fields; i++) {
7509 struct field *f = table->fields[i].field;
7510 uint32_t start = (f->offset - first->offset) / 8;
7511 size_t size = f->n_bits / 8;
7513 memset(&key_mask[start], 0xFF, size);
7516 /* Action data size. */
7517 action_data_size = 0;
7518 for (i = 0; i < table->n_actions; i++) {
7519 struct action *action = table->actions[i];
7520 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7522 if (ads > action_data_size)
7523 action_data_size = ads;
7527 params->match_type = table->type->match_type;
7528 params->key_size = key_size;
7529 params->key_offset = key_offset;
7530 params->key_mask0 = key_mask;
7531 params->action_data_size = action_data_size;
7532 params->n_keys_max = table->size;
7538 table_params_free(struct rte_swx_table_params *params)
7543 free(params->key_mask0);
7548 table_stub_lkp(void *table __rte_unused,
7549 void *mailbox __rte_unused,
7550 uint8_t **key __rte_unused,
7551 uint64_t *action_id __rte_unused,
7552 uint8_t **action_data __rte_unused,
7556 return 1; /* DONE. */
7560 table_build(struct rte_swx_pipeline *p)
7564 /* Per pipeline: table statistics. */
7565 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7566 CHECK(p->table_stats, ENOMEM);
7568 for (i = 0; i < p->n_tables; i++) {
7569 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7570 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7573 /* Per thread: table runt-time. */
7574 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7575 struct thread *t = &p->threads[i];
7576 struct table *table;
7578 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7579 CHECK(t->tables, ENOMEM);
7581 TAILQ_FOREACH(table, &p->tables, node) {
7582 struct table_runtime *r = &t->tables[table->id];
7587 size = table->type->ops.mailbox_size_get();
7590 r->func = table->type->ops.lkp;
7594 r->mailbox = calloc(1, size);
7595 CHECK(r->mailbox, ENOMEM);
7599 r->key = table->header ?
7600 &t->structs[table->header->struct_id] :
7601 &t->structs[p->metadata_struct_id];
7603 r->func = table_stub_lkp;
7612 table_build_free(struct rte_swx_pipeline *p)
7616 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7617 struct thread *t = &p->threads[i];
7623 for (j = 0; j < p->n_tables; j++) {
7624 struct table_runtime *r = &t->tables[j];
7633 if (p->table_stats) {
7634 for (i = 0; i < p->n_tables; i++)
7635 free(p->table_stats[i].n_pkts_action);
7637 free(p->table_stats);
7642 table_free(struct rte_swx_pipeline *p)
7644 table_build_free(p);
7650 elem = TAILQ_FIRST(&p->tables);
7654 TAILQ_REMOVE(&p->tables, elem, node);
7656 free(elem->actions);
7657 free(elem->default_action_data);
7663 struct table_type *elem;
7665 elem = TAILQ_FIRST(&p->table_types);
7669 TAILQ_REMOVE(&p->table_types, elem, node);
7677 static struct selector *
7678 selector_find(struct rte_swx_pipeline *p, const char *name)
7682 TAILQ_FOREACH(s, &p->selectors, node)
7683 if (strcmp(s->name, name) == 0)
7689 static struct selector *
7690 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7692 struct selector *s = NULL;
7694 TAILQ_FOREACH(s, &p->selectors, node)
7702 selector_fields_check(struct rte_swx_pipeline *p,
7703 struct rte_swx_pipeline_selector_params *params,
7704 struct header **header)
7706 struct header *h0 = NULL;
7707 struct field *hf, *mf;
7710 /* Return if no selector fields. */
7711 if (!params->n_selector_fields || !params->selector_field_names)
7714 /* Check that all the selector fields either belong to the same header
7715 * or are all meta-data fields.
7717 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7718 mf = metadata_field_parse(p, params->selector_field_names[0]);
7722 for (i = 1; i < params->n_selector_fields; i++)
7726 hf = header_field_parse(p, params->selector_field_names[i], &h);
7727 if (!hf || (h->id != h0->id))
7730 mf = metadata_field_parse(p, params->selector_field_names[i]);
7735 /* Check that there are no duplicated match fields. */
7736 for (i = 0; i < params->n_selector_fields; i++) {
7737 const char *field_name = params->selector_field_names[i];
7740 for (j = i + 1; j < params->n_selector_fields; j++)
7741 if (!strcmp(params->selector_field_names[j], field_name))
7753 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7755 struct rte_swx_pipeline_selector_params *params)
7758 struct header *selector_header = NULL;
7759 struct field *group_id_field, *member_id_field;
7765 CHECK_NAME(name, EINVAL);
7766 CHECK(!table_find(p, name), EEXIST);
7767 CHECK(!selector_find(p, name), EEXIST);
7768 CHECK(!learner_find(p, name), EEXIST);
7770 CHECK(params, EINVAL);
7772 CHECK_NAME(params->group_id_field_name, EINVAL);
7773 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7774 CHECK(group_id_field, EINVAL);
7776 for (i = 0; i < params->n_selector_fields; i++) {
7777 const char *field_name = params->selector_field_names[i];
7779 CHECK_NAME(field_name, EINVAL);
7781 status = selector_fields_check(p, params, &selector_header);
7785 CHECK_NAME(params->member_id_field_name, EINVAL);
7786 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7787 CHECK(member_id_field, EINVAL);
7789 CHECK(params->n_groups_max, EINVAL);
7791 CHECK(params->n_members_per_group_max, EINVAL);
7793 /* Memory allocation. */
7794 s = calloc(1, sizeof(struct selector));
7800 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7801 if (!s->selector_fields) {
7806 /* Node initialization. */
7807 strcpy(s->name, name);
7809 s->group_id_field = group_id_field;
7811 for (i = 0; i < params->n_selector_fields; i++) {
7812 const char *field_name = params->selector_field_names[i];
7814 s->selector_fields[i] = selector_header ?
7815 header_field_parse(p, field_name, NULL) :
7816 metadata_field_parse(p, field_name);
7819 s->n_selector_fields = params->n_selector_fields;
7821 s->selector_header = selector_header;
7823 s->member_id_field = member_id_field;
7825 s->n_groups_max = params->n_groups_max;
7827 s->n_members_per_group_max = params->n_members_per_group_max;
7829 s->id = p->n_selectors;
7831 /* Node add to tailq. */
7832 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7841 free(s->selector_fields);
7849 selector_params_free(struct rte_swx_table_selector_params *params)
7854 free(params->selector_mask);
7859 static struct rte_swx_table_selector_params *
7860 selector_table_params_get(struct selector *s)
7862 struct rte_swx_table_selector_params *params = NULL;
7863 struct field *first, *last;
7866 /* Memory allocation. */
7867 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7872 params->group_id_offset = s->group_id_field->offset / 8;
7874 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7875 first = s->selector_fields[0];
7876 last = s->selector_fields[0];
7878 for (i = 0; i < s->n_selector_fields; i++) {
7879 struct field *f = s->selector_fields[i];
7881 if (f->offset < first->offset)
7884 if (f->offset > last->offset)
7888 /* Selector offset and size. */
7889 params->selector_offset = first->offset / 8;
7890 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7892 /* Memory allocation. */
7893 params->selector_mask = calloc(1, params->selector_size);
7894 if (!params->selector_mask)
7897 /* Selector mask. */
7898 for (i = 0; i < s->n_selector_fields; i++) {
7899 struct field *f = s->selector_fields[i];
7900 uint32_t start = (f->offset - first->offset) / 8;
7901 size_t size = f->n_bits / 8;
7903 memset(¶ms->selector_mask[start], 0xFF, size);
7907 params->member_id_offset = s->member_id_field->offset / 8;
7909 /* Maximum number of groups. */
7910 params->n_groups_max = s->n_groups_max;
7912 /* Maximum number of members per group. */
7913 params->n_members_per_group_max = s->n_members_per_group_max;
7918 selector_params_free(params);
7923 selector_build_free(struct rte_swx_pipeline *p)
7927 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7928 struct thread *t = &p->threads[i];
7934 for (j = 0; j < p->n_selectors; j++) {
7935 struct selector_runtime *r = &t->selectors[j];
7941 t->selectors = NULL;
7944 free(p->selector_stats);
7945 p->selector_stats = NULL;
7949 selector_build(struct rte_swx_pipeline *p)
7954 /* Per pipeline: selector statistics. */
7955 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7956 if (!p->selector_stats) {
7961 /* Per thread: selector run-time. */
7962 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7963 struct thread *t = &p->threads[i];
7966 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7967 if (!t->selectors) {
7972 TAILQ_FOREACH(s, &p->selectors, node) {
7973 struct selector_runtime *r = &t->selectors[s->id];
7977 size = rte_swx_table_selector_mailbox_size_get();
7979 r->mailbox = calloc(1, size);
7986 /* r->group_id_buffer. */
7987 r->group_id_buffer = &t->structs[p->metadata_struct_id];
7989 /* r->selector_buffer. */
7990 r->selector_buffer = s->selector_header ?
7991 &t->structs[s->selector_header->struct_id] :
7992 &t->structs[p->metadata_struct_id];
7994 /* r->member_id_buffer. */
7995 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8002 selector_build_free(p);
8007 selector_free(struct rte_swx_pipeline *p)
8009 selector_build_free(p);
8011 /* Selector tables. */
8013 struct selector *elem;
8015 elem = TAILQ_FIRST(&p->selectors);
8019 TAILQ_REMOVE(&p->selectors, elem, node);
8020 free(elem->selector_fields);
8028 static struct learner *
8029 learner_find(struct rte_swx_pipeline *p, const char *name)
8033 TAILQ_FOREACH(l, &p->learners, node)
8034 if (!strcmp(l->name, name))
8040 static struct learner *
8041 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8043 struct learner *l = NULL;
8045 TAILQ_FOREACH(l, &p->learners, node)
8053 learner_match_fields_check(struct rte_swx_pipeline *p,
8054 struct rte_swx_pipeline_learner_params *params,
8055 struct header **header)
8057 struct header *h0 = NULL;
8058 struct field *hf, *mf;
8061 /* Return if no match fields. */
8062 if (!params->n_fields || !params->field_names)
8065 /* Check that all the match fields either belong to the same header
8066 * or are all meta-data fields.
8068 hf = header_field_parse(p, params->field_names[0], &h0);
8069 mf = metadata_field_parse(p, params->field_names[0]);
8073 for (i = 1; i < params->n_fields; i++)
8077 hf = header_field_parse(p, params->field_names[i], &h);
8078 if (!hf || (h->id != h0->id))
8081 mf = metadata_field_parse(p, params->field_names[i]);
8086 /* Check that there are no duplicated match fields. */
8087 for (i = 0; i < params->n_fields; i++) {
8088 const char *field_name = params->field_names[i];
8091 for (j = i + 1; j < params->n_fields; j++)
8092 if (!strcmp(params->field_names[j], field_name))
8104 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8106 struct struct_type *mst = p->metadata_st, *ast = a->st;
8107 struct field *mf, *af;
8117 /* Check that mf_name is the name of a valid meta-data field. */
8118 CHECK_NAME(mf_name, EINVAL);
8119 mf = metadata_field_parse(p, mf_name);
8122 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8123 * all the action arguments.
8125 mf_pos = mf - mst->fields;
8126 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8128 /* Check that the size of each of the identified meta-data fields matches exactly the size
8129 * of the corresponding action argument.
8131 for (i = 0; i < ast->n_fields; i++) {
8132 mf = &mst->fields[mf_pos + i];
8133 af = &ast->fields[i];
8135 CHECK(mf->n_bits == af->n_bits, EINVAL);
8142 learner_action_learning_check(struct rte_swx_pipeline *p,
8143 struct action *action,
8144 const char **action_names,
8149 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8150 * the action passed as argument to the "learn" instruction) is also enabled for the
8151 * current learner table.
8153 for (i = 0; i < action->n_instructions; i++) {
8154 struct instruction *instr = &action->instructions[i];
8155 uint32_t found = 0, j;
8157 if (instr->type != INSTR_LEARNER_LEARN)
8160 for (j = 0; j < n_actions; j++) {
8163 a = action_find(p, action_names[j]);
8167 if (a->id == instr->learn.action_id)
8179 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8181 struct rte_swx_pipeline_learner_params *params,
8185 struct learner *l = NULL;
8186 struct action *default_action;
8187 struct header *header = NULL;
8188 uint32_t action_data_size_max = 0, i;
8193 CHECK_NAME(name, EINVAL);
8194 CHECK(!table_find(p, name), EEXIST);
8195 CHECK(!selector_find(p, name), EEXIST);
8196 CHECK(!learner_find(p, name), EEXIST);
8198 CHECK(params, EINVAL);
8201 status = learner_match_fields_check(p, params, &header);
8205 /* Action checks. */
8206 CHECK(params->n_actions, EINVAL);
8207 CHECK(params->action_names, EINVAL);
8208 for (i = 0; i < params->n_actions; i++) {
8209 const char *action_name = params->action_names[i];
8211 uint32_t action_data_size;
8212 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8214 CHECK_NAME(action_name, EINVAL);
8216 a = action_find(p, action_name);
8219 status = learner_action_learning_check(p,
8221 params->action_names,
8226 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8227 if (action_data_size > action_data_size_max)
8228 action_data_size_max = action_data_size;
8230 if (params->action_is_for_table_entries)
8231 action_is_for_table_entries = params->action_is_for_table_entries[i];
8232 if (params->action_is_for_default_entry)
8233 action_is_for_default_entry = params->action_is_for_default_entry[i];
8234 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8237 CHECK_NAME(params->default_action_name, EINVAL);
8238 for (i = 0; i < p->n_actions; i++)
8239 if (!strcmp(params->action_names[i],
8240 params->default_action_name))
8242 CHECK(i < params->n_actions, EINVAL);
8243 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8246 default_action = action_find(p, params->default_action_name);
8247 CHECK((default_action->st && params->default_action_data) ||
8248 !params->default_action_data, EINVAL);
8250 /* Any other checks. */
8251 CHECK(size, EINVAL);
8252 CHECK(timeout, EINVAL);
8254 /* Memory allocation. */
8255 l = calloc(1, sizeof(struct learner));
8259 l->fields = calloc(params->n_fields, sizeof(struct field *));
8263 l->actions = calloc(params->n_actions, sizeof(struct action *));
8267 if (action_data_size_max) {
8268 l->default_action_data = calloc(1, action_data_size_max);
8269 if (!l->default_action_data)
8273 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8274 if (!l->action_is_for_table_entries)
8277 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8278 if (!l->action_is_for_default_entry)
8281 /* Node initialization. */
8282 strcpy(l->name, name);
8284 for (i = 0; i < params->n_fields; i++) {
8285 const char *field_name = params->field_names[i];
8287 l->fields[i] = header ?
8288 header_field_parse(p, field_name, NULL) :
8289 metadata_field_parse(p, field_name);
8292 l->n_fields = params->n_fields;
8296 for (i = 0; i < params->n_actions; i++) {
8297 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8299 if (params->action_is_for_table_entries)
8300 action_is_for_table_entries = params->action_is_for_table_entries[i];
8301 if (params->action_is_for_default_entry)
8302 action_is_for_default_entry = params->action_is_for_default_entry[i];
8304 l->actions[i] = action_find(p, params->action_names[i]);
8305 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8306 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8309 l->default_action = default_action;
8311 if (default_action->st)
8312 memcpy(l->default_action_data,
8313 params->default_action_data,
8314 default_action->st->n_bits / 8);
8316 l->n_actions = params->n_actions;
8318 l->default_action_is_const = params->default_action_is_const;
8320 l->action_data_size_max = action_data_size_max;
8324 l->timeout = timeout;
8326 l->id = p->n_learners;
8328 /* Node add to tailq. */
8329 TAILQ_INSERT_TAIL(&p->learners, l, node);
8338 free(l->action_is_for_default_entry);
8339 free(l->action_is_for_table_entries);
8340 free(l->default_action_data);
8349 learner_params_free(struct rte_swx_table_learner_params *params)
8354 free(params->key_mask0);
8359 static struct rte_swx_table_learner_params *
8360 learner_params_get(struct learner *l)
8362 struct rte_swx_table_learner_params *params = NULL;
8363 struct field *first, *last;
8366 /* Memory allocation. */
8367 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8371 /* Find first (smallest offset) and last (biggest offset) match fields. */
8372 first = l->fields[0];
8373 last = l->fields[0];
8375 for (i = 0; i < l->n_fields; i++) {
8376 struct field *f = l->fields[i];
8378 if (f->offset < first->offset)
8381 if (f->offset > last->offset)
8385 /* Key offset and size. */
8386 params->key_offset = first->offset / 8;
8387 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8389 /* Memory allocation. */
8390 params->key_mask0 = calloc(1, params->key_size);
8391 if (!params->key_mask0)
8395 for (i = 0; i < l->n_fields; i++) {
8396 struct field *f = l->fields[i];
8397 uint32_t start = (f->offset - first->offset) / 8;
8398 size_t size = f->n_bits / 8;
8400 memset(¶ms->key_mask0[start], 0xFF, size);
8403 /* Action data size. */
8404 params->action_data_size = l->action_data_size_max;
8406 /* Maximum number of keys. */
8407 params->n_keys_max = l->size;
8410 params->key_timeout = l->timeout;
8415 learner_params_free(params);
8420 learner_build_free(struct rte_swx_pipeline *p)
8424 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8425 struct thread *t = &p->threads[i];
8431 for (j = 0; j < p->n_learners; j++) {
8432 struct learner_runtime *r = &t->learners[j];
8441 if (p->learner_stats) {
8442 for (i = 0; i < p->n_learners; i++)
8443 free(p->learner_stats[i].n_pkts_action);
8445 free(p->learner_stats);
8450 learner_build(struct rte_swx_pipeline *p)
8455 /* Per pipeline: learner statistics. */
8456 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8457 CHECK(p->learner_stats, ENOMEM);
8459 for (i = 0; i < p->n_learners; i++) {
8460 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8461 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8464 /* Per thread: learner run-time. */
8465 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8466 struct thread *t = &p->threads[i];
8469 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8475 TAILQ_FOREACH(l, &p->learners, node) {
8476 struct learner_runtime *r = &t->learners[l->id];
8480 size = rte_swx_table_learner_mailbox_size_get();
8482 r->mailbox = calloc(1, size);
8490 r->key = l->header ?
8491 &t->structs[l->header->struct_id] :
8492 &t->structs[p->metadata_struct_id];
8499 learner_build_free(p);
8504 learner_free(struct rte_swx_pipeline *p)
8506 learner_build_free(p);
8508 /* Learner tables. */
8512 l = TAILQ_FIRST(&p->learners);
8516 TAILQ_REMOVE(&p->learners, l, node);
8519 free(l->default_action_data);
8528 table_state_build(struct rte_swx_pipeline *p)
8530 struct table *table;
8534 p->table_state = calloc(p->n_tables + p->n_selectors,
8535 sizeof(struct rte_swx_table_state));
8536 CHECK(p->table_state, ENOMEM);
8538 TAILQ_FOREACH(table, &p->tables, node) {
8539 struct rte_swx_table_state *ts = &p->table_state[table->id];
8542 struct rte_swx_table_params *params;
8545 params = table_params_get(table);
8546 CHECK(params, ENOMEM);
8548 ts->obj = table->type->ops.create(params,
8553 table_params_free(params);
8554 CHECK(ts->obj, ENODEV);
8557 /* ts->default_action_data. */
8558 if (table->action_data_size_max) {
8559 ts->default_action_data =
8560 malloc(table->action_data_size_max);
8561 CHECK(ts->default_action_data, ENOMEM);
8563 memcpy(ts->default_action_data,
8564 table->default_action_data,
8565 table->action_data_size_max);
8568 /* ts->default_action_id. */
8569 ts->default_action_id = table->default_action->id;
8572 TAILQ_FOREACH(s, &p->selectors, node) {
8573 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8574 struct rte_swx_table_selector_params *params;
8577 params = selector_table_params_get(s);
8578 CHECK(params, ENOMEM);
8580 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8582 selector_params_free(params);
8583 CHECK(ts->obj, ENODEV);
8586 TAILQ_FOREACH(l, &p->learners, node) {
8587 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8588 p->n_selectors + l->id];
8589 struct rte_swx_table_learner_params *params;
8592 params = learner_params_get(l);
8593 CHECK(params, ENOMEM);
8595 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8596 learner_params_free(params);
8597 CHECK(ts->obj, ENODEV);
8599 /* ts->default_action_data. */
8600 if (l->action_data_size_max) {
8601 ts->default_action_data = malloc(l->action_data_size_max);
8602 CHECK(ts->default_action_data, ENOMEM);
8604 memcpy(ts->default_action_data,
8605 l->default_action_data,
8606 l->action_data_size_max);
8609 /* ts->default_action_id. */
8610 ts->default_action_id = l->default_action->id;
8617 table_state_build_free(struct rte_swx_pipeline *p)
8621 if (!p->table_state)
8624 for (i = 0; i < p->n_tables; i++) {
8625 struct rte_swx_table_state *ts = &p->table_state[i];
8626 struct table *table = table_find_by_id(p, i);
8629 if (table->type && ts->obj)
8630 table->type->ops.free(ts->obj);
8632 /* ts->default_action_data. */
8633 free(ts->default_action_data);
8636 for (i = 0; i < p->n_selectors; i++) {
8637 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8641 rte_swx_table_selector_free(ts->obj);
8644 for (i = 0; i < p->n_learners; i++) {
8645 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8649 rte_swx_table_learner_free(ts->obj);
8651 /* ts->default_action_data. */
8652 free(ts->default_action_data);
8655 free(p->table_state);
8656 p->table_state = NULL;
8660 table_state_free(struct rte_swx_pipeline *p)
8662 table_state_build_free(p);
8668 static struct regarray *
8669 regarray_find(struct rte_swx_pipeline *p, const char *name)
8671 struct regarray *elem;
8673 TAILQ_FOREACH(elem, &p->regarrays, node)
8674 if (!strcmp(elem->name, name))
8680 static struct regarray *
8681 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8683 struct regarray *elem = NULL;
8685 TAILQ_FOREACH(elem, &p->regarrays, node)
8693 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8702 CHECK_NAME(name, EINVAL);
8703 CHECK(!regarray_find(p, name), EEXIST);
8705 CHECK(size, EINVAL);
8706 size = rte_align32pow2(size);
8708 /* Memory allocation. */
8709 r = calloc(1, sizeof(struct regarray));
8712 /* Node initialization. */
8713 strcpy(r->name, name);
8714 r->init_val = init_val;
8716 r->id = p->n_regarrays;
8718 /* Node add to tailq. */
8719 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8726 regarray_build(struct rte_swx_pipeline *p)
8728 struct regarray *regarray;
8730 if (!p->n_regarrays)
8733 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8734 CHECK(p->regarray_runtime, ENOMEM);
8736 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8737 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8740 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8741 RTE_CACHE_LINE_SIZE,
8743 CHECK(r->regarray, ENOMEM);
8745 if (regarray->init_val)
8746 for (i = 0; i < regarray->size; i++)
8747 r->regarray[i] = regarray->init_val;
8749 r->size_mask = regarray->size - 1;
8756 regarray_build_free(struct rte_swx_pipeline *p)
8760 if (!p->regarray_runtime)
8763 for (i = 0; i < p->n_regarrays; i++) {
8764 struct regarray *regarray = regarray_find_by_id(p, i);
8765 struct regarray_runtime *r = &p->regarray_runtime[i];
8767 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8770 free(p->regarray_runtime);
8771 p->regarray_runtime = NULL;
8775 regarray_free(struct rte_swx_pipeline *p)
8777 regarray_build_free(p);
8780 struct regarray *elem;
8782 elem = TAILQ_FIRST(&p->regarrays);
8786 TAILQ_REMOVE(&p->regarrays, elem, node);
8794 static struct meter_profile *
8795 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8797 struct meter_profile *elem;
8799 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8800 if (!strcmp(elem->name, name))
8806 static struct metarray *
8807 metarray_find(struct rte_swx_pipeline *p, const char *name)
8809 struct metarray *elem;
8811 TAILQ_FOREACH(elem, &p->metarrays, node)
8812 if (!strcmp(elem->name, name))
8818 static struct metarray *
8819 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8821 struct metarray *elem = NULL;
8823 TAILQ_FOREACH(elem, &p->metarrays, node)
8831 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8839 CHECK_NAME(name, EINVAL);
8840 CHECK(!metarray_find(p, name), EEXIST);
8842 CHECK(size, EINVAL);
8843 size = rte_align32pow2(size);
8845 /* Memory allocation. */
8846 m = calloc(1, sizeof(struct metarray));
8849 /* Node initialization. */
8850 strcpy(m->name, name);
8852 m->id = p->n_metarrays;
8854 /* Node add to tailq. */
8855 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8861 struct meter_profile meter_profile_default = {
8870 .cir_bytes_per_period = 1,
8872 .pir_bytes_per_period = 1,
8879 meter_init(struct meter *m)
8881 memset(m, 0, sizeof(struct meter));
8882 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8883 m->profile = &meter_profile_default;
8884 m->color_mask = RTE_COLOR_GREEN;
8886 meter_profile_default.n_users++;
8890 metarray_build(struct rte_swx_pipeline *p)
8894 if (!p->n_metarrays)
8897 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8898 CHECK(p->metarray_runtime, ENOMEM);
8900 TAILQ_FOREACH(m, &p->metarrays, node) {
8901 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8904 r->metarray = env_malloc(m->size * sizeof(struct meter),
8905 RTE_CACHE_LINE_SIZE,
8907 CHECK(r->metarray, ENOMEM);
8909 for (i = 0; i < m->size; i++)
8910 meter_init(&r->metarray[i]);
8912 r->size_mask = m->size - 1;
8919 metarray_build_free(struct rte_swx_pipeline *p)
8923 if (!p->metarray_runtime)
8926 for (i = 0; i < p->n_metarrays; i++) {
8927 struct metarray *m = metarray_find_by_id(p, i);
8928 struct metarray_runtime *r = &p->metarray_runtime[i];
8930 env_free(r->metarray, m->size * sizeof(struct meter));
8933 free(p->metarray_runtime);
8934 p->metarray_runtime = NULL;
8938 metarray_free(struct rte_swx_pipeline *p)
8940 metarray_build_free(p);
8944 struct metarray *elem;
8946 elem = TAILQ_FIRST(&p->metarrays);
8950 TAILQ_REMOVE(&p->metarrays, elem, node);
8954 /* Meter profiles. */
8956 struct meter_profile *elem;
8958 elem = TAILQ_FIRST(&p->meter_profiles);
8962 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8971 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8973 struct rte_swx_pipeline *pipeline;
8975 /* Check input parameters. */
8978 /* Memory allocation. */
8979 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8980 CHECK(pipeline, ENOMEM);
8982 /* Initialization. */
8983 TAILQ_INIT(&pipeline->struct_types);
8984 TAILQ_INIT(&pipeline->port_in_types);
8985 TAILQ_INIT(&pipeline->ports_in);
8986 TAILQ_INIT(&pipeline->port_out_types);
8987 TAILQ_INIT(&pipeline->ports_out);
8988 TAILQ_INIT(&pipeline->extern_types);
8989 TAILQ_INIT(&pipeline->extern_objs);
8990 TAILQ_INIT(&pipeline->extern_funcs);
8991 TAILQ_INIT(&pipeline->headers);
8992 TAILQ_INIT(&pipeline->actions);
8993 TAILQ_INIT(&pipeline->table_types);
8994 TAILQ_INIT(&pipeline->tables);
8995 TAILQ_INIT(&pipeline->selectors);
8996 TAILQ_INIT(&pipeline->learners);
8997 TAILQ_INIT(&pipeline->regarrays);
8998 TAILQ_INIT(&pipeline->meter_profiles);
8999 TAILQ_INIT(&pipeline->metarrays);
9001 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9002 pipeline->numa_node = numa_node;
9009 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9018 free(p->instruction_data);
9019 free(p->instructions);
9023 table_state_free(p);
9028 instruction_table_free(p);
9031 extern_func_free(p);
9044 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9045 const char **instructions,
9046 uint32_t n_instructions)
9051 err = instruction_config(p, NULL, instructions, n_instructions);
9055 /* Thread instruction pointer reset. */
9056 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9057 struct thread *t = &p->threads[i];
9059 thread_ip_reset(p, t);
9066 pipeline_compile(struct rte_swx_pipeline *p);
9069 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9074 CHECK(p->build_done == 0, EEXIST);
9076 status = port_in_build(p);
9080 status = port_out_build(p);
9084 status = struct_build(p);
9088 status = extern_obj_build(p);
9092 status = extern_func_build(p);
9096 status = header_build(p);
9100 status = metadata_build(p);
9104 status = instruction_table_build(p);
9108 status = action_build(p);
9112 status = table_build(p);
9116 status = selector_build(p);
9120 status = learner_build(p);
9124 status = table_state_build(p);
9128 status = regarray_build(p);
9132 status = metarray_build(p);
9138 pipeline_compile(p);
9143 metarray_build_free(p);
9144 regarray_build_free(p);
9145 table_state_build_free(p);
9146 learner_build_free(p);
9147 selector_build_free(p);
9148 table_build_free(p);
9149 action_build_free(p);
9150 instruction_table_build_free(p);
9151 metadata_build_free(p);
9152 header_build_free(p);
9153 extern_func_build_free(p);
9154 extern_obj_build_free(p);
9155 port_out_build_free(p);
9156 port_in_build_free(p);
9157 struct_build_free(p);
9163 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9167 for (i = 0; i < n_instructions; i++)
9172 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9176 for (i = 0; i < p->n_ports_out; i++) {
9177 struct port_out_runtime *port = &p->out[i];
9180 port->flush(port->obj);
9188 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9189 struct rte_swx_ctl_pipeline_info *pipeline)
9191 struct action *action;
9192 struct table *table;
9193 uint32_t n_actions = 0, n_tables = 0;
9195 if (!p || !pipeline)
9198 TAILQ_FOREACH(action, &p->actions, node)
9201 TAILQ_FOREACH(table, &p->tables, node)
9204 pipeline->n_ports_in = p->n_ports_in;
9205 pipeline->n_ports_out = p->n_ports_out;
9206 pipeline->n_actions = n_actions;
9207 pipeline->n_tables = n_tables;
9208 pipeline->n_selectors = p->n_selectors;
9209 pipeline->n_learners = p->n_learners;
9210 pipeline->n_regarrays = p->n_regarrays;
9211 pipeline->n_metarrays = p->n_metarrays;
9217 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9219 if (!p || !numa_node)
9222 *numa_node = p->numa_node;
9227 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9229 struct rte_swx_ctl_action_info *action)
9231 struct action *a = NULL;
9233 if (!p || (action_id >= p->n_actions) || !action)
9236 a = action_find_by_id(p, action_id);
9240 strcpy(action->name, a->name);
9241 action->n_args = a->st ? a->st->n_fields : 0;
9246 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9248 uint32_t action_arg_id,
9249 struct rte_swx_ctl_action_arg_info *action_arg)
9251 struct action *a = NULL;
9252 struct field *arg = NULL;
9254 if (!p || (action_id >= p->n_actions) || !action_arg)
9257 a = action_find_by_id(p, action_id);
9258 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9261 arg = &a->st->fields[action_arg_id];
9262 strcpy(action_arg->name, arg->name);
9263 action_arg->n_bits = arg->n_bits;
9264 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9270 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9272 struct rte_swx_ctl_table_info *table)
9274 struct table *t = NULL;
9279 t = table_find_by_id(p, table_id);
9283 strcpy(table->name, t->name);
9284 strcpy(table->args, t->args);
9285 table->n_match_fields = t->n_fields;
9286 table->n_actions = t->n_actions;
9287 table->default_action_is_const = t->default_action_is_const;
9288 table->size = t->size;
9293 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9295 uint32_t match_field_id,
9296 struct rte_swx_ctl_table_match_field_info *match_field)
9299 struct match_field *f;
9301 if (!p || (table_id >= p->n_tables) || !match_field)
9304 t = table_find_by_id(p, table_id);
9305 if (!t || (match_field_id >= t->n_fields))
9308 f = &t->fields[match_field_id];
9309 match_field->match_type = f->match_type;
9310 match_field->is_header = t->header ? 1 : 0;
9311 match_field->n_bits = f->field->n_bits;
9312 match_field->offset = f->field->offset;
9318 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9320 uint32_t table_action_id,
9321 struct rte_swx_ctl_table_action_info *table_action)
9325 if (!p || (table_id >= p->n_tables) || !table_action)
9328 t = table_find_by_id(p, table_id);
9329 if (!t || (table_action_id >= t->n_actions))
9332 table_action->action_id = t->actions[table_action_id]->id;
9334 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9335 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9341 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9343 struct rte_swx_table_ops *table_ops,
9348 if (!p || (table_id >= p->n_tables))
9351 t = table_find_by_id(p, table_id);
9357 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9367 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9368 uint32_t selector_id,
9369 struct rte_swx_ctl_selector_info *selector)
9371 struct selector *s = NULL;
9373 if (!p || !selector)
9376 s = selector_find_by_id(p, selector_id);
9380 strcpy(selector->name, s->name);
9382 selector->n_selector_fields = s->n_selector_fields;
9383 selector->n_groups_max = s->n_groups_max;
9384 selector->n_members_per_group_max = s->n_members_per_group_max;
9390 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9391 uint32_t selector_id,
9392 struct rte_swx_ctl_table_match_field_info *field)
9396 if (!p || (selector_id >= p->n_selectors) || !field)
9399 s = selector_find_by_id(p, selector_id);
9403 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9404 field->is_header = 0;
9405 field->n_bits = s->group_id_field->n_bits;
9406 field->offset = s->group_id_field->offset;
9412 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9413 uint32_t selector_id,
9414 uint32_t selector_field_id,
9415 struct rte_swx_ctl_table_match_field_info *field)
9420 if (!p || (selector_id >= p->n_selectors) || !field)
9423 s = selector_find_by_id(p, selector_id);
9424 if (!s || (selector_field_id >= s->n_selector_fields))
9427 f = s->selector_fields[selector_field_id];
9428 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9429 field->is_header = s->selector_header ? 1 : 0;
9430 field->n_bits = f->n_bits;
9431 field->offset = f->offset;
9437 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9438 uint32_t selector_id,
9439 struct rte_swx_ctl_table_match_field_info *field)
9443 if (!p || (selector_id >= p->n_selectors) || !field)
9446 s = selector_find_by_id(p, selector_id);
9450 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9451 field->is_header = 0;
9452 field->n_bits = s->member_id_field->n_bits;
9453 field->offset = s->member_id_field->offset;
9459 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9460 uint32_t learner_id,
9461 struct rte_swx_ctl_learner_info *learner)
9463 struct learner *l = NULL;
9468 l = learner_find_by_id(p, learner_id);
9472 strcpy(learner->name, l->name);
9474 learner->n_match_fields = l->n_fields;
9475 learner->n_actions = l->n_actions;
9476 learner->default_action_is_const = l->default_action_is_const;
9477 learner->size = l->size;
9483 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9484 uint32_t learner_id,
9485 uint32_t match_field_id,
9486 struct rte_swx_ctl_table_match_field_info *match_field)
9491 if (!p || (learner_id >= p->n_learners) || !match_field)
9494 l = learner_find_by_id(p, learner_id);
9495 if (!l || (match_field_id >= l->n_fields))
9498 f = l->fields[match_field_id];
9499 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9500 match_field->is_header = l->header ? 1 : 0;
9501 match_field->n_bits = f->n_bits;
9502 match_field->offset = f->offset;
9508 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9509 uint32_t learner_id,
9510 uint32_t learner_action_id,
9511 struct rte_swx_ctl_table_action_info *learner_action)
9515 if (!p || (learner_id >= p->n_learners) || !learner_action)
9518 l = learner_find_by_id(p, learner_id);
9519 if (!l || (learner_action_id >= l->n_actions))
9522 learner_action->action_id = l->actions[learner_action_id]->id;
9524 learner_action->action_is_for_table_entries =
9525 l->action_is_for_table_entries[learner_action_id];
9527 learner_action->action_is_for_default_entry =
9528 l->action_is_for_default_entry[learner_action_id];
9534 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9535 struct rte_swx_table_state **table_state)
9537 if (!p || !table_state || !p->build_done)
9540 *table_state = p->table_state;
9545 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9546 struct rte_swx_table_state *table_state)
9548 if (!p || !table_state || !p->build_done)
9551 p->table_state = table_state;
9556 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9558 struct rte_swx_port_in_stats *stats)
9560 struct port_in *port;
9565 port = port_in_find(p, port_id);
9569 port->type->ops.stats_read(port->obj, stats);
9574 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9576 struct rte_swx_port_out_stats *stats)
9578 struct port_out *port;
9583 port = port_out_find(p, port_id);
9587 port->type->ops.stats_read(port->obj, stats);
9592 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9593 const char *table_name,
9594 struct rte_swx_table_stats *stats)
9596 struct table *table;
9597 struct table_statistics *table_stats;
9599 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9602 table = table_find(p, table_name);
9606 table_stats = &p->table_stats[table->id];
9608 memcpy(stats->n_pkts_action,
9609 table_stats->n_pkts_action,
9610 p->n_actions * sizeof(uint64_t));
9612 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9613 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9619 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9620 const char *selector_name,
9621 struct rte_swx_pipeline_selector_stats *stats)
9625 if (!p || !selector_name || !selector_name[0] || !stats)
9628 s = selector_find(p, selector_name);
9632 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9638 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9639 const char *learner_name,
9640 struct rte_swx_learner_stats *stats)
9643 struct learner_statistics *learner_stats;
9645 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9648 l = learner_find(p, learner_name);
9652 learner_stats = &p->learner_stats[l->id];
9654 memcpy(stats->n_pkts_action,
9655 learner_stats->n_pkts_action,
9656 p->n_actions * sizeof(uint64_t));
9658 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9659 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9661 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9662 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9664 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9670 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9671 uint32_t regarray_id,
9672 struct rte_swx_ctl_regarray_info *regarray)
9676 if (!p || !regarray)
9679 r = regarray_find_by_id(p, regarray_id);
9683 strcpy(regarray->name, r->name);
9684 regarray->size = r->size;
9689 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9690 const char *regarray_name,
9691 uint32_t regarray_index,
9694 struct regarray *regarray;
9695 struct regarray_runtime *r;
9697 if (!p || !regarray_name || !value)
9700 regarray = regarray_find(p, regarray_name);
9701 if (!regarray || (regarray_index >= regarray->size))
9704 r = &p->regarray_runtime[regarray->id];
9705 *value = r->regarray[regarray_index];
9710 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9711 const char *regarray_name,
9712 uint32_t regarray_index,
9715 struct regarray *regarray;
9716 struct regarray_runtime *r;
9718 if (!p || !regarray_name)
9721 regarray = regarray_find(p, regarray_name);
9722 if (!regarray || (regarray_index >= regarray->size))
9725 r = &p->regarray_runtime[regarray->id];
9726 r->regarray[regarray_index] = value;
9731 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9732 uint32_t metarray_id,
9733 struct rte_swx_ctl_metarray_info *metarray)
9737 if (!p || !metarray)
9740 m = metarray_find_by_id(p, metarray_id);
9744 strcpy(metarray->name, m->name);
9745 metarray->size = m->size;
9750 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9752 struct rte_meter_trtcm_params *params)
9754 struct meter_profile *mp;
9758 CHECK_NAME(name, EINVAL);
9759 CHECK(params, EINVAL);
9760 CHECK(!meter_profile_find(p, name), EEXIST);
9762 /* Node allocation. */
9763 mp = calloc(1, sizeof(struct meter_profile));
9766 /* Node initialization. */
9767 strcpy(mp->name, name);
9768 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9769 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9775 /* Node add to tailq. */
9776 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9782 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9785 struct meter_profile *mp;
9788 CHECK_NAME(name, EINVAL);
9790 mp = meter_profile_find(p, name);
9792 CHECK(!mp->n_users, EBUSY);
9794 /* Remove node from tailq. */
9795 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9802 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9803 const char *metarray_name,
9804 uint32_t metarray_index)
9806 struct meter_profile *mp_old;
9807 struct metarray *metarray;
9808 struct metarray_runtime *metarray_runtime;
9812 CHECK_NAME(metarray_name, EINVAL);
9814 metarray = metarray_find(p, metarray_name);
9815 CHECK(metarray, EINVAL);
9816 CHECK(metarray_index < metarray->size, EINVAL);
9818 metarray_runtime = &p->metarray_runtime[metarray->id];
9819 m = &metarray_runtime->metarray[metarray_index];
9820 mp_old = m->profile;
9830 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9831 const char *metarray_name,
9832 uint32_t metarray_index,
9833 const char *profile_name)
9835 struct meter_profile *mp, *mp_old;
9836 struct metarray *metarray;
9837 struct metarray_runtime *metarray_runtime;
9841 CHECK_NAME(metarray_name, EINVAL);
9843 metarray = metarray_find(p, metarray_name);
9844 CHECK(metarray, EINVAL);
9845 CHECK(metarray_index < metarray->size, EINVAL);
9847 mp = meter_profile_find(p, profile_name);
9850 metarray_runtime = &p->metarray_runtime[metarray->id];
9851 m = &metarray_runtime->metarray[metarray_index];
9852 mp_old = m->profile;
9854 memset(m, 0, sizeof(struct meter));
9855 rte_meter_trtcm_config(&m->m, &mp->profile);
9857 m->color_mask = RTE_COLORS;
9866 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9867 const char *metarray_name,
9868 uint32_t metarray_index,
9869 struct rte_swx_ctl_meter_stats *stats)
9871 struct metarray *metarray;
9872 struct metarray_runtime *metarray_runtime;
9876 CHECK_NAME(metarray_name, EINVAL);
9878 metarray = metarray_find(p, metarray_name);
9879 CHECK(metarray, EINVAL);
9880 CHECK(metarray_index < metarray->size, EINVAL);
9882 CHECK(stats, EINVAL);
9884 metarray_runtime = &p->metarray_runtime[metarray->id];
9885 m = &metarray_runtime->metarray[metarray_index];
9887 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9888 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9894 * Pipeline compilation.
9897 instr_type_to_name(struct instruction *instr)
9899 switch (instr->type) {
9900 case INSTR_RX: return "INSTR_RX";
9902 case INSTR_TX: return "INSTR_TX";
9903 case INSTR_TX_I: return "INSTR_TX_I";
9905 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9906 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9907 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9908 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9909 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9910 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9911 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9912 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9914 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9916 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9918 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9919 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9920 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9921 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9922 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9923 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9924 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9925 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9926 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9928 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9929 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9931 case INSTR_MOV: return "INSTR_MOV";
9932 case INSTR_MOV_MH: return "INSTR_MOV_MH";
9933 case INSTR_MOV_HM: return "INSTR_MOV_HM";
9934 case INSTR_MOV_HH: return "INSTR_MOV_HH";
9935 case INSTR_MOV_I: return "INSTR_MOV_I";
9937 case INSTR_DMA_HT: return "INSTR_DMA_HT";
9938 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9939 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9940 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9941 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9942 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9943 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9944 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9946 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9947 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9948 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9949 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9950 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9951 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9953 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9954 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9955 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9956 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9957 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9958 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9960 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9961 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9962 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9963 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9965 case INSTR_ALU_AND: return "INSTR_ALU_AND";
9966 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9967 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9968 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9969 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9971 case INSTR_ALU_OR: return "INSTR_ALU_OR";
9972 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9973 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9974 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9975 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9977 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9978 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9979 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9980 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9981 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9983 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9984 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9985 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9986 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9987 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9988 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9990 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9991 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9992 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9993 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9994 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9995 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9997 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9998 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9999 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10001 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10002 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10003 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10004 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10005 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10006 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10008 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10009 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10010 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10011 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10012 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10013 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10014 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10015 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10016 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10018 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10019 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10020 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10021 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10022 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10023 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10024 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10025 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10026 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10028 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10029 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10030 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10032 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10033 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10034 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10035 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10036 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10037 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10038 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10039 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10040 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10041 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10042 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10043 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10045 case INSTR_TABLE: return "INSTR_TABLE";
10046 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10047 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10048 case INSTR_LEARNER: return "INSTR_LEARNER";
10049 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10051 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10052 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10054 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10055 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10057 case INSTR_JMP: return "INSTR_JMP";
10058 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10059 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10060 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10061 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10062 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10063 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10064 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10065 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10066 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10067 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10068 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10069 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10070 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10071 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10072 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10073 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10074 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10075 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10076 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10077 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10078 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10079 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10080 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10081 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10082 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10083 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10084 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10085 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10087 case INSTR_RETURN: return "INSTR_RETURN";
10089 default: return "INSTR_UNKNOWN";
10094 (*instruction_export_t)(struct instruction *, FILE *);
10097 instr_io_export(struct instruction *instr, FILE *f)
10099 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10101 /* n_io, n_io_imm, n_hdrs. */
10102 if (instr->type == INSTR_RX ||
10103 instr->type == INSTR_TX ||
10104 instr->type == INSTR_HDR_EXTRACT_M ||
10105 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10108 if (instr->type == INSTR_TX_I)
10111 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10112 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10114 if (instr->type == INSTR_HDR_EXTRACT_M ||
10115 instr->type == INSTR_HDR_LOOKAHEAD ||
10116 instr->type == INSTR_HDR_EMIT)
10119 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10120 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10125 "\t\t.type = %s,\n",
10126 instr_type_to_name(instr));
10136 "\t\t\t\t.offset = %u,\n"
10137 "\t\t\t\t.n_bits = %u,\n"
10139 instr->io.io.offset,
10140 instr->io.io.n_bits);
10145 "\t\t\t\t.val = %u,\n"
10149 /* instr.io.hdr. */
10154 /* instr.io.hdr.header_id. */
10156 "\t\t\t.header_id = {");
10158 for (i = 0; i < n_hdrs; i++)
10161 instr->io.hdr.header_id[i]);
10166 /* instr.io.hdr.struct_id. */
10168 "\t\t\t.struct_id = {");
10170 for (i = 0; i < n_hdrs; i++)
10173 instr->io.hdr.struct_id[i]);
10178 /* instr.io.hdr.n_bytes. */
10180 "\t\t\t.n_bytes = {");
10182 for (i = 0; i < n_hdrs; i++)
10185 instr->io.hdr.n_bytes[i]);
10190 /* instr.io.hdr - closing curly brace. */
10195 /* instr.io - closing curly brace. */
10199 /* instr - closing curly brace. */
10205 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10209 "\t\t.type = %s,\n"
10211 "\t\t\t.header_id = %u,\n"
10214 instr_type_to_name(instr),
10215 instr->valid.header_id);
10219 instr_mov_export(struct instruction *instr, FILE *f)
10221 if (instr->type != INSTR_MOV_I)
10224 "\t\t.type = %s,\n"
10227 "\t\t\t\t.struct_id = %u,\n"
10228 "\t\t\t\t.n_bits = %u,\n"
10229 "\t\t\t\t.offset = %u,\n"
10232 "\t\t\t\t.struct_id = %u,\n"
10233 "\t\t\t\t.n_bits = %u,\n"
10234 "\t\t\t\t.offset = %u,\n"
10238 instr_type_to_name(instr),
10239 instr->mov.dst.struct_id,
10240 instr->mov.dst.n_bits,
10241 instr->mov.dst.offset,
10242 instr->mov.src.struct_id,
10243 instr->mov.src.n_bits,
10244 instr->mov.src.offset);
10248 "\t\t.type = %s,\n"
10251 "\t\t\t\t.struct_id = %u,\n"
10252 "\t\t\t\t.n_bits = %u,\n"
10253 "\t\t\t\t.offset = %u,\n"
10255 "\t\t\t.src_val = %" PRIu64 ",\n"
10258 instr_type_to_name(instr),
10259 instr->mov.dst.struct_id,
10260 instr->mov.dst.n_bits,
10261 instr->mov.dst.offset,
10262 instr->mov.src_val);
10266 instr_dma_ht_export(struct instruction *instr, FILE *f)
10268 uint32_t n_dma = 0, i;
10271 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10276 "\t\t.type = %s,\n",
10277 instr_type_to_name(instr));
10283 /* instr.dma.dst. */
10285 "\t\t\t.dst = {\n");
10287 /* instr.dma.dst.header_id. */
10289 "\t\t\t\t.header_id = {");
10291 for (i = 0; i < n_dma; i++)
10294 instr->dma.dst.header_id[i]);
10299 /* instr.dma.dst.struct_id. */
10301 "\t\t\t\t.struct_id = {");
10303 for (i = 0; i < n_dma; i++)
10306 instr->dma.dst.struct_id[i]);
10311 /* instr.dma.dst - closing curly brace. */
10315 /* instr.dma.src. */
10317 "\t\t\t.src = {\n");
10319 /* instr.dma.src.offset. */
10321 "\t\t\t\t.offset = {");
10323 for (i = 0; i < n_dma; i++)
10326 instr->dma.src.offset[i]);
10331 /* instr.dma.src - closing curly brace. */
10335 /* instr.dma.n_bytes. */
10337 "\t\t\t.n_bytes = {");
10339 for (i = 0; i < n_dma; i++)
10342 instr->dma.n_bytes[i]);
10347 /* instr.dma - closing curly brace. */
10351 /* instr - closing curly brace. */
10357 instr_alu_export(struct instruction *instr, FILE *f)
10361 if (instr->type == INSTR_ALU_ADD_MI ||
10362 instr->type == INSTR_ALU_ADD_HI ||
10363 instr->type == INSTR_ALU_SUB_MI ||
10364 instr->type == INSTR_ALU_SUB_HI ||
10365 instr->type == INSTR_ALU_SHL_MI ||
10366 instr->type == INSTR_ALU_SHL_HI ||
10367 instr->type == INSTR_ALU_SHR_MI ||
10368 instr->type == INSTR_ALU_SHR_HI ||
10369 instr->type == INSTR_ALU_AND_I ||
10370 instr->type == INSTR_ALU_OR_I ||
10371 instr->type == INSTR_ALU_XOR_I)
10377 "\t\t.type = %s,\n"
10380 "\t\t\t\t.struct_id = %u,\n"
10381 "\t\t\t\t.n_bits = %u,\n"
10382 "\t\t\t\t.offset = %u,\n"
10385 "\t\t\t\t.struct_id = %u,\n"
10386 "\t\t\t\t.n_bits = %u,\n"
10387 "\t\t\t\t.offset = %u,\n"
10391 instr_type_to_name(instr),
10392 instr->alu.dst.struct_id,
10393 instr->alu.dst.n_bits,
10394 instr->alu.dst.offset,
10395 instr->alu.src.struct_id,
10396 instr->alu.src.n_bits,
10397 instr->alu.src.offset);
10401 "\t\t.type = %s,\n"
10404 "\t\t\t\t.struct_id = %u,\n"
10405 "\t\t\t\t.n_bits = %u,\n"
10406 "\t\t\t\t.offset = %u,\n"
10408 "\t\t\t.src_val = %" PRIu64 ",\n"
10411 instr_type_to_name(instr),
10412 instr->alu.dst.struct_id,
10413 instr->alu.dst.n_bits,
10414 instr->alu.dst.offset,
10415 instr->alu.src_val);
10419 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10421 int prefetch = 0, idx_imm = 0, src_imm = 0;
10423 if (instr->type == INSTR_REGPREFETCH_RH ||
10424 instr->type == INSTR_REGPREFETCH_RM ||
10425 instr->type == INSTR_REGPREFETCH_RI)
10428 /* index is the 3rd operand for the regrd instruction and the 2nd
10429 * operand for the regwr and regadd instructions.
10431 if (instr->type == INSTR_REGPREFETCH_RI ||
10432 instr->type == INSTR_REGRD_HRI ||
10433 instr->type == INSTR_REGRD_MRI ||
10434 instr->type == INSTR_REGWR_RIH ||
10435 instr->type == INSTR_REGWR_RIM ||
10436 instr->type == INSTR_REGWR_RII ||
10437 instr->type == INSTR_REGADD_RIH ||
10438 instr->type == INSTR_REGADD_RIM ||
10439 instr->type == INSTR_REGADD_RII)
10442 /* src is the 3rd operand for the regwr and regadd instructions. */
10443 if (instr->type == INSTR_REGWR_RHI ||
10444 instr->type == INSTR_REGWR_RMI ||
10445 instr->type == INSTR_REGWR_RII ||
10446 instr->type == INSTR_REGADD_RHI ||
10447 instr->type == INSTR_REGADD_RMI ||
10448 instr->type == INSTR_REGADD_RII)
10451 /* instr.regarray.regarray_id. */
10454 "\t\t.type = %s,\n"
10455 "\t\t.regarray = {\n"
10456 "\t\t\t.regarray_id = %u,\n",
10457 instr_type_to_name(instr),
10458 instr->regarray.regarray_id);
10460 /* instr.regarray.idx / instr.regarray.idx_val. */
10463 "\t\t\t\t.idx = {\n"
10464 "\t\t\t\t\t.struct_id = %u,\n"
10465 "\t\t\t\t\t.n_bits = %u,\n"
10466 "\t\t\t\t\t.offset = %u,\n"
10468 instr->regarray.idx.struct_id,
10469 instr->regarray.idx.n_bits,
10470 instr->regarray.idx.offset);
10473 "\t\t\t\t.idx_val = %u,\n",
10474 instr->regarray.idx_val);
10476 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10480 "\t\t\t\t.dstsrc = {\n"
10481 "\t\t\t\t\t.struct_id = %u,\n"
10482 "\t\t\t\t\t.n_bits = %u,\n"
10483 "\t\t\t\t\t.offset = %u,\n"
10485 instr->regarray.dstsrc.struct_id,
10486 instr->regarray.dstsrc.n_bits,
10487 instr->regarray.dstsrc.offset);
10490 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10491 instr->regarray.dstsrc_val);
10494 /* instr.regarray and instr - closing curly braces. */
10501 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10503 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10505 if (instr->type == INSTR_METPREFETCH_H ||
10506 instr->type == INSTR_METPREFETCH_M ||
10507 instr->type == INSTR_METPREFETCH_I)
10511 if (instr->type == INSTR_METPREFETCH_I ||
10512 instr->type == INSTR_METER_IHM ||
10513 instr->type == INSTR_METER_IHI ||
10514 instr->type == INSTR_METER_IMM ||
10515 instr->type == INSTR_METER_IMI)
10518 /* color_in_imm. */
10519 if (instr->type == INSTR_METER_HHI ||
10520 instr->type == INSTR_METER_HMI ||
10521 instr->type == INSTR_METER_MHI ||
10522 instr->type == INSTR_METER_MMI ||
10523 instr->type == INSTR_METER_IHI ||
10524 instr->type == INSTR_METER_IMI)
10527 /* instr.meter.metarray_id. */
10530 "\t\t.type = %s,\n"
10532 "\t\t\t.metarray_id = %u,\n",
10533 instr_type_to_name(instr),
10534 instr->meter.metarray_id);
10536 /* instr.meter.idx / instr.meter.idx_val. */
10540 "\t\t\t\t.struct_id = %u,\n"
10541 "\t\t\t\t.n_bits = %u,\n"
10542 "\t\t\t\t.offset = %u,\n"
10544 instr->meter.idx.struct_id,
10545 instr->meter.idx.n_bits,
10546 instr->meter.idx.offset);
10549 "\t\t\t.idx_val = %u,\n",
10550 instr->meter.idx_val);
10553 /* instr.meter.length. */
10555 "\t\t\t.length = {\n"
10556 "\t\t\t\t.struct_id = %u,\n"
10557 "\t\t\t\t.n_bits = %u,\n"
10558 "\t\t\t\t.offset = %u,\n"
10560 instr->meter.length.struct_id,
10561 instr->meter.length.n_bits,
10562 instr->meter.length.offset);
10564 /* instr.meter.color_in / instr.meter.color_in_val. */
10567 "\t\t\t.color_in = {\n"
10568 "\t\t\t\t.struct_id = %u,\n"
10569 "\t\t\t\t.n_bits = %u,\n"
10570 "\t\t\t\t.offset = %u,\n"
10572 instr->meter.color_in.struct_id,
10573 instr->meter.color_in.n_bits,
10574 instr->meter.color_in.offset);
10577 "\t\t\t.color_in_val = %u,\n",
10578 (uint32_t)instr->meter.color_in_val);
10580 /* instr.meter.color_out. */
10582 "\t\t\t.color_out = {\n"
10583 "\t\t\t\t.struct_id = %u,\n"
10584 "\t\t\t\t.n_bits = %u,\n"
10585 "\t\t\t\t.offset = %u,\n"
10587 instr->meter.color_out.struct_id,
10588 instr->meter.color_out.n_bits,
10589 instr->meter.color_out.offset);
10592 /* instr.meter and instr - closing curly braces. */
10599 instr_table_export(struct instruction *instr,
10604 "\t\t.type = %s,\n"
10606 "\t\t\t.table_id = %u,\n"
10609 instr_type_to_name(instr),
10610 instr->table.table_id);
10614 instr_learn_export(struct instruction *instr, FILE *f)
10618 "\t\t.type = %s,\n"
10620 "\t\t\t\t.action_id = %u,\n"
10623 instr_type_to_name(instr),
10624 instr->learn.action_id);
10628 instr_forget_export(struct instruction *instr, FILE *f)
10632 "\t\t.type = %s,\n"
10634 instr_type_to_name(instr));
10638 instr_extern_export(struct instruction *instr, FILE *f)
10640 if (instr->type == INSTR_EXTERN_OBJ)
10643 "\t\t.type = %s,\n"
10644 "\t\t.ext_obj = {\n"
10645 "\t\t\t.ext_obj_id = %u,\n"
10646 "\t\t\t.func_id = %u,\n"
10649 instr_type_to_name(instr),
10650 instr->ext_obj.ext_obj_id,
10651 instr->ext_obj.func_id);
10655 "\t\t.type = %s,\n"
10656 "\t\t.ext_func = {\n"
10657 "\t\t\t.ext_func_id = %u,\n"
10660 instr_type_to_name(instr),
10661 instr->ext_func.ext_func_id);
10665 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10669 "\t\t.type = %s,\n"
10671 "\t\t\t.ip = NULL,\n",
10672 instr_type_to_name(instr));
10674 switch (instr->type) {
10675 case INSTR_JMP_VALID:
10676 case INSTR_JMP_INVALID:
10678 "\t\t\t.header_id = %u,\n",
10679 instr->jmp.header_id);
10682 case INSTR_JMP_ACTION_HIT:
10683 case INSTR_JMP_ACTION_MISS:
10685 "\t\t\t.action_id = %u,\n",
10686 instr->jmp.action_id);
10690 case INSTR_JMP_EQ_MH:
10691 case INSTR_JMP_EQ_HM:
10692 case INSTR_JMP_EQ_HH:
10693 case INSTR_JMP_NEQ:
10694 case INSTR_JMP_NEQ_MH:
10695 case INSTR_JMP_NEQ_HM:
10696 case INSTR_JMP_NEQ_HH:
10698 case INSTR_JMP_LT_MH:
10699 case INSTR_JMP_LT_HM:
10700 case INSTR_JMP_LT_HH:
10702 case INSTR_JMP_GT_MH:
10703 case INSTR_JMP_GT_HM:
10704 case INSTR_JMP_GT_HH:
10707 "\t\t\t\t.struct_id = %u,\n"
10708 "\t\t\t\t.n_bits = %u,\n"
10709 "\t\t\t\t.offset = %u,\n"
10712 "\t\t\t\t.struct_id = %u,\n"
10713 "\t\t\t\t.n_bits = %u,\n"
10714 "\t\t\t\t.offset = %u,\n"
10716 instr->jmp.a.struct_id,
10717 instr->jmp.a.n_bits,
10718 instr->jmp.a.offset,
10719 instr->jmp.b.struct_id,
10720 instr->jmp.b.n_bits,
10721 instr->jmp.b.offset);
10724 case INSTR_JMP_EQ_I:
10725 case INSTR_JMP_NEQ_I:
10726 case INSTR_JMP_LT_MI:
10727 case INSTR_JMP_LT_HI:
10728 case INSTR_JMP_GT_MI:
10729 case INSTR_JMP_GT_HI:
10732 "\t\t\t\t.struct_id = %u,\n"
10733 "\t\t\t\t.n_bits = %u,\n"
10734 "\t\t\t\t.offset = %u,\n"
10736 "\t\t\t.b_val = %" PRIu64 ",\n",
10737 instr->jmp.a.struct_id,
10738 instr->jmp.a.n_bits,
10739 instr->jmp.a.offset,
10753 instr_return_export(struct instruction *instr,
10758 "\t\t.type = %s,\n",
10759 instr_type_to_name(instr));
10765 static instruction_export_t export_table[] = {
10766 [INSTR_RX] = instr_io_export,
10768 [INSTR_TX] = instr_io_export,
10769 [INSTR_TX_I] = instr_io_export,
10771 [INSTR_HDR_EXTRACT] = instr_io_export,
10772 [INSTR_HDR_EXTRACT2] = instr_io_export,
10773 [INSTR_HDR_EXTRACT3] = instr_io_export,
10774 [INSTR_HDR_EXTRACT4] = instr_io_export,
10775 [INSTR_HDR_EXTRACT5] = instr_io_export,
10776 [INSTR_HDR_EXTRACT6] = instr_io_export,
10777 [INSTR_HDR_EXTRACT7] = instr_io_export,
10778 [INSTR_HDR_EXTRACT8] = instr_io_export,
10780 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10782 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10784 [INSTR_HDR_EMIT] = instr_io_export,
10785 [INSTR_HDR_EMIT_TX] = instr_io_export,
10786 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10787 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10788 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10789 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10790 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10791 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10792 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10794 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10795 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10797 [INSTR_MOV] = instr_mov_export,
10798 [INSTR_MOV_MH] = instr_mov_export,
10799 [INSTR_MOV_HM] = instr_mov_export,
10800 [INSTR_MOV_HH] = instr_mov_export,
10801 [INSTR_MOV_I] = instr_mov_export,
10803 [INSTR_DMA_HT] = instr_dma_ht_export,
10804 [INSTR_DMA_HT2] = instr_dma_ht_export,
10805 [INSTR_DMA_HT3] = instr_dma_ht_export,
10806 [INSTR_DMA_HT4] = instr_dma_ht_export,
10807 [INSTR_DMA_HT5] = instr_dma_ht_export,
10808 [INSTR_DMA_HT6] = instr_dma_ht_export,
10809 [INSTR_DMA_HT7] = instr_dma_ht_export,
10810 [INSTR_DMA_HT8] = instr_dma_ht_export,
10812 [INSTR_ALU_ADD] = instr_alu_export,
10813 [INSTR_ALU_ADD_MH] = instr_alu_export,
10814 [INSTR_ALU_ADD_HM] = instr_alu_export,
10815 [INSTR_ALU_ADD_HH] = instr_alu_export,
10816 [INSTR_ALU_ADD_MI] = instr_alu_export,
10817 [INSTR_ALU_ADD_HI] = instr_alu_export,
10819 [INSTR_ALU_SUB] = instr_alu_export,
10820 [INSTR_ALU_SUB_MH] = instr_alu_export,
10821 [INSTR_ALU_SUB_HM] = instr_alu_export,
10822 [INSTR_ALU_SUB_HH] = instr_alu_export,
10823 [INSTR_ALU_SUB_MI] = instr_alu_export,
10824 [INSTR_ALU_SUB_HI] = instr_alu_export,
10826 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10827 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10828 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10829 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10831 [INSTR_ALU_AND] = instr_alu_export,
10832 [INSTR_ALU_AND_MH] = instr_alu_export,
10833 [INSTR_ALU_AND_HM] = instr_alu_export,
10834 [INSTR_ALU_AND_HH] = instr_alu_export,
10835 [INSTR_ALU_AND_I] = instr_alu_export,
10837 [INSTR_ALU_OR] = instr_alu_export,
10838 [INSTR_ALU_OR_MH] = instr_alu_export,
10839 [INSTR_ALU_OR_HM] = instr_alu_export,
10840 [INSTR_ALU_OR_HH] = instr_alu_export,
10841 [INSTR_ALU_OR_I] = instr_alu_export,
10843 [INSTR_ALU_XOR] = instr_alu_export,
10844 [INSTR_ALU_XOR_MH] = instr_alu_export,
10845 [INSTR_ALU_XOR_HM] = instr_alu_export,
10846 [INSTR_ALU_XOR_HH] = instr_alu_export,
10847 [INSTR_ALU_XOR_I] = instr_alu_export,
10849 [INSTR_ALU_SHL] = instr_alu_export,
10850 [INSTR_ALU_SHL_MH] = instr_alu_export,
10851 [INSTR_ALU_SHL_HM] = instr_alu_export,
10852 [INSTR_ALU_SHL_HH] = instr_alu_export,
10853 [INSTR_ALU_SHL_MI] = instr_alu_export,
10854 [INSTR_ALU_SHL_HI] = instr_alu_export,
10856 [INSTR_ALU_SHR] = instr_alu_export,
10857 [INSTR_ALU_SHR_MH] = instr_alu_export,
10858 [INSTR_ALU_SHR_HM] = instr_alu_export,
10859 [INSTR_ALU_SHR_HH] = instr_alu_export,
10860 [INSTR_ALU_SHR_MI] = instr_alu_export,
10861 [INSTR_ALU_SHR_HI] = instr_alu_export,
10863 [INSTR_REGPREFETCH_RH] = instr_reg_export,
10864 [INSTR_REGPREFETCH_RM] = instr_reg_export,
10865 [INSTR_REGPREFETCH_RI] = instr_reg_export,
10867 [INSTR_REGRD_HRH] = instr_reg_export,
10868 [INSTR_REGRD_HRM] = instr_reg_export,
10869 [INSTR_REGRD_MRH] = instr_reg_export,
10870 [INSTR_REGRD_MRM] = instr_reg_export,
10871 [INSTR_REGRD_HRI] = instr_reg_export,
10872 [INSTR_REGRD_MRI] = instr_reg_export,
10874 [INSTR_REGWR_RHH] = instr_reg_export,
10875 [INSTR_REGWR_RHM] = instr_reg_export,
10876 [INSTR_REGWR_RMH] = instr_reg_export,
10877 [INSTR_REGWR_RMM] = instr_reg_export,
10878 [INSTR_REGWR_RHI] = instr_reg_export,
10879 [INSTR_REGWR_RMI] = instr_reg_export,
10880 [INSTR_REGWR_RIH] = instr_reg_export,
10881 [INSTR_REGWR_RIM] = instr_reg_export,
10882 [INSTR_REGWR_RII] = instr_reg_export,
10884 [INSTR_REGADD_RHH] = instr_reg_export,
10885 [INSTR_REGADD_RHM] = instr_reg_export,
10886 [INSTR_REGADD_RMH] = instr_reg_export,
10887 [INSTR_REGADD_RMM] = instr_reg_export,
10888 [INSTR_REGADD_RHI] = instr_reg_export,
10889 [INSTR_REGADD_RMI] = instr_reg_export,
10890 [INSTR_REGADD_RIH] = instr_reg_export,
10891 [INSTR_REGADD_RIM] = instr_reg_export,
10892 [INSTR_REGADD_RII] = instr_reg_export,
10894 [INSTR_METPREFETCH_H] = instr_meter_export,
10895 [INSTR_METPREFETCH_M] = instr_meter_export,
10896 [INSTR_METPREFETCH_I] = instr_meter_export,
10898 [INSTR_METER_HHM] = instr_meter_export,
10899 [INSTR_METER_HHI] = instr_meter_export,
10900 [INSTR_METER_HMM] = instr_meter_export,
10901 [INSTR_METER_HMI] = instr_meter_export,
10902 [INSTR_METER_MHM] = instr_meter_export,
10903 [INSTR_METER_MHI] = instr_meter_export,
10904 [INSTR_METER_MMM] = instr_meter_export,
10905 [INSTR_METER_MMI] = instr_meter_export,
10906 [INSTR_METER_IHM] = instr_meter_export,
10907 [INSTR_METER_IHI] = instr_meter_export,
10908 [INSTR_METER_IMM] = instr_meter_export,
10909 [INSTR_METER_IMI] = instr_meter_export,
10911 [INSTR_TABLE] = instr_table_export,
10912 [INSTR_TABLE_AF] = instr_table_export,
10913 [INSTR_SELECTOR] = instr_table_export,
10914 [INSTR_LEARNER] = instr_table_export,
10915 [INSTR_LEARNER_AF] = instr_table_export,
10917 [INSTR_LEARNER_LEARN] = instr_learn_export,
10918 [INSTR_LEARNER_FORGET] = instr_forget_export,
10920 [INSTR_EXTERN_OBJ] = instr_extern_export,
10921 [INSTR_EXTERN_FUNC] = instr_extern_export,
10923 [INSTR_JMP] = instr_jmp_export,
10924 [INSTR_JMP_VALID] = instr_jmp_export,
10925 [INSTR_JMP_INVALID] = instr_jmp_export,
10926 [INSTR_JMP_HIT] = instr_jmp_export,
10927 [INSTR_JMP_MISS] = instr_jmp_export,
10928 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10929 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10931 [INSTR_JMP_EQ] = instr_jmp_export,
10932 [INSTR_JMP_EQ_MH] = instr_jmp_export,
10933 [INSTR_JMP_EQ_HM] = instr_jmp_export,
10934 [INSTR_JMP_EQ_HH] = instr_jmp_export,
10935 [INSTR_JMP_EQ_I] = instr_jmp_export,
10937 [INSTR_JMP_NEQ] = instr_jmp_export,
10938 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10939 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10940 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10941 [INSTR_JMP_NEQ_I] = instr_jmp_export,
10943 [INSTR_JMP_LT] = instr_jmp_export,
10944 [INSTR_JMP_LT_MH] = instr_jmp_export,
10945 [INSTR_JMP_LT_HM] = instr_jmp_export,
10946 [INSTR_JMP_LT_HH] = instr_jmp_export,
10947 [INSTR_JMP_LT_MI] = instr_jmp_export,
10948 [INSTR_JMP_LT_HI] = instr_jmp_export,
10950 [INSTR_JMP_GT] = instr_jmp_export,
10951 [INSTR_JMP_GT_MH] = instr_jmp_export,
10952 [INSTR_JMP_GT_HM] = instr_jmp_export,
10953 [INSTR_JMP_GT_HH] = instr_jmp_export,
10954 [INSTR_JMP_GT_MI] = instr_jmp_export,
10955 [INSTR_JMP_GT_HI] = instr_jmp_export,
10957 [INSTR_RETURN] = instr_return_export,
10961 action_data_codegen(struct action *a, FILE *f)
10966 "static const struct instruction action_%s_instructions[] = {\n",
10969 for (i = 0; i < a->n_instructions; i++) {
10970 struct instruction *instr = &a->instructions[i];
10971 instruction_export_t func = export_table[instr->type];
10976 fprintf(f, "};\n");
10979 static const char *
10980 instr_type_to_func(struct instruction *instr)
10982 switch (instr->type) {
10983 case INSTR_RX: return NULL;
10985 case INSTR_TX: return "__instr_tx_exec";
10986 case INSTR_TX_I: return "__instr_tx_i_exec";
10988 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10989 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10990 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10991 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10992 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10993 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10994 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10995 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10997 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10999 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11001 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11002 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11003 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11004 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11005 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11006 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11007 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11008 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11009 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11011 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11012 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11014 case INSTR_MOV: return "__instr_mov_exec";
11015 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11016 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11017 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11018 case INSTR_MOV_I: return "__instr_mov_i_exec";
11020 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11021 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11022 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11023 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11024 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11025 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11026 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11027 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11029 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11030 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11031 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11032 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11033 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11034 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11036 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11037 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11038 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11039 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11040 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11041 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11043 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11044 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11045 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11046 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11048 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11049 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11050 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11051 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11052 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11054 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11055 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11056 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11057 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11058 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11060 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11061 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11062 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11063 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11064 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11066 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11067 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11068 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11069 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11070 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11071 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11073 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11074 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11075 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11076 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11077 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11078 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11080 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11081 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11082 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11084 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11085 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11086 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11087 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11088 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11089 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11091 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11092 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11093 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11094 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11095 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11096 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11097 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11098 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11099 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11101 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11102 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11103 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11104 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11105 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11106 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11107 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11108 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11109 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11111 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11112 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11113 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11115 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11116 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11117 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11118 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11119 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11120 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11121 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11122 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11123 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11124 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11125 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11126 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11128 case INSTR_TABLE: return NULL;
11129 case INSTR_TABLE_AF: return NULL;
11130 case INSTR_SELECTOR: return NULL;
11131 case INSTR_LEARNER: return NULL;
11132 case INSTR_LEARNER_AF: return NULL;
11134 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11135 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11137 case INSTR_EXTERN_OBJ: return NULL;
11138 case INSTR_EXTERN_FUNC: return NULL;
11140 case INSTR_JMP: return NULL;
11141 case INSTR_JMP_VALID: return NULL;
11142 case INSTR_JMP_INVALID: return NULL;
11143 case INSTR_JMP_HIT: return NULL;
11144 case INSTR_JMP_MISS: return NULL;
11145 case INSTR_JMP_ACTION_HIT: return NULL;
11146 case INSTR_JMP_ACTION_MISS: return NULL;
11147 case INSTR_JMP_EQ: return NULL;
11148 case INSTR_JMP_EQ_MH: return NULL;
11149 case INSTR_JMP_EQ_HM: return NULL;
11150 case INSTR_JMP_EQ_HH: return NULL;
11151 case INSTR_JMP_EQ_I: return NULL;
11152 case INSTR_JMP_NEQ: return NULL;
11153 case INSTR_JMP_NEQ_MH: return NULL;
11154 case INSTR_JMP_NEQ_HM: return NULL;
11155 case INSTR_JMP_NEQ_HH: return NULL;
11156 case INSTR_JMP_NEQ_I: return NULL;
11157 case INSTR_JMP_LT: return NULL;
11158 case INSTR_JMP_LT_MH: return NULL;
11159 case INSTR_JMP_LT_HM: return NULL;
11160 case INSTR_JMP_LT_HH: return NULL;
11161 case INSTR_JMP_LT_MI: return NULL;
11162 case INSTR_JMP_LT_HI: return NULL;
11163 case INSTR_JMP_GT: return NULL;
11164 case INSTR_JMP_GT_MH: return NULL;
11165 case INSTR_JMP_GT_HM: return NULL;
11166 case INSTR_JMP_GT_HH: return NULL;
11167 case INSTR_JMP_GT_MI: return NULL;
11168 case INSTR_JMP_GT_HI: return NULL;
11170 case INSTR_RETURN: return NULL;
11172 default: return NULL;
11177 action_instr_does_tx_codegen(struct action *a,
11178 uint32_t instr_pos,
11179 struct instruction *instr,
11183 "%s(p, t, &action_%s_instructions[%u]);\n"
11184 "\tthread_ip_reset(p, t);\n"
11185 "\tinstr_rx_exec(p);\n"
11187 instr_type_to_func(instr),
11193 action_instr_extern_obj_codegen(struct action *a,
11194 uint32_t instr_pos,
11198 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11204 action_instr_extern_func_codegen(struct action *a,
11205 uint32_t instr_pos,
11209 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11215 action_instr_jmp_codegen(struct action *a,
11216 uint32_t instr_pos,
11217 struct instruction *instr,
11218 struct instruction_data *data,
11221 switch (instr->type) {
11228 case INSTR_JMP_VALID:
11230 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11237 case INSTR_JMP_INVALID:
11239 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11246 case INSTR_JMP_HIT:
11253 case INSTR_JMP_MISS:
11260 case INSTR_JMP_ACTION_HIT:
11262 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11269 case INSTR_JMP_ACTION_MISS:
11271 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11280 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11281 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11290 case INSTR_JMP_EQ_MH:
11292 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11293 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11302 case INSTR_JMP_EQ_HM:
11304 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11305 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11314 case INSTR_JMP_EQ_HH:
11316 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11317 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11326 case INSTR_JMP_EQ_I:
11328 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11329 "action_%s_instructions[%u].jmp.b_val)\n"
11338 case INSTR_JMP_NEQ:
11340 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11341 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11350 case INSTR_JMP_NEQ_MH:
11352 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11353 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11362 case INSTR_JMP_NEQ_HM:
11364 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11365 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11374 case INSTR_JMP_NEQ_HH:
11376 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11377 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11386 case INSTR_JMP_NEQ_I:
11388 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11389 "action_%s_instructions[%u].jmp.b_val)\n"
11400 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11401 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11410 case INSTR_JMP_LT_MH:
11412 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11413 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11422 case INSTR_JMP_LT_HM:
11424 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11425 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11434 case INSTR_JMP_LT_HH:
11436 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11437 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11446 case INSTR_JMP_LT_MI:
11448 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11449 "action_%s_instructions[%u].jmp.b_val)\n"
11458 case INSTR_JMP_LT_HI:
11460 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11461 "action_%s_instructions[%u].jmp.b_val)\n"
11472 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11473 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11482 case INSTR_JMP_GT_MH:
11484 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11485 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11494 case INSTR_JMP_GT_HM:
11496 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11497 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11506 case INSTR_JMP_GT_HH:
11508 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11509 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11518 case INSTR_JMP_GT_MI:
11520 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11521 "action_%s_instructions[%u].jmp.b_val)\n"
11530 case INSTR_JMP_GT_HI:
11532 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11533 "action_%s_instructions[%u].jmp.b_val)\n"
11548 action_instr_return_codegen(FILE *f)
11555 action_instr_codegen(struct action *a, FILE *f)
11561 "action_%s_run(struct rte_swx_pipeline *p)\n"
11563 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11567 for (i = 0; i < a->n_instructions; i++) {
11568 struct instruction *instr = &a->instructions[i];
11569 struct instruction_data *data = &a->instruction_data[i];
11571 /* Label, if present. */
11572 if (data->label[0])
11573 fprintf(f, "\n%s : ", data->label);
11575 fprintf(f, "\n\t");
11577 /* TX instruction type. */
11578 if (instruction_does_tx(instr)) {
11579 action_instr_does_tx_codegen(a, i, instr, f);
11583 /* Extern object/function instruction type. */
11584 if (instr->type == INSTR_EXTERN_OBJ) {
11585 action_instr_extern_obj_codegen(a, i, f);
11589 if (instr->type == INSTR_EXTERN_FUNC) {
11590 action_instr_extern_func_codegen(a, i, f);
11594 /* Jump instruction type. */
11595 if (instruction_is_jmp(instr)) {
11596 action_instr_jmp_codegen(a, i, instr, data, f);
11600 /* Return instruction type. */
11601 if (instr->type == INSTR_RETURN) {
11602 action_instr_return_codegen(f);
11606 /* Any other instruction type. */
11608 "%s(p, t, &action_%s_instructions[%u]);\n",
11609 instr_type_to_func(instr),
11614 fprintf(f, "}\n\n");
11617 struct instruction_group {
11618 TAILQ_ENTRY(instruction_group) node;
11622 uint32_t first_instr_id;
11624 uint32_t last_instr_id;
11629 TAILQ_HEAD(instruction_group_list, instruction_group);
11631 static struct instruction_group *
11632 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11634 struct instruction_group *g;
11636 TAILQ_FOREACH(g, igl, node)
11637 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11644 instruction_group_list_free(struct instruction_group_list *igl)
11650 struct instruction_group *g;
11652 g = TAILQ_FIRST(igl);
11656 TAILQ_REMOVE(igl, g, node);
11663 static struct instruction_group_list *
11664 instruction_group_list_create(struct rte_swx_pipeline *p)
11666 struct instruction_group_list *igl = NULL;
11667 struct instruction_group *g = NULL;
11668 uint32_t n_groups = 0, i;
11670 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11674 igl = calloc(1, sizeof(struct instruction_group_list));
11680 /* Allocate the first group. */
11681 g = calloc(1, sizeof(struct instruction_group));
11685 /* Iteration 1: Separate the instructions into groups based on the thread yield
11686 * instructions. Do not worry about the jump instructions at this point.
11688 for (i = 0; i < p->n_instructions; i++) {
11689 struct instruction *instr = &p->instructions[i];
11691 /* Check for thread yield instructions. */
11692 if (!instruction_does_thread_yield(instr))
11695 /* If the current group contains at least one instruction, then finalize it (with
11696 * the previous instruction), add it to the list and allocate a new group (that
11697 * starts with the current instruction).
11699 if (i - g->first_instr_id) {
11700 /* Finalize the group. */
11701 g->last_instr_id = i - 1;
11703 /* Add the group to the list. Advance the number of groups. */
11704 TAILQ_INSERT_TAIL(igl, g, node);
11707 /* Allocate a new group. */
11708 g = calloc(1, sizeof(struct instruction_group));
11712 /* Initialize the new group. */
11713 g->group_id = n_groups;
11714 g->first_instr_id = i;
11717 /* Finalize the current group (with the current instruction, therefore this group
11718 * contains just the current thread yield instruction), add it to the list and
11719 * allocate a new group (that starts with the next instruction).
11722 /* Finalize the group. */
11723 g->last_instr_id = i;
11725 /* Add the group to the list. Advance the number of groups. */
11726 TAILQ_INSERT_TAIL(igl, g, node);
11729 /* Allocate a new group. */
11730 g = calloc(1, sizeof(struct instruction_group));
11734 /* Initialize the new group. */
11735 g->group_id = n_groups;
11736 g->first_instr_id = i + 1;
11739 /* Handle the last group. */
11740 if (i - g->first_instr_id) {
11741 /* Finalize the group. */
11742 g->last_instr_id = i - 1;
11744 /* Add the group to the list. Advance the number of groups. */
11745 TAILQ_INSERT_TAIL(igl, g, node);
11752 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11753 * the destination of a jump instruction located in a different group ("far jump"), then the
11754 * current group has to be split, so that the instruction representing the far jump
11755 * destination is at the start of its group.
11758 int is_modified = 0;
11760 for (i = 0; i < p->n_instructions; i++) {
11761 struct instruction_data *data = &p->instruction_data[i];
11762 struct instruction_group *g;
11765 /* Continue when the current instruction is not a jump destination. */
11766 if (!data->n_users)
11769 g = instruction_group_list_group_find(igl, i);
11773 /* Find out all the jump instructions with this destination. */
11774 for (j = 0; j < p->n_instructions; j++) {
11775 struct instruction *jmp_instr = &p->instructions[j];
11776 struct instruction_data *jmp_data = &p->instruction_data[j];
11777 struct instruction_group *jmp_g, *new_g;
11779 /* Continue when not a jump instruction. Even when jump instruction,
11780 * continue when the jump destination is not this instruction.
11782 if (!instruction_is_jmp(jmp_instr) ||
11783 strcmp(jmp_data->jmp_label, data->label))
11786 jmp_g = instruction_group_list_group_find(igl, j);
11790 /* Continue when both the jump instruction and the jump destination
11791 * instruction are in the same group. Even when in different groups,
11792 * still continue if the jump destination instruction is already the
11793 * first instruction of its group.
11795 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11798 /* Split the group of the current jump destination instruction to
11799 * make this instruction the first instruction of a new group.
11801 new_g = calloc(1, sizeof(struct instruction_group));
11805 new_g->group_id = n_groups;
11806 new_g->first_instr_id = i;
11807 new_g->last_instr_id = g->last_instr_id;
11809 g->last_instr_id = i - 1;
11811 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11815 /* The decision to split this group (to make the current instruction
11816 * the first instruction of a new group) is already taken and fully
11817 * implemented, so no need to search for more reasons to do it.
11823 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11824 * previously considered local (i.e. the jump destination is in the same group as
11825 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11826 * different group than the jump instruction). Wost case scenario: each instruction
11827 * that is a jump destination ends up as the first instruction of its group.
11833 /* Re-assign the group IDs to be in incremental order. */
11835 TAILQ_FOREACH(g, igl, node) {
11844 instruction_group_list_free(igl);
11852 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11853 uint32_t instr_pos,
11854 struct instruction *instr,
11858 "%s(p, t, &pipeline_instructions[%u]);\n"
11859 "\tthread_ip_reset(p, t);\n"
11860 "\tinstr_rx_exec(p);\n"
11862 instr_type_to_func(instr),
11867 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11868 struct instruction_group_list *igl,
11869 uint32_t jmp_instr_id,
11870 struct instruction *jmp_instr,
11871 struct instruction_data *jmp_data,
11874 struct instruction_group *jmp_g, *g;
11875 struct instruction_data *data;
11878 switch (jmp_instr->type) {
11882 case INSTR_JMP_VALID:
11884 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11888 case INSTR_JMP_INVALID:
11890 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11894 case INSTR_JMP_HIT:
11899 case INSTR_JMP_MISS:
11904 case INSTR_JMP_ACTION_HIT:
11906 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11910 case INSTR_JMP_ACTION_MISS:
11912 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11918 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11919 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11924 case INSTR_JMP_EQ_MH:
11926 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11927 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11932 case INSTR_JMP_EQ_HM:
11934 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11935 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11940 case INSTR_JMP_EQ_HH:
11942 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11943 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11948 case INSTR_JMP_EQ_I:
11950 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11951 "pipeline_instructions[%u].jmp.b_val)",
11956 case INSTR_JMP_NEQ:
11958 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11959 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11964 case INSTR_JMP_NEQ_MH:
11966 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11967 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11972 case INSTR_JMP_NEQ_HM:
11974 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11975 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11980 case INSTR_JMP_NEQ_HH:
11982 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11983 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11988 case INSTR_JMP_NEQ_I:
11990 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11991 "pipeline_instructions[%u].jmp.b_val)",
11998 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11999 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12004 case INSTR_JMP_LT_MH:
12006 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12007 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12012 case INSTR_JMP_LT_HM:
12014 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12015 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12020 case INSTR_JMP_LT_HH:
12022 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12023 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12028 case INSTR_JMP_LT_MI:
12030 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12031 "pipeline_instructions[%u].jmp.b_val)",
12036 case INSTR_JMP_LT_HI:
12038 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12039 "pipeline_instructions[%u].jmp.b_val)",
12046 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12047 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12052 case INSTR_JMP_GT_MH:
12054 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12055 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12060 case INSTR_JMP_GT_HM:
12062 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12063 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12068 case INSTR_JMP_GT_HH:
12070 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12071 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12076 case INSTR_JMP_GT_MI:
12078 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12079 "pipeline_instructions[%u].jmp.b_val)",
12084 case INSTR_JMP_GT_HI:
12086 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12087 "pipeline_instructions[%u].jmp.b_val)",
12096 /* Find the instruction group of the jump instruction. */
12097 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12101 /* Find the instruction group of the jump destination instruction. */
12102 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12106 instr_id = data - p->instruction_data;
12108 g = instruction_group_list_group_find(igl, instr_id);
12112 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12113 * instruction group).
12115 if (g->group_id == jmp_g->group_id)
12117 "\n\t\tgoto %s;\n",
12118 jmp_data->jmp_label);
12122 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12131 instruction_group_list_codegen(struct instruction_group_list *igl,
12132 struct rte_swx_pipeline *p,
12135 struct instruction_group *g;
12137 int is_required = 0;
12139 /* Check if code generation is required. */
12140 TAILQ_FOREACH(g, igl, node)
12141 if (g->first_instr_id < g->last_instr_id)
12147 /* Generate the code for the pipeline instruction array. */
12149 "static const struct instruction pipeline_instructions[] = {\n");
12151 for (i = 0; i < p->n_instructions; i++) {
12152 struct instruction *instr = &p->instructions[i];
12153 instruction_export_t func = export_table[instr->type];
12158 fprintf(f, "};\n\n");
12160 /* Generate the code for the pipeline functions: one function for each instruction group
12161 * that contains more than one instruction.
12163 TAILQ_FOREACH(g, igl, node) {
12164 struct instruction *last_instr;
12167 /* Skip if group contains a single instruction. */
12168 if (g->last_instr_id == g->first_instr_id)
12171 /* Generate new pipeline function. */
12174 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12176 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12180 /* Generate the code for each pipeline instruction. */
12181 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12182 struct instruction *instr = &p->instructions[j];
12183 struct instruction_data *data = &p->instruction_data[j];
12185 /* Label, if present. */
12186 if (data->label[0])
12187 fprintf(f, "\n%s : ", data->label);
12189 fprintf(f, "\n\t");
12191 /* TX instruction type. */
12192 if (instruction_does_tx(instr)) {
12193 pipeline_instr_does_tx_codegen(p, j, instr, f);
12197 /* Jump instruction type. */
12198 if (instruction_is_jmp(instr)) {
12199 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12203 /* Any other instruction type. */
12205 "%s(p, t, &pipeline_instructions[%u]);\n",
12206 instr_type_to_func(instr),
12210 /* Finalize the generated pipeline function. For some instructions such as TX,
12211 * emit-many-and-TX and unconditional jump, the next instruction has been already
12212 * decided unconditionally and the instruction pointer of the current thread set
12213 * accordingly; for all the other instructions, the instruction pointer must be
12216 last_instr = &p->instructions[g->last_instr_id];
12218 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12220 "thread_ip_inc(p);\n");
12229 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12231 struct instruction_group *g;
12232 uint32_t n_custom_instr = 0;
12234 /* Groups with a single instruction: no function is generated for this group, the group
12235 * keeps its current instruction. Groups with more than two instructions: one function and
12236 * the associated custom instruction get generated for each such group.
12238 TAILQ_FOREACH(g, igl, node) {
12239 if (g->first_instr_id == g->last_instr_id)
12245 return n_custom_instr;
12249 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12254 /* Create the .c file. */
12255 f = fopen("/tmp/pipeline.c", "w");
12259 /* Include the .h file. */
12260 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12262 /* Add the code for each action. */
12263 TAILQ_FOREACH(a, &p->actions, node) {
12264 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12266 action_data_codegen(a, f);
12270 action_instr_codegen(a, f);
12275 /* Add the pipeline code. */
12276 instruction_group_list_codegen(igl, p, f);
12278 /* Close the .c file. */
12284 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12285 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12289 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12292 struct instruction_group *g;
12293 char *dir_in, *buffer = NULL;
12294 const char *dir_out;
12297 /* Get the environment variables. */
12298 dir_in = getenv("RTE_INSTALL_DIR");
12306 /* Memory allocation for the command buffer. */
12307 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12314 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12315 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12316 "-I %s/lib/pipeline "
12317 "-I %s/lib/eal/include "
12318 "-I %s/lib/eal/x86/include "
12319 "-I %s/lib/eal/include/generic "
12323 "-I %s/lib/pipeline "
12326 "-I %s/lib/eal/linux/include "
12327 ">%s/pipeline.log 2>&1 "
12329 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12330 ">>%s/pipeline.log 2>&1",
12349 /* Build the shared object library. */
12350 status = system(buffer);
12354 /* Open library. */
12356 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12357 "%s/libpipeline.so",
12360 p->lib = dlopen(buffer, RTLD_LAZY);
12366 /* Get the action function symbols. */
12367 TAILQ_FOREACH(a, &p->actions, node) {
12368 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12370 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12371 if (!p->action_funcs[a->id]) {
12377 /* Get the pipeline function symbols. */
12378 TAILQ_FOREACH(g, igl, node) {
12379 if (g->first_instr_id == g->last_instr_id)
12382 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12384 g->func = dlsym(p->lib, buffer);
12392 if (status && p->lib) {
12403 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12404 struct instruction_group_list *igl)
12406 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12408 /* Check that enough space is available within the pipeline instruction table to store all
12409 * the custom instructions.
12411 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12418 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12420 struct instruction_group *g;
12423 /* Pipeline table instructions. */
12424 for (i = 0; i < p->n_instructions; i++) {
12425 struct instruction *instr = &p->instructions[i];
12427 if (instr->type == INSTR_TABLE)
12428 instr->type = INSTR_TABLE_AF;
12430 if (instr->type == INSTR_LEARNER)
12431 instr->type = INSTR_LEARNER_AF;
12434 /* Pipeline custom instructions. */
12436 TAILQ_FOREACH(g, igl, node) {
12437 struct instruction *instr = &p->instructions[g->first_instr_id];
12440 if (g->first_instr_id == g->last_instr_id)
12443 /* Install a new custom instruction. */
12444 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12446 /* First instruction of the group: change its type to the new custom instruction. */
12447 instr->type = INSTR_CUSTOM_0 + i;
12449 /* All the subsequent instructions of the group: invalidate. */
12450 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12451 struct instruction_data *data = &p->instruction_data[j];
12459 /* Remove the invalidated instructions. */
12460 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12462 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12463 * instructions that are the only instruction within their group, so they were left
12466 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12470 pipeline_compile(struct rte_swx_pipeline *p)
12472 struct instruction_group_list *igl = NULL;
12475 igl = instruction_group_list_create(p);
12481 /* Code generation. */
12482 status = pipeline_codegen(p, igl);
12486 /* Build and load the shared object library. */
12487 status = pipeline_libload(p, igl);
12491 /* Adjust instructions. */
12492 status = pipeline_adjust_check(p, igl);
12496 pipeline_adjust(p, igl);
12499 instruction_group_list_free(igl);