1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include "rte_swx_pipeline_internal.h"
12 #define CHECK(condition, err_code) \
18 #define CHECK_NAME(name, err_code) \
21 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
24 #define CHECK_INSTRUCTION(instr, err_code) \
27 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
28 RTE_SWX_INSTRUCTION_SIZE), \
34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
36 #include <rte_malloc.h>
39 env_malloc(size_t size, size_t alignment, int numa_node)
41 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
45 env_free(void *start, size_t size __rte_unused)
55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
59 if (numa_available() == -1)
62 start = numa_alloc_onnode(size, numa_node);
66 memset(start, 0, size);
71 env_free(void *start, size_t size)
73 if (numa_available() == -1)
76 numa_free(start, size);
84 static struct struct_type *
85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
87 struct struct_type *elem;
89 TAILQ_FOREACH(elem, &p->struct_types, node)
90 if (strcmp(elem->name, name) == 0)
97 struct_type_field_find(struct struct_type *st, const char *name)
101 for (i = 0; i < st->n_fields; i++) {
102 struct field *f = &st->fields[i];
104 if (strcmp(f->name, name) == 0)
112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
114 struct rte_swx_field_params *fields,
116 int last_field_has_variable_size)
118 struct struct_type *st;
122 CHECK_NAME(name, EINVAL);
123 CHECK(fields, EINVAL);
124 CHECK(n_fields, EINVAL);
126 for (i = 0; i < n_fields; i++) {
127 struct rte_swx_field_params *f = &fields[i];
128 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
131 CHECK_NAME(f->name, EINVAL);
132 CHECK(f->n_bits, EINVAL);
133 CHECK((f->n_bits <= 64) || var_size, EINVAL);
134 CHECK((f->n_bits & 7) == 0, EINVAL);
136 for (j = 0; j < i; j++) {
137 struct rte_swx_field_params *f_prev = &fields[j];
139 CHECK(strcmp(f->name, f_prev->name), EINVAL);
143 CHECK(!struct_type_find(p, name), EEXIST);
145 /* Node allocation. */
146 st = calloc(1, sizeof(struct struct_type));
149 st->fields = calloc(n_fields, sizeof(struct field));
155 /* Node initialization. */
156 strcpy(st->name, name);
157 for (i = 0; i < n_fields; i++) {
158 struct field *dst = &st->fields[i];
159 struct rte_swx_field_params *src = &fields[i];
160 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
162 strcpy(dst->name, src->name);
163 dst->n_bits = src->n_bits;
164 dst->offset = st->n_bits;
165 dst->var_size = var_size;
167 st->n_bits += src->n_bits;
168 st->n_bits_min += var_size ? 0 : src->n_bits;
170 st->n_fields = n_fields;
171 st->var_size = last_field_has_variable_size;
173 /* Node add to tailq. */
174 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
180 struct_build(struct rte_swx_pipeline *p)
184 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
185 struct thread *t = &p->threads[i];
187 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
188 CHECK(t->structs, ENOMEM);
195 struct_build_free(struct rte_swx_pipeline *p)
199 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
200 struct thread *t = &p->threads[i];
208 struct_free(struct rte_swx_pipeline *p)
210 struct_build_free(p);
214 struct struct_type *elem;
216 elem = TAILQ_FIRST(&p->struct_types);
220 TAILQ_REMOVE(&p->struct_types, elem, node);
229 static struct port_in_type *
230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
232 struct port_in_type *elem;
237 TAILQ_FOREACH(elem, &p->port_in_types, node)
238 if (strcmp(elem->name, name) == 0)
245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
247 struct rte_swx_port_in_ops *ops)
249 struct port_in_type *elem;
252 CHECK_NAME(name, EINVAL);
254 CHECK(ops->create, EINVAL);
255 CHECK(ops->free, EINVAL);
256 CHECK(ops->pkt_rx, EINVAL);
257 CHECK(ops->stats_read, EINVAL);
259 CHECK(!port_in_type_find(p, name), EEXIST);
261 /* Node allocation. */
262 elem = calloc(1, sizeof(struct port_in_type));
265 /* Node initialization. */
266 strcpy(elem->name, name);
267 memcpy(&elem->ops, ops, sizeof(*ops));
269 /* Node add to tailq. */
270 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
275 static struct port_in *
276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
278 struct port_in *port;
280 TAILQ_FOREACH(port, &p->ports_in, node)
281 if (port->id == port_id)
288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
290 const char *port_type_name,
293 struct port_in_type *type = NULL;
294 struct port_in *port = NULL;
299 CHECK(!port_in_find(p, port_id), EINVAL);
301 CHECK_NAME(port_type_name, EINVAL);
302 type = port_in_type_find(p, port_type_name);
305 obj = type->ops.create(args);
308 /* Node allocation. */
309 port = calloc(1, sizeof(struct port_in));
312 /* Node initialization. */
317 /* Node add to tailq. */
318 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
319 if (p->n_ports_in < port_id + 1)
320 p->n_ports_in = port_id + 1;
326 port_in_build(struct rte_swx_pipeline *p)
328 struct port_in *port;
331 CHECK(p->n_ports_in, EINVAL);
332 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
334 for (i = 0; i < p->n_ports_in; i++)
335 CHECK(port_in_find(p, i), EINVAL);
337 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
338 CHECK(p->in, ENOMEM);
340 TAILQ_FOREACH(port, &p->ports_in, node) {
341 struct port_in_runtime *in = &p->in[port->id];
343 in->pkt_rx = port->type->ops.pkt_rx;
351 port_in_build_free(struct rte_swx_pipeline *p)
358 port_in_free(struct rte_swx_pipeline *p)
360 port_in_build_free(p);
364 struct port_in *port;
366 port = TAILQ_FIRST(&p->ports_in);
370 TAILQ_REMOVE(&p->ports_in, port, node);
371 port->type->ops.free(port->obj);
375 /* Input port types. */
377 struct port_in_type *elem;
379 elem = TAILQ_FIRST(&p->port_in_types);
383 TAILQ_REMOVE(&p->port_in_types, elem, node);
391 static struct port_out_type *
392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
394 struct port_out_type *elem;
399 TAILQ_FOREACH(elem, &p->port_out_types, node)
400 if (!strcmp(elem->name, name))
407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
409 struct rte_swx_port_out_ops *ops)
411 struct port_out_type *elem;
414 CHECK_NAME(name, EINVAL);
416 CHECK(ops->create, EINVAL);
417 CHECK(ops->free, EINVAL);
418 CHECK(ops->pkt_tx, EINVAL);
419 CHECK(ops->stats_read, EINVAL);
421 CHECK(!port_out_type_find(p, name), EEXIST);
423 /* Node allocation. */
424 elem = calloc(1, sizeof(struct port_out_type));
427 /* Node initialization. */
428 strcpy(elem->name, name);
429 memcpy(&elem->ops, ops, sizeof(*ops));
431 /* Node add to tailq. */
432 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
437 static struct port_out *
438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
440 struct port_out *port;
442 TAILQ_FOREACH(port, &p->ports_out, node)
443 if (port->id == port_id)
450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
452 const char *port_type_name,
455 struct port_out_type *type = NULL;
456 struct port_out *port = NULL;
461 CHECK(!port_out_find(p, port_id), EINVAL);
463 CHECK_NAME(port_type_name, EINVAL);
464 type = port_out_type_find(p, port_type_name);
467 obj = type->ops.create(args);
470 /* Node allocation. */
471 port = calloc(1, sizeof(struct port_out));
474 /* Node initialization. */
479 /* Node add to tailq. */
480 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
481 if (p->n_ports_out < port_id + 1)
482 p->n_ports_out = port_id + 1;
488 port_out_build(struct rte_swx_pipeline *p)
490 struct port_out *port;
493 CHECK(p->n_ports_out, EINVAL);
495 for (i = 0; i < p->n_ports_out; i++)
496 CHECK(port_out_find(p, i), EINVAL);
498 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
499 CHECK(p->out, ENOMEM);
501 TAILQ_FOREACH(port, &p->ports_out, node) {
502 struct port_out_runtime *out = &p->out[port->id];
504 out->pkt_tx = port->type->ops.pkt_tx;
505 out->flush = port->type->ops.flush;
506 out->obj = port->obj;
513 port_out_build_free(struct rte_swx_pipeline *p)
520 port_out_free(struct rte_swx_pipeline *p)
522 port_out_build_free(p);
526 struct port_out *port;
528 port = TAILQ_FIRST(&p->ports_out);
532 TAILQ_REMOVE(&p->ports_out, port, node);
533 port->type->ops.free(port->obj);
537 /* Output port types. */
539 struct port_out_type *elem;
541 elem = TAILQ_FIRST(&p->port_out_types);
545 TAILQ_REMOVE(&p->port_out_types, elem, node);
553 static struct extern_type *
554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
556 struct extern_type *elem;
558 TAILQ_FOREACH(elem, &p->extern_types, node)
559 if (strcmp(elem->name, name) == 0)
565 static struct extern_type_member_func *
566 extern_type_member_func_find(struct extern_type *type, const char *name)
568 struct extern_type_member_func *elem;
570 TAILQ_FOREACH(elem, &type->funcs, node)
571 if (strcmp(elem->name, name) == 0)
577 static struct extern_obj *
578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
580 struct extern_obj *elem;
582 TAILQ_FOREACH(elem, &p->extern_objs, node)
583 if (strcmp(elem->name, name) == 0)
589 static struct extern_type_member_func *
590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
592 struct extern_obj **obj)
594 struct extern_obj *object;
595 struct extern_type_member_func *func;
596 char *object_name, *func_name;
598 if (name[0] != 'e' || name[1] != '.')
601 object_name = strdup(&name[2]);
605 func_name = strchr(object_name, '.');
614 object = extern_obj_find(p, object_name);
620 func = extern_type_member_func_find(object->type, func_name);
633 static struct field *
634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
636 struct extern_obj **object)
638 struct extern_obj *obj;
640 char *obj_name, *field_name;
642 if ((name[0] != 'e') || (name[1] != '.'))
645 obj_name = strdup(&name[2]);
649 field_name = strchr(obj_name, '.');
658 obj = extern_obj_find(p, obj_name);
664 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
680 const char *mailbox_struct_type_name,
681 rte_swx_extern_type_constructor_t constructor,
682 rte_swx_extern_type_destructor_t destructor)
684 struct extern_type *elem;
685 struct struct_type *mailbox_struct_type;
689 CHECK_NAME(name, EINVAL);
690 CHECK(!extern_type_find(p, name), EEXIST);
692 CHECK_NAME(mailbox_struct_type_name, EINVAL);
693 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
694 CHECK(mailbox_struct_type, EINVAL);
695 CHECK(!mailbox_struct_type->var_size, EINVAL);
697 CHECK(constructor, EINVAL);
698 CHECK(destructor, EINVAL);
700 /* Node allocation. */
701 elem = calloc(1, sizeof(struct extern_type));
704 /* Node initialization. */
705 strcpy(elem->name, name);
706 elem->mailbox_struct_type = mailbox_struct_type;
707 elem->constructor = constructor;
708 elem->destructor = destructor;
709 TAILQ_INIT(&elem->funcs);
711 /* Node add to tailq. */
712 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
719 const char *extern_type_name,
721 rte_swx_extern_type_member_func_t member_func)
723 struct extern_type *type;
724 struct extern_type_member_func *type_member;
728 CHECK_NAME(extern_type_name, EINVAL);
729 type = extern_type_find(p, extern_type_name);
731 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
733 CHECK_NAME(name, EINVAL);
734 CHECK(!extern_type_member_func_find(type, name), EEXIST);
736 CHECK(member_func, EINVAL);
738 /* Node allocation. */
739 type_member = calloc(1, sizeof(struct extern_type_member_func));
740 CHECK(type_member, ENOMEM);
742 /* Node initialization. */
743 strcpy(type_member->name, name);
744 type_member->func = member_func;
745 type_member->id = type->n_funcs;
747 /* Node add to tailq. */
748 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
756 const char *extern_type_name,
760 struct extern_type *type;
761 struct extern_obj *obj;
766 CHECK_NAME(extern_type_name, EINVAL);
767 type = extern_type_find(p, extern_type_name);
770 CHECK_NAME(name, EINVAL);
771 CHECK(!extern_obj_find(p, name), EEXIST);
773 /* Node allocation. */
774 obj = calloc(1, sizeof(struct extern_obj));
777 /* Object construction. */
778 obj_handle = type->constructor(args);
784 /* Node initialization. */
785 strcpy(obj->name, name);
787 obj->obj = obj_handle;
788 obj->struct_id = p->n_structs;
789 obj->id = p->n_extern_objs;
791 /* Node add to tailq. */
792 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
800 extern_obj_build(struct rte_swx_pipeline *p)
804 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
805 struct thread *t = &p->threads[i];
806 struct extern_obj *obj;
808 t->extern_objs = calloc(p->n_extern_objs,
809 sizeof(struct extern_obj_runtime));
810 CHECK(t->extern_objs, ENOMEM);
812 TAILQ_FOREACH(obj, &p->extern_objs, node) {
813 struct extern_obj_runtime *r =
814 &t->extern_objs[obj->id];
815 struct extern_type_member_func *func;
816 uint32_t mailbox_size =
817 obj->type->mailbox_struct_type->n_bits / 8;
821 r->mailbox = calloc(1, mailbox_size);
822 CHECK(r->mailbox, ENOMEM);
824 TAILQ_FOREACH(func, &obj->type->funcs, node)
825 r->funcs[func->id] = func->func;
827 t->structs[obj->struct_id] = r->mailbox;
835 extern_obj_build_free(struct rte_swx_pipeline *p)
839 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
840 struct thread *t = &p->threads[i];
846 for (j = 0; j < p->n_extern_objs; j++) {
847 struct extern_obj_runtime *r = &t->extern_objs[j];
852 free(t->extern_objs);
853 t->extern_objs = NULL;
858 extern_obj_free(struct rte_swx_pipeline *p)
860 extern_obj_build_free(p);
862 /* Extern objects. */
864 struct extern_obj *elem;
866 elem = TAILQ_FIRST(&p->extern_objs);
870 TAILQ_REMOVE(&p->extern_objs, elem, node);
872 elem->type->destructor(elem->obj);
878 struct extern_type *elem;
880 elem = TAILQ_FIRST(&p->extern_types);
884 TAILQ_REMOVE(&p->extern_types, elem, node);
887 struct extern_type_member_func *func;
889 func = TAILQ_FIRST(&elem->funcs);
893 TAILQ_REMOVE(&elem->funcs, func, node);
904 static struct extern_func *
905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
907 struct extern_func *elem;
909 TAILQ_FOREACH(elem, &p->extern_funcs, node)
910 if (strcmp(elem->name, name) == 0)
916 static struct extern_func *
917 extern_func_parse(struct rte_swx_pipeline *p,
920 if (name[0] != 'f' || name[1] != '.')
923 return extern_func_find(p, &name[2]);
926 static struct field *
927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
929 struct extern_func **function)
931 struct extern_func *func;
933 char *func_name, *field_name;
935 if ((name[0] != 'f') || (name[1] != '.'))
938 func_name = strdup(&name[2]);
942 field_name = strchr(func_name, '.');
951 func = extern_func_find(p, func_name);
957 f = struct_type_field_find(func->mailbox_struct_type, field_name);
971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
973 const char *mailbox_struct_type_name,
974 rte_swx_extern_func_t func)
976 struct extern_func *f;
977 struct struct_type *mailbox_struct_type;
981 CHECK_NAME(name, EINVAL);
982 CHECK(!extern_func_find(p, name), EEXIST);
984 CHECK_NAME(mailbox_struct_type_name, EINVAL);
985 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
986 CHECK(mailbox_struct_type, EINVAL);
987 CHECK(!mailbox_struct_type->var_size, EINVAL);
991 /* Node allocation. */
992 f = calloc(1, sizeof(struct extern_func));
995 /* Node initialization. */
996 strcpy(f->name, name);
997 f->mailbox_struct_type = mailbox_struct_type;
999 f->struct_id = p->n_structs;
1000 f->id = p->n_extern_funcs;
1002 /* Node add to tailq. */
1003 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004 p->n_extern_funcs++;
1011 extern_func_build(struct rte_swx_pipeline *p)
1015 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016 struct thread *t = &p->threads[i];
1017 struct extern_func *func;
1019 /* Memory allocation. */
1020 t->extern_funcs = calloc(p->n_extern_funcs,
1021 sizeof(struct extern_func_runtime));
1022 CHECK(t->extern_funcs, ENOMEM);
1024 /* Extern function. */
1025 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026 struct extern_func_runtime *r =
1027 &t->extern_funcs[func->id];
1028 uint32_t mailbox_size =
1029 func->mailbox_struct_type->n_bits / 8;
1031 r->func = func->func;
1033 r->mailbox = calloc(1, mailbox_size);
1034 CHECK(r->mailbox, ENOMEM);
1036 t->structs[func->struct_id] = r->mailbox;
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1048 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049 struct thread *t = &p->threads[i];
1052 if (!t->extern_funcs)
1055 for (j = 0; j < p->n_extern_funcs; j++) {
1056 struct extern_func_runtime *r = &t->extern_funcs[j];
1061 free(t->extern_funcs);
1062 t->extern_funcs = NULL;
1067 extern_func_free(struct rte_swx_pipeline *p)
1069 extern_func_build_free(p);
1072 struct extern_func *elem;
1074 elem = TAILQ_FIRST(&p->extern_funcs);
1078 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1089 struct header *elem;
1091 TAILQ_FOREACH(elem, &p->headers, node)
1092 if (strcmp(elem->name, name) == 0)
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1101 struct header *elem;
1103 TAILQ_FOREACH(elem, &p->headers, node)
1104 if (elem->struct_id == struct_id)
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1114 if (name[0] != 'h' || name[1] != '.')
1117 return header_find(p, &name[2]);
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1123 struct header **header)
1127 char *header_name, *field_name;
1129 if ((name[0] != 'h') || (name[1] != '.'))
1132 header_name = strdup(&name[2]);
1136 field_name = strchr(header_name, '.');
1145 h = header_find(p, header_name);
1151 f = struct_type_field_find(h->st, field_name);
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1167 const char *struct_type_name)
1169 struct struct_type *st;
1171 size_t n_headers_max;
1174 CHECK_NAME(name, EINVAL);
1175 CHECK_NAME(struct_type_name, EINVAL);
1177 CHECK(!header_find(p, name), EEXIST);
1179 st = struct_type_find(p, struct_type_name);
1182 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183 CHECK(p->n_headers < n_headers_max, ENOSPC);
1185 /* Node allocation. */
1186 h = calloc(1, sizeof(struct header));
1189 /* Node initialization. */
1190 strcpy(h->name, name);
1192 h->struct_id = p->n_structs;
1193 h->id = p->n_headers;
1195 /* Node add to tailq. */
1196 TAILQ_INSERT_TAIL(&p->headers, h, node);
1204 header_build(struct rte_swx_pipeline *p)
1207 uint32_t n_bytes = 0, i;
1209 TAILQ_FOREACH(h, &p->headers, node) {
1210 n_bytes += h->st->n_bits / 8;
1213 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214 struct thread *t = &p->threads[i];
1215 uint32_t offset = 0;
1217 t->headers = calloc(p->n_headers,
1218 sizeof(struct header_runtime));
1219 CHECK(t->headers, ENOMEM);
1221 t->headers_out = calloc(p->n_headers,
1222 sizeof(struct header_out_runtime));
1223 CHECK(t->headers_out, ENOMEM);
1225 t->header_storage = calloc(1, n_bytes);
1226 CHECK(t->header_storage, ENOMEM);
1228 t->header_out_storage = calloc(1, n_bytes);
1229 CHECK(t->header_out_storage, ENOMEM);
1231 TAILQ_FOREACH(h, &p->headers, node) {
1232 uint8_t *header_storage;
1233 uint32_t n_bytes = h->st->n_bits / 8;
1235 header_storage = &t->header_storage[offset];
1238 t->headers[h->id].ptr0 = header_storage;
1239 t->headers[h->id].n_bytes = n_bytes;
1241 t->structs[h->struct_id] = header_storage;
1249 header_build_free(struct rte_swx_pipeline *p)
1253 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254 struct thread *t = &p->threads[i];
1256 free(t->headers_out);
1257 t->headers_out = NULL;
1262 free(t->header_out_storage);
1263 t->header_out_storage = NULL;
1265 free(t->header_storage);
1266 t->header_storage = NULL;
1271 header_free(struct rte_swx_pipeline *p)
1273 header_build_free(p);
1276 struct header *elem;
1278 elem = TAILQ_FIRST(&p->headers);
1282 TAILQ_REMOVE(&p->headers, elem, node);
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1293 if (!p->metadata_st)
1296 if (name[0] != 'm' || name[1] != '.')
1299 return struct_type_field_find(p->metadata_st, &name[2]);
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304 const char *struct_type_name)
1306 struct struct_type *st = NULL;
1310 CHECK_NAME(struct_type_name, EINVAL);
1311 st = struct_type_find(p, struct_type_name);
1313 CHECK(!st->var_size, EINVAL);
1314 CHECK(!p->metadata_st, EINVAL);
1316 p->metadata_st = st;
1317 p->metadata_struct_id = p->n_structs;
1325 metadata_build(struct rte_swx_pipeline *p)
1327 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1330 /* Thread-level initialization. */
1331 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332 struct thread *t = &p->threads[i];
1335 metadata = calloc(1, n_bytes);
1336 CHECK(metadata, ENOMEM);
1338 t->metadata = metadata;
1339 t->structs[p->metadata_struct_id] = metadata;
1346 metadata_build_free(struct rte_swx_pipeline *p)
1350 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351 struct thread *t = &p->threads[i];
1359 metadata_free(struct rte_swx_pipeline *p)
1361 metadata_build_free(p);
1368 instruction_is_tx(enum instruction_type type)
1381 instruction_does_tx(struct instruction *instr)
1383 switch (instr->type) {
1386 case INSTR_HDR_EMIT_TX:
1387 case INSTR_HDR_EMIT2_TX:
1388 case INSTR_HDR_EMIT3_TX:
1389 case INSTR_HDR_EMIT4_TX:
1390 case INSTR_HDR_EMIT5_TX:
1391 case INSTR_HDR_EMIT6_TX:
1392 case INSTR_HDR_EMIT7_TX:
1393 case INSTR_HDR_EMIT8_TX:
1401 instruction_is_jmp(struct instruction *instr)
1403 switch (instr->type) {
1405 case INSTR_JMP_VALID:
1406 case INSTR_JMP_INVALID:
1408 case INSTR_JMP_MISS:
1409 case INSTR_JMP_ACTION_HIT:
1410 case INSTR_JMP_ACTION_MISS:
1412 case INSTR_JMP_EQ_MH:
1413 case INSTR_JMP_EQ_HM:
1414 case INSTR_JMP_EQ_HH:
1415 case INSTR_JMP_EQ_I:
1417 case INSTR_JMP_NEQ_MH:
1418 case INSTR_JMP_NEQ_HM:
1419 case INSTR_JMP_NEQ_HH:
1420 case INSTR_JMP_NEQ_I:
1422 case INSTR_JMP_LT_MH:
1423 case INSTR_JMP_LT_HM:
1424 case INSTR_JMP_LT_HH:
1425 case INSTR_JMP_LT_MI:
1426 case INSTR_JMP_LT_HI:
1428 case INSTR_JMP_GT_MH:
1429 case INSTR_JMP_GT_HM:
1430 case INSTR_JMP_GT_HH:
1431 case INSTR_JMP_GT_MI:
1432 case INSTR_JMP_GT_HI:
1441 instruction_does_thread_yield(struct instruction *instr)
1443 switch (instr->type) {
1446 case INSTR_TABLE_AF:
1447 case INSTR_SELECTOR:
1449 case INSTR_LEARNER_AF:
1450 case INSTR_EXTERN_OBJ:
1451 case INSTR_EXTERN_FUNC:
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463 struct action *action,
1465 uint32_t *struct_id)
1472 struct header *header;
1474 f = header_field_parse(p, name, &header);
1478 *struct_id = header->struct_id;
1484 f = metadata_field_parse(p, name);
1488 *struct_id = p->metadata_struct_id;
1497 f = action_field_parse(action, name);
1507 struct extern_obj *obj;
1509 f = extern_obj_mailbox_field_parse(p, name, &obj);
1513 *struct_id = obj->struct_id;
1519 struct extern_func *func;
1521 f = extern_func_mailbox_field_parse(p, name, &func);
1525 *struct_id = func->struct_id;
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539 struct action *action,
1542 struct instruction *instr,
1543 struct instruction_data *data __rte_unused)
1547 CHECK(!action, EINVAL);
1548 CHECK(n_tokens == 2, EINVAL);
1550 f = metadata_field_parse(p, tokens[1]);
1553 instr->type = INSTR_RX;
1554 instr->io.io.offset = f->offset / 8;
1555 instr->io.io.n_bits = f->n_bits;
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564 struct action *action __rte_unused,
1567 struct instruction *instr,
1568 struct instruction_data *data __rte_unused)
1570 char *port = tokens[1];
1574 CHECK(n_tokens == 2, EINVAL);
1576 f = metadata_field_parse(p, port);
1578 instr->type = INSTR_TX;
1579 instr->io.io.offset = f->offset / 8;
1580 instr->io.io.n_bits = f->n_bits;
1585 port_val = strtoul(port, &port, 0);
1586 CHECK(!port[0], EINVAL);
1588 instr->type = INSTR_TX_I;
1589 instr->io.io.val = port_val;
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595 struct action *action __rte_unused,
1596 char **tokens __rte_unused,
1598 struct instruction *instr,
1599 struct instruction_data *data __rte_unused)
1601 CHECK(n_tokens == 1, EINVAL);
1604 instr->type = INSTR_TX_I;
1605 instr->io.io.val = p->n_ports_out - 1;
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1612 struct thread *t = &p->threads[p->thread_id];
1613 struct instruction *ip = t->ip;
1615 __instr_tx_exec(p, t, ip);
1618 thread_ip_reset(p, t);
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1625 struct thread *t = &p->threads[p->thread_id];
1626 struct instruction *ip = t->ip;
1628 __instr_tx_i_exec(p, t, ip);
1631 thread_ip_reset(p, t);
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640 struct action *action,
1643 struct instruction *instr,
1644 struct instruction_data *data __rte_unused)
1648 CHECK(!action, EINVAL);
1649 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1651 h = header_parse(p, tokens[1]);
1654 if (n_tokens == 2) {
1655 CHECK(!h->st->var_size, EINVAL);
1657 instr->type = INSTR_HDR_EXTRACT;
1658 instr->io.hdr.header_id[0] = h->id;
1659 instr->io.hdr.struct_id[0] = h->struct_id;
1660 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1664 CHECK(h->st->var_size, EINVAL);
1666 mf = metadata_field_parse(p, tokens[2]);
1668 CHECK(!mf->var_size, EINVAL);
1670 instr->type = INSTR_HDR_EXTRACT_M;
1671 instr->io.io.offset = mf->offset / 8;
1672 instr->io.io.n_bits = mf->n_bits;
1673 instr->io.hdr.header_id[0] = h->id;
1674 instr->io.hdr.struct_id[0] = h->struct_id;
1675 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683 struct action *action,
1686 struct instruction *instr,
1687 struct instruction_data *data __rte_unused)
1691 CHECK(!action, EINVAL);
1692 CHECK(n_tokens == 2, EINVAL);
1694 h = header_parse(p, tokens[1]);
1696 CHECK(!h->st->var_size, EINVAL);
1698 instr->type = INSTR_HDR_LOOKAHEAD;
1699 instr->io.hdr.header_id[0] = h->id;
1700 instr->io.hdr.struct_id[0] = h->struct_id;
1701 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1709 struct thread *t = &p->threads[p->thread_id];
1710 struct instruction *ip = t->ip;
1712 __instr_hdr_extract_exec(p, t, ip);
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1721 struct thread *t = &p->threads[p->thread_id];
1722 struct instruction *ip = t->ip;
1724 __instr_hdr_extract2_exec(p, t, ip);
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1733 struct thread *t = &p->threads[p->thread_id];
1734 struct instruction *ip = t->ip;
1736 __instr_hdr_extract3_exec(p, t, ip);
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1745 struct thread *t = &p->threads[p->thread_id];
1746 struct instruction *ip = t->ip;
1748 __instr_hdr_extract4_exec(p, t, ip);
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1757 struct thread *t = &p->threads[p->thread_id];
1758 struct instruction *ip = t->ip;
1760 __instr_hdr_extract5_exec(p, t, ip);
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1769 struct thread *t = &p->threads[p->thread_id];
1770 struct instruction *ip = t->ip;
1772 __instr_hdr_extract6_exec(p, t, ip);
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1781 struct thread *t = &p->threads[p->thread_id];
1782 struct instruction *ip = t->ip;
1784 __instr_hdr_extract7_exec(p, t, ip);
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1793 struct thread *t = &p->threads[p->thread_id];
1794 struct instruction *ip = t->ip;
1796 __instr_hdr_extract8_exec(p, t, ip);
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1805 struct thread *t = &p->threads[p->thread_id];
1806 struct instruction *ip = t->ip;
1808 __instr_hdr_extract_m_exec(p, t, ip);
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1817 struct thread *t = &p->threads[p->thread_id];
1818 struct instruction *ip = t->ip;
1820 __instr_hdr_lookahead_exec(p, t, ip);
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831 struct action *action __rte_unused,
1834 struct instruction *instr,
1835 struct instruction_data *data __rte_unused)
1839 CHECK(n_tokens == 2, EINVAL);
1841 h = header_parse(p, tokens[1]);
1844 instr->type = INSTR_HDR_EMIT;
1845 instr->io.hdr.header_id[0] = h->id;
1846 instr->io.hdr.struct_id[0] = h->struct_id;
1847 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1854 struct thread *t = &p->threads[p->thread_id];
1855 struct instruction *ip = t->ip;
1857 __instr_hdr_emit_exec(p, t, ip);
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1866 struct thread *t = &p->threads[p->thread_id];
1867 struct instruction *ip = t->ip;
1869 __instr_hdr_emit_tx_exec(p, t, ip);
1872 thread_ip_reset(p, t);
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1879 struct thread *t = &p->threads[p->thread_id];
1880 struct instruction *ip = t->ip;
1882 __instr_hdr_emit2_tx_exec(p, t, ip);
1885 thread_ip_reset(p, t);
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1892 struct thread *t = &p->threads[p->thread_id];
1893 struct instruction *ip = t->ip;
1895 __instr_hdr_emit3_tx_exec(p, t, ip);
1898 thread_ip_reset(p, t);
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1905 struct thread *t = &p->threads[p->thread_id];
1906 struct instruction *ip = t->ip;
1908 __instr_hdr_emit4_tx_exec(p, t, ip);
1911 thread_ip_reset(p, t);
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1918 struct thread *t = &p->threads[p->thread_id];
1919 struct instruction *ip = t->ip;
1921 __instr_hdr_emit5_tx_exec(p, t, ip);
1924 thread_ip_reset(p, t);
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1931 struct thread *t = &p->threads[p->thread_id];
1932 struct instruction *ip = t->ip;
1934 __instr_hdr_emit6_tx_exec(p, t, ip);
1937 thread_ip_reset(p, t);
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1944 struct thread *t = &p->threads[p->thread_id];
1945 struct instruction *ip = t->ip;
1947 __instr_hdr_emit7_tx_exec(p, t, ip);
1950 thread_ip_reset(p, t);
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1957 struct thread *t = &p->threads[p->thread_id];
1958 struct instruction *ip = t->ip;
1960 __instr_hdr_emit8_tx_exec(p, t, ip);
1963 thread_ip_reset(p, t);
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972 struct action *action __rte_unused,
1975 struct instruction *instr,
1976 struct instruction_data *data __rte_unused)
1980 CHECK(n_tokens == 2, EINVAL);
1982 h = header_parse(p, tokens[1]);
1985 instr->type = INSTR_HDR_VALIDATE;
1986 instr->valid.header_id = h->id;
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1993 struct thread *t = &p->threads[p->thread_id];
1994 struct instruction *ip = t->ip;
1996 __instr_hdr_validate_exec(p, t, ip);
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007 struct action *action __rte_unused,
2010 struct instruction *instr,
2011 struct instruction_data *data __rte_unused)
2015 CHECK(n_tokens == 2, EINVAL);
2017 h = header_parse(p, tokens[1]);
2020 instr->type = INSTR_HDR_INVALIDATE;
2021 instr->valid.header_id = h->id;
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2028 struct thread *t = &p->threads[p->thread_id];
2029 struct instruction *ip = t->ip;
2031 __instr_hdr_invalidate_exec(p, t, ip);
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051 struct action *action,
2054 struct instruction *instr,
2055 struct instruction_data *data __rte_unused)
2061 CHECK(!action, EINVAL);
2062 CHECK(n_tokens == 2, EINVAL);
2064 t = table_find(p, tokens[1]);
2066 instr->type = INSTR_TABLE;
2067 instr->table.table_id = t->id;
2071 s = selector_find(p, tokens[1]);
2073 instr->type = INSTR_SELECTOR;
2074 instr->table.table_id = s->id;
2078 l = learner_find(p, tokens[1]);
2080 instr->type = INSTR_LEARNER;
2081 instr->table.table_id = l->id;
2089 instr_table_exec(struct rte_swx_pipeline *p)
2091 struct thread *t = &p->threads[p->thread_id];
2092 struct instruction *ip = t->ip;
2093 uint32_t table_id = ip->table.table_id;
2094 struct rte_swx_table_state *ts = &t->table_state[table_id];
2095 struct table_runtime *table = &t->tables[table_id];
2096 struct table_statistics *stats = &p->table_stats[table_id];
2097 uint64_t action_id, n_pkts_hit, n_pkts_action;
2098 uint8_t *action_data;
2102 done = table->func(ts->obj,
2110 TRACE("[Thread %2u] table %u (not finalized)\n",
2118 action_id = hit ? action_id : ts->default_action_id;
2119 action_data = hit ? action_data : ts->default_action_data;
2120 n_pkts_hit = stats->n_pkts_hit[hit];
2121 n_pkts_action = stats->n_pkts_action[action_id];
2123 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2126 hit ? "hit" : "miss",
2127 (uint32_t)action_id);
2129 t->action_id = action_id;
2130 t->structs[0] = action_data;
2132 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2136 thread_ip_action_call(p, t, action_id);
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2142 struct thread *t = &p->threads[p->thread_id];
2143 struct instruction *ip = t->ip;
2144 uint32_t table_id = ip->table.table_id;
2145 struct rte_swx_table_state *ts = &t->table_state[table_id];
2146 struct table_runtime *table = &t->tables[table_id];
2147 struct table_statistics *stats = &p->table_stats[table_id];
2148 uint64_t action_id, n_pkts_hit, n_pkts_action;
2149 uint8_t *action_data;
2150 action_func_t action_func;
2154 done = table->func(ts->obj,
2162 TRACE("[Thread %2u] table %u (not finalized)\n",
2170 action_id = hit ? action_id : ts->default_action_id;
2171 action_data = hit ? action_data : ts->default_action_data;
2172 action_func = p->action_funcs[action_id];
2173 n_pkts_hit = stats->n_pkts_hit[hit];
2174 n_pkts_action = stats->n_pkts_action[action_id];
2176 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2179 hit ? "hit" : "miss",
2180 (uint32_t)action_id);
2182 t->action_id = action_id;
2183 t->structs[0] = action_data;
2185 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2198 struct thread *t = &p->threads[p->thread_id];
2199 struct instruction *ip = t->ip;
2200 uint32_t selector_id = ip->table.table_id;
2201 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202 struct selector_runtime *selector = &t->selectors[selector_id];
2203 struct selector_statistics *stats = &p->selector_stats[selector_id];
2204 uint64_t n_pkts = stats->n_pkts;
2208 done = rte_swx_table_selector_select(ts->obj,
2210 selector->group_id_buffer,
2211 selector->selector_buffer,
2212 selector->member_id_buffer);
2215 TRACE("[Thread %2u] selector %u (not finalized)\n",
2224 TRACE("[Thread %2u] selector %u\n",
2228 stats->n_pkts = n_pkts + 1;
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2237 struct thread *t = &p->threads[p->thread_id];
2238 struct instruction *ip = t->ip;
2239 uint32_t learner_id = ip->table.table_id;
2240 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241 p->n_selectors + learner_id];
2242 struct learner_runtime *l = &t->learners[learner_id];
2243 struct learner_statistics *stats = &p->learner_stats[learner_id];
2244 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245 uint8_t *action_data;
2249 time = rte_get_tsc_cycles();
2251 done = rte_swx_table_learner_lookup(ts->obj,
2260 TRACE("[Thread %2u] learner %u (not finalized)\n",
2268 action_id = hit ? action_id : ts->default_action_id;
2269 action_data = hit ? action_data : ts->default_action_data;
2270 n_pkts_hit = stats->n_pkts_hit[hit];
2271 n_pkts_action = stats->n_pkts_action[action_id];
2273 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2276 hit ? "hit" : "miss",
2277 (uint32_t)action_id);
2279 t->action_id = action_id;
2280 t->structs[0] = action_data;
2282 t->learner_id = learner_id;
2284 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2288 thread_ip_action_call(p, t, action_id);
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2294 struct thread *t = &p->threads[p->thread_id];
2295 struct instruction *ip = t->ip;
2296 uint32_t learner_id = ip->table.table_id;
2297 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298 p->n_selectors + learner_id];
2299 struct learner_runtime *l = &t->learners[learner_id];
2300 struct learner_statistics *stats = &p->learner_stats[learner_id];
2301 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302 uint8_t *action_data;
2303 action_func_t action_func;
2307 time = rte_get_tsc_cycles();
2309 done = rte_swx_table_learner_lookup(ts->obj,
2318 TRACE("[Thread %2u] learner %u (not finalized)\n",
2326 action_id = hit ? action_id : ts->default_action_id;
2327 action_data = hit ? action_data : ts->default_action_data;
2328 action_func = p->action_funcs[action_id];
2329 n_pkts_hit = stats->n_pkts_hit[hit];
2330 n_pkts_action = stats->n_pkts_action[action_id];
2332 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2335 hit ? "hit" : "miss",
2336 (uint32_t)action_id);
2338 t->action_id = action_id;
2339 t->structs[0] = action_data;
2341 t->learner_id = learner_id;
2343 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2347 thread_ip_action_call(p, t, action_id);
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2360 action_has_nbo_args(struct action *a);
2363 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2366 instr_learn_translate(struct rte_swx_pipeline *p,
2367 struct action *action,
2370 struct instruction *instr,
2371 struct instruction_data *data __rte_unused)
2374 const char *mf_name;
2375 uint32_t mf_offset = 0;
2377 CHECK(action, EINVAL);
2378 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2380 a = action_find(p, tokens[1]);
2382 CHECK(!action_has_nbo_args(a), EINVAL);
2384 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2385 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2390 mf = metadata_field_parse(p, mf_name);
2393 mf_offset = mf->offset / 8;
2396 instr->type = INSTR_LEARNER_LEARN;
2397 instr->learn.action_id = a->id;
2398 instr->learn.mf_offset = mf_offset;
2404 instr_learn_exec(struct rte_swx_pipeline *p)
2406 struct thread *t = &p->threads[p->thread_id];
2407 struct instruction *ip = t->ip;
2409 __instr_learn_exec(p, t, ip);
2419 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2420 struct action *action,
2421 char **tokens __rte_unused,
2423 struct instruction *instr,
2424 struct instruction_data *data __rte_unused)
2426 CHECK(action, EINVAL);
2427 CHECK(n_tokens == 1, EINVAL);
2429 instr->type = INSTR_LEARNER_FORGET;
2435 instr_forget_exec(struct rte_swx_pipeline *p)
2437 struct thread *t = &p->threads[p->thread_id];
2438 struct instruction *ip = t->ip;
2440 __instr_forget_exec(p, t, ip);
2450 instr_extern_translate(struct rte_swx_pipeline *p,
2451 struct action *action __rte_unused,
2454 struct instruction *instr,
2455 struct instruction_data *data __rte_unused)
2457 char *token = tokens[1];
2459 CHECK(n_tokens == 2, EINVAL);
2461 if (token[0] == 'e') {
2462 struct extern_obj *obj;
2463 struct extern_type_member_func *func;
2465 func = extern_obj_member_func_parse(p, token, &obj);
2466 CHECK(func, EINVAL);
2468 instr->type = INSTR_EXTERN_OBJ;
2469 instr->ext_obj.ext_obj_id = obj->id;
2470 instr->ext_obj.func_id = func->id;
2475 if (token[0] == 'f') {
2476 struct extern_func *func;
2478 func = extern_func_parse(p, token);
2479 CHECK(func, EINVAL);
2481 instr->type = INSTR_EXTERN_FUNC;
2482 instr->ext_func.ext_func_id = func->id;
2491 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2493 struct thread *t = &p->threads[p->thread_id];
2494 struct instruction *ip = t->ip;
2497 /* Extern object member function execute. */
2498 done = __instr_extern_obj_exec(p, t, ip);
2501 thread_ip_inc_cond(t, done);
2502 thread_yield_cond(p, done ^ 1);
2506 instr_extern_func_exec(struct rte_swx_pipeline *p)
2508 struct thread *t = &p->threads[p->thread_id];
2509 struct instruction *ip = t->ip;
2512 /* Extern function execute. */
2513 done = __instr_extern_func_exec(p, t, ip);
2516 thread_ip_inc_cond(t, done);
2517 thread_yield_cond(p, done ^ 1);
2524 instr_mov_translate(struct rte_swx_pipeline *p,
2525 struct action *action,
2528 struct instruction *instr,
2529 struct instruction_data *data __rte_unused)
2531 char *dst = tokens[1], *src = tokens[2];
2532 struct field *fdst, *fsrc;
2534 uint32_t dst_struct_id = 0, src_struct_id = 0;
2536 CHECK(n_tokens == 3, EINVAL);
2538 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2539 CHECK(fdst, EINVAL);
2540 CHECK(!fdst->var_size, EINVAL);
2542 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2543 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2545 CHECK(!fsrc->var_size, EINVAL);
2547 instr->type = INSTR_MOV;
2548 if (dst[0] != 'h' && src[0] == 'h')
2549 instr->type = INSTR_MOV_MH;
2550 if (dst[0] == 'h' && src[0] != 'h')
2551 instr->type = INSTR_MOV_HM;
2552 if (dst[0] == 'h' && src[0] == 'h')
2553 instr->type = INSTR_MOV_HH;
2555 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2556 instr->mov.dst.n_bits = fdst->n_bits;
2557 instr->mov.dst.offset = fdst->offset / 8;
2558 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2559 instr->mov.src.n_bits = fsrc->n_bits;
2560 instr->mov.src.offset = fsrc->offset / 8;
2565 src_val = strtoull(src, &src, 0);
2566 CHECK(!src[0], EINVAL);
2569 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2571 instr->type = INSTR_MOV_I;
2572 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2573 instr->mov.dst.n_bits = fdst->n_bits;
2574 instr->mov.dst.offset = fdst->offset / 8;
2575 instr->mov.src_val = src_val;
2580 instr_mov_exec(struct rte_swx_pipeline *p)
2582 struct thread *t = &p->threads[p->thread_id];
2583 struct instruction *ip = t->ip;
2585 __instr_mov_exec(p, t, ip);
2592 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2594 struct thread *t = &p->threads[p->thread_id];
2595 struct instruction *ip = t->ip;
2597 __instr_mov_mh_exec(p, t, ip);
2604 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2606 struct thread *t = &p->threads[p->thread_id];
2607 struct instruction *ip = t->ip;
2609 __instr_mov_hm_exec(p, t, ip);
2616 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2618 struct thread *t = &p->threads[p->thread_id];
2619 struct instruction *ip = t->ip;
2621 __instr_mov_hh_exec(p, t, ip);
2628 instr_mov_i_exec(struct rte_swx_pipeline *p)
2630 struct thread *t = &p->threads[p->thread_id];
2631 struct instruction *ip = t->ip;
2633 __instr_mov_i_exec(p, t, ip);
2643 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2645 struct thread *t = &p->threads[p->thread_id];
2646 struct instruction *ip = t->ip;
2648 __instr_dma_ht_exec(p, t, ip);
2655 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2657 struct thread *t = &p->threads[p->thread_id];
2658 struct instruction *ip = t->ip;
2660 __instr_dma_ht2_exec(p, t, ip);
2667 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2669 struct thread *t = &p->threads[p->thread_id];
2670 struct instruction *ip = t->ip;
2672 __instr_dma_ht3_exec(p, t, ip);
2679 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2681 struct thread *t = &p->threads[p->thread_id];
2682 struct instruction *ip = t->ip;
2684 __instr_dma_ht4_exec(p, t, ip);
2691 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2693 struct thread *t = &p->threads[p->thread_id];
2694 struct instruction *ip = t->ip;
2696 __instr_dma_ht5_exec(p, t, ip);
2703 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2705 struct thread *t = &p->threads[p->thread_id];
2706 struct instruction *ip = t->ip;
2708 __instr_dma_ht6_exec(p, t, ip);
2715 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2717 struct thread *t = &p->threads[p->thread_id];
2718 struct instruction *ip = t->ip;
2720 __instr_dma_ht7_exec(p, t, ip);
2727 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2729 struct thread *t = &p->threads[p->thread_id];
2730 struct instruction *ip = t->ip;
2732 __instr_dma_ht8_exec(p, t, ip);
2742 instr_alu_add_translate(struct rte_swx_pipeline *p,
2743 struct action *action,
2746 struct instruction *instr,
2747 struct instruction_data *data __rte_unused)
2749 char *dst = tokens[1], *src = tokens[2];
2750 struct field *fdst, *fsrc;
2752 uint32_t dst_struct_id = 0, src_struct_id = 0;
2754 CHECK(n_tokens == 3, EINVAL);
2756 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2757 CHECK(fdst, EINVAL);
2758 CHECK(!fdst->var_size, EINVAL);
2760 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2761 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2763 CHECK(!fsrc->var_size, EINVAL);
2765 instr->type = INSTR_ALU_ADD;
2766 if (dst[0] == 'h' && src[0] != 'h')
2767 instr->type = INSTR_ALU_ADD_HM;
2768 if (dst[0] != 'h' && src[0] == 'h')
2769 instr->type = INSTR_ALU_ADD_MH;
2770 if (dst[0] == 'h' && src[0] == 'h')
2771 instr->type = INSTR_ALU_ADD_HH;
2773 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2774 instr->alu.dst.n_bits = fdst->n_bits;
2775 instr->alu.dst.offset = fdst->offset / 8;
2776 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2777 instr->alu.src.n_bits = fsrc->n_bits;
2778 instr->alu.src.offset = fsrc->offset / 8;
2782 /* ADD_MI, ADD_HI. */
2783 src_val = strtoull(src, &src, 0);
2784 CHECK(!src[0], EINVAL);
2786 instr->type = INSTR_ALU_ADD_MI;
2788 instr->type = INSTR_ALU_ADD_HI;
2790 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2791 instr->alu.dst.n_bits = fdst->n_bits;
2792 instr->alu.dst.offset = fdst->offset / 8;
2793 instr->alu.src_val = src_val;
2798 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2799 struct action *action,
2802 struct instruction *instr,
2803 struct instruction_data *data __rte_unused)
2805 char *dst = tokens[1], *src = tokens[2];
2806 struct field *fdst, *fsrc;
2808 uint32_t dst_struct_id = 0, src_struct_id = 0;
2810 CHECK(n_tokens == 3, EINVAL);
2812 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2813 CHECK(fdst, EINVAL);
2814 CHECK(!fdst->var_size, EINVAL);
2816 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2817 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2819 CHECK(!fsrc->var_size, EINVAL);
2821 instr->type = INSTR_ALU_SUB;
2822 if (dst[0] == 'h' && src[0] != 'h')
2823 instr->type = INSTR_ALU_SUB_HM;
2824 if (dst[0] != 'h' && src[0] == 'h')
2825 instr->type = INSTR_ALU_SUB_MH;
2826 if (dst[0] == 'h' && src[0] == 'h')
2827 instr->type = INSTR_ALU_SUB_HH;
2829 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2830 instr->alu.dst.n_bits = fdst->n_bits;
2831 instr->alu.dst.offset = fdst->offset / 8;
2832 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2833 instr->alu.src.n_bits = fsrc->n_bits;
2834 instr->alu.src.offset = fsrc->offset / 8;
2838 /* SUB_MI, SUB_HI. */
2839 src_val = strtoull(src, &src, 0);
2840 CHECK(!src[0], EINVAL);
2842 instr->type = INSTR_ALU_SUB_MI;
2844 instr->type = INSTR_ALU_SUB_HI;
2846 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2847 instr->alu.dst.n_bits = fdst->n_bits;
2848 instr->alu.dst.offset = fdst->offset / 8;
2849 instr->alu.src_val = src_val;
2854 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2855 struct action *action __rte_unused,
2858 struct instruction *instr,
2859 struct instruction_data *data __rte_unused)
2861 char *dst = tokens[1], *src = tokens[2];
2862 struct header *hdst, *hsrc;
2863 struct field *fdst, *fsrc;
2865 CHECK(n_tokens == 3, EINVAL);
2867 fdst = header_field_parse(p, dst, &hdst);
2868 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2869 CHECK(!fdst->var_size, EINVAL);
2872 fsrc = header_field_parse(p, src, &hsrc);
2874 CHECK(!fsrc->var_size, EINVAL);
2876 instr->type = INSTR_ALU_CKADD_FIELD;
2877 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878 instr->alu.dst.n_bits = fdst->n_bits;
2879 instr->alu.dst.offset = fdst->offset / 8;
2880 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881 instr->alu.src.n_bits = fsrc->n_bits;
2882 instr->alu.src.offset = fsrc->offset / 8;
2886 /* CKADD_STRUCT, CKADD_STRUCT20. */
2887 hsrc = header_parse(p, src);
2888 CHECK(hsrc, EINVAL);
2889 CHECK(!hsrc->st->var_size, EINVAL);
2891 instr->type = INSTR_ALU_CKADD_STRUCT;
2892 if ((hsrc->st->n_bits / 8) == 20)
2893 instr->type = INSTR_ALU_CKADD_STRUCT20;
2895 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2896 instr->alu.dst.n_bits = fdst->n_bits;
2897 instr->alu.dst.offset = fdst->offset / 8;
2898 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2899 instr->alu.src.n_bits = hsrc->st->n_bits;
2900 instr->alu.src.offset = 0; /* Unused. */
2905 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2906 struct action *action __rte_unused,
2909 struct instruction *instr,
2910 struct instruction_data *data __rte_unused)
2912 char *dst = tokens[1], *src = tokens[2];
2913 struct header *hdst, *hsrc;
2914 struct field *fdst, *fsrc;
2916 CHECK(n_tokens == 3, EINVAL);
2918 fdst = header_field_parse(p, dst, &hdst);
2919 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2920 CHECK(!fdst->var_size, EINVAL);
2922 fsrc = header_field_parse(p, src, &hsrc);
2923 CHECK(fsrc, EINVAL);
2924 CHECK(!fsrc->var_size, EINVAL);
2926 instr->type = INSTR_ALU_CKSUB_FIELD;
2927 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2928 instr->alu.dst.n_bits = fdst->n_bits;
2929 instr->alu.dst.offset = fdst->offset / 8;
2930 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2931 instr->alu.src.n_bits = fsrc->n_bits;
2932 instr->alu.src.offset = fsrc->offset / 8;
2937 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2938 struct action *action,
2941 struct instruction *instr,
2942 struct instruction_data *data __rte_unused)
2944 char *dst = tokens[1], *src = tokens[2];
2945 struct field *fdst, *fsrc;
2947 uint32_t dst_struct_id = 0, src_struct_id = 0;
2949 CHECK(n_tokens == 3, EINVAL);
2951 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2952 CHECK(fdst, EINVAL);
2953 CHECK(!fdst->var_size, EINVAL);
2955 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2956 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2958 CHECK(!fsrc->var_size, EINVAL);
2960 instr->type = INSTR_ALU_SHL;
2961 if (dst[0] == 'h' && src[0] != 'h')
2962 instr->type = INSTR_ALU_SHL_HM;
2963 if (dst[0] != 'h' && src[0] == 'h')
2964 instr->type = INSTR_ALU_SHL_MH;
2965 if (dst[0] == 'h' && src[0] == 'h')
2966 instr->type = INSTR_ALU_SHL_HH;
2968 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2969 instr->alu.dst.n_bits = fdst->n_bits;
2970 instr->alu.dst.offset = fdst->offset / 8;
2971 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2972 instr->alu.src.n_bits = fsrc->n_bits;
2973 instr->alu.src.offset = fsrc->offset / 8;
2977 /* SHL_MI, SHL_HI. */
2978 src_val = strtoull(src, &src, 0);
2979 CHECK(!src[0], EINVAL);
2981 instr->type = INSTR_ALU_SHL_MI;
2983 instr->type = INSTR_ALU_SHL_HI;
2985 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986 instr->alu.dst.n_bits = fdst->n_bits;
2987 instr->alu.dst.offset = fdst->offset / 8;
2988 instr->alu.src_val = src_val;
2993 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2994 struct action *action,
2997 struct instruction *instr,
2998 struct instruction_data *data __rte_unused)
3000 char *dst = tokens[1], *src = tokens[2];
3001 struct field *fdst, *fsrc;
3003 uint32_t dst_struct_id = 0, src_struct_id = 0;
3005 CHECK(n_tokens == 3, EINVAL);
3007 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3008 CHECK(fdst, EINVAL);
3009 CHECK(!fdst->var_size, EINVAL);
3011 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3012 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3014 CHECK(!fsrc->var_size, EINVAL);
3016 instr->type = INSTR_ALU_SHR;
3017 if (dst[0] == 'h' && src[0] != 'h')
3018 instr->type = INSTR_ALU_SHR_HM;
3019 if (dst[0] != 'h' && src[0] == 'h')
3020 instr->type = INSTR_ALU_SHR_MH;
3021 if (dst[0] == 'h' && src[0] == 'h')
3022 instr->type = INSTR_ALU_SHR_HH;
3024 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3025 instr->alu.dst.n_bits = fdst->n_bits;
3026 instr->alu.dst.offset = fdst->offset / 8;
3027 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3028 instr->alu.src.n_bits = fsrc->n_bits;
3029 instr->alu.src.offset = fsrc->offset / 8;
3033 /* SHR_MI, SHR_HI. */
3034 src_val = strtoull(src, &src, 0);
3035 CHECK(!src[0], EINVAL);
3037 instr->type = INSTR_ALU_SHR_MI;
3039 instr->type = INSTR_ALU_SHR_HI;
3041 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042 instr->alu.dst.n_bits = fdst->n_bits;
3043 instr->alu.dst.offset = fdst->offset / 8;
3044 instr->alu.src_val = src_val;
3049 instr_alu_and_translate(struct rte_swx_pipeline *p,
3050 struct action *action,
3053 struct instruction *instr,
3054 struct instruction_data *data __rte_unused)
3056 char *dst = tokens[1], *src = tokens[2];
3057 struct field *fdst, *fsrc;
3059 uint32_t dst_struct_id = 0, src_struct_id = 0;
3061 CHECK(n_tokens == 3, EINVAL);
3063 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3064 CHECK(fdst, EINVAL);
3065 CHECK(!fdst->var_size, EINVAL);
3067 /* AND, AND_MH, AND_HM, AND_HH. */
3068 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3070 CHECK(!fsrc->var_size, EINVAL);
3072 instr->type = INSTR_ALU_AND;
3073 if (dst[0] != 'h' && src[0] == 'h')
3074 instr->type = INSTR_ALU_AND_MH;
3075 if (dst[0] == 'h' && src[0] != 'h')
3076 instr->type = INSTR_ALU_AND_HM;
3077 if (dst[0] == 'h' && src[0] == 'h')
3078 instr->type = INSTR_ALU_AND_HH;
3080 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3081 instr->alu.dst.n_bits = fdst->n_bits;
3082 instr->alu.dst.offset = fdst->offset / 8;
3083 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3084 instr->alu.src.n_bits = fsrc->n_bits;
3085 instr->alu.src.offset = fsrc->offset / 8;
3090 src_val = strtoull(src, &src, 0);
3091 CHECK(!src[0], EINVAL);
3094 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3096 instr->type = INSTR_ALU_AND_I;
3097 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3098 instr->alu.dst.n_bits = fdst->n_bits;
3099 instr->alu.dst.offset = fdst->offset / 8;
3100 instr->alu.src_val = src_val;
3105 instr_alu_or_translate(struct rte_swx_pipeline *p,
3106 struct action *action,
3109 struct instruction *instr,
3110 struct instruction_data *data __rte_unused)
3112 char *dst = tokens[1], *src = tokens[2];
3113 struct field *fdst, *fsrc;
3115 uint32_t dst_struct_id = 0, src_struct_id = 0;
3117 CHECK(n_tokens == 3, EINVAL);
3119 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3120 CHECK(fdst, EINVAL);
3121 CHECK(!fdst->var_size, EINVAL);
3123 /* OR, OR_MH, OR_HM, OR_HH. */
3124 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3126 CHECK(!fsrc->var_size, EINVAL);
3128 instr->type = INSTR_ALU_OR;
3129 if (dst[0] != 'h' && src[0] == 'h')
3130 instr->type = INSTR_ALU_OR_MH;
3131 if (dst[0] == 'h' && src[0] != 'h')
3132 instr->type = INSTR_ALU_OR_HM;
3133 if (dst[0] == 'h' && src[0] == 'h')
3134 instr->type = INSTR_ALU_OR_HH;
3136 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3137 instr->alu.dst.n_bits = fdst->n_bits;
3138 instr->alu.dst.offset = fdst->offset / 8;
3139 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3140 instr->alu.src.n_bits = fsrc->n_bits;
3141 instr->alu.src.offset = fsrc->offset / 8;
3146 src_val = strtoull(src, &src, 0);
3147 CHECK(!src[0], EINVAL);
3150 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3152 instr->type = INSTR_ALU_OR_I;
3153 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3154 instr->alu.dst.n_bits = fdst->n_bits;
3155 instr->alu.dst.offset = fdst->offset / 8;
3156 instr->alu.src_val = src_val;
3161 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3162 struct action *action,
3165 struct instruction *instr,
3166 struct instruction_data *data __rte_unused)
3168 char *dst = tokens[1], *src = tokens[2];
3169 struct field *fdst, *fsrc;
3171 uint32_t dst_struct_id = 0, src_struct_id = 0;
3173 CHECK(n_tokens == 3, EINVAL);
3175 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3176 CHECK(fdst, EINVAL);
3177 CHECK(!fdst->var_size, EINVAL);
3179 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3180 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3182 CHECK(!fsrc->var_size, EINVAL);
3184 instr->type = INSTR_ALU_XOR;
3185 if (dst[0] != 'h' && src[0] == 'h')
3186 instr->type = INSTR_ALU_XOR_MH;
3187 if (dst[0] == 'h' && src[0] != 'h')
3188 instr->type = INSTR_ALU_XOR_HM;
3189 if (dst[0] == 'h' && src[0] == 'h')
3190 instr->type = INSTR_ALU_XOR_HH;
3192 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3193 instr->alu.dst.n_bits = fdst->n_bits;
3194 instr->alu.dst.offset = fdst->offset / 8;
3195 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3196 instr->alu.src.n_bits = fsrc->n_bits;
3197 instr->alu.src.offset = fsrc->offset / 8;
3202 src_val = strtoull(src, &src, 0);
3203 CHECK(!src[0], EINVAL);
3206 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3208 instr->type = INSTR_ALU_XOR_I;
3209 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3210 instr->alu.dst.n_bits = fdst->n_bits;
3211 instr->alu.dst.offset = fdst->offset / 8;
3212 instr->alu.src_val = src_val;
3217 instr_alu_add_exec(struct rte_swx_pipeline *p)
3219 struct thread *t = &p->threads[p->thread_id];
3220 struct instruction *ip = t->ip;
3223 __instr_alu_add_exec(p, t, ip);
3230 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3232 struct thread *t = &p->threads[p->thread_id];
3233 struct instruction *ip = t->ip;
3236 __instr_alu_add_mh_exec(p, t, ip);
3243 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3245 struct thread *t = &p->threads[p->thread_id];
3246 struct instruction *ip = t->ip;
3249 __instr_alu_add_hm_exec(p, t, ip);
3256 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3258 struct thread *t = &p->threads[p->thread_id];
3259 struct instruction *ip = t->ip;
3262 __instr_alu_add_hh_exec(p, t, ip);
3269 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3271 struct thread *t = &p->threads[p->thread_id];
3272 struct instruction *ip = t->ip;
3275 __instr_alu_add_mi_exec(p, t, ip);
3282 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3284 struct thread *t = &p->threads[p->thread_id];
3285 struct instruction *ip = t->ip;
3288 __instr_alu_add_hi_exec(p, t, ip);
3295 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3297 struct thread *t = &p->threads[p->thread_id];
3298 struct instruction *ip = t->ip;
3301 __instr_alu_sub_exec(p, t, ip);
3308 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3310 struct thread *t = &p->threads[p->thread_id];
3311 struct instruction *ip = t->ip;
3314 __instr_alu_sub_mh_exec(p, t, ip);
3321 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3323 struct thread *t = &p->threads[p->thread_id];
3324 struct instruction *ip = t->ip;
3327 __instr_alu_sub_hm_exec(p, t, ip);
3334 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3336 struct thread *t = &p->threads[p->thread_id];
3337 struct instruction *ip = t->ip;
3340 __instr_alu_sub_hh_exec(p, t, ip);
3347 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3349 struct thread *t = &p->threads[p->thread_id];
3350 struct instruction *ip = t->ip;
3353 __instr_alu_sub_mi_exec(p, t, ip);
3360 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3362 struct thread *t = &p->threads[p->thread_id];
3363 struct instruction *ip = t->ip;
3366 __instr_alu_sub_hi_exec(p, t, ip);
3373 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3375 struct thread *t = &p->threads[p->thread_id];
3376 struct instruction *ip = t->ip;
3379 __instr_alu_shl_exec(p, t, ip);
3386 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3388 struct thread *t = &p->threads[p->thread_id];
3389 struct instruction *ip = t->ip;
3392 __instr_alu_shl_mh_exec(p, t, ip);
3399 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3401 struct thread *t = &p->threads[p->thread_id];
3402 struct instruction *ip = t->ip;
3405 __instr_alu_shl_hm_exec(p, t, ip);
3412 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3414 struct thread *t = &p->threads[p->thread_id];
3415 struct instruction *ip = t->ip;
3418 __instr_alu_shl_hh_exec(p, t, ip);
3425 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3427 struct thread *t = &p->threads[p->thread_id];
3428 struct instruction *ip = t->ip;
3431 __instr_alu_shl_mi_exec(p, t, ip);
3438 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3440 struct thread *t = &p->threads[p->thread_id];
3441 struct instruction *ip = t->ip;
3444 __instr_alu_shl_hi_exec(p, t, ip);
3451 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3453 struct thread *t = &p->threads[p->thread_id];
3454 struct instruction *ip = t->ip;
3457 __instr_alu_shr_exec(p, t, ip);
3464 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3466 struct thread *t = &p->threads[p->thread_id];
3467 struct instruction *ip = t->ip;
3470 __instr_alu_shr_mh_exec(p, t, ip);
3477 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3479 struct thread *t = &p->threads[p->thread_id];
3480 struct instruction *ip = t->ip;
3483 __instr_alu_shr_hm_exec(p, t, ip);
3490 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3492 struct thread *t = &p->threads[p->thread_id];
3493 struct instruction *ip = t->ip;
3496 __instr_alu_shr_hh_exec(p, t, ip);
3503 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3505 struct thread *t = &p->threads[p->thread_id];
3506 struct instruction *ip = t->ip;
3509 __instr_alu_shr_mi_exec(p, t, ip);
3516 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3518 struct thread *t = &p->threads[p->thread_id];
3519 struct instruction *ip = t->ip;
3522 __instr_alu_shr_hi_exec(p, t, ip);
3529 instr_alu_and_exec(struct rte_swx_pipeline *p)
3531 struct thread *t = &p->threads[p->thread_id];
3532 struct instruction *ip = t->ip;
3535 __instr_alu_and_exec(p, t, ip);
3542 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3544 struct thread *t = &p->threads[p->thread_id];
3545 struct instruction *ip = t->ip;
3548 __instr_alu_and_mh_exec(p, t, ip);
3555 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3557 struct thread *t = &p->threads[p->thread_id];
3558 struct instruction *ip = t->ip;
3561 __instr_alu_and_hm_exec(p, t, ip);
3568 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3570 struct thread *t = &p->threads[p->thread_id];
3571 struct instruction *ip = t->ip;
3574 __instr_alu_and_hh_exec(p, t, ip);
3581 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3583 struct thread *t = &p->threads[p->thread_id];
3584 struct instruction *ip = t->ip;
3587 __instr_alu_and_i_exec(p, t, ip);
3594 instr_alu_or_exec(struct rte_swx_pipeline *p)
3596 struct thread *t = &p->threads[p->thread_id];
3597 struct instruction *ip = t->ip;
3600 __instr_alu_or_exec(p, t, ip);
3607 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3609 struct thread *t = &p->threads[p->thread_id];
3610 struct instruction *ip = t->ip;
3613 __instr_alu_or_mh_exec(p, t, ip);
3620 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3622 struct thread *t = &p->threads[p->thread_id];
3623 struct instruction *ip = t->ip;
3626 __instr_alu_or_hm_exec(p, t, ip);
3633 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3635 struct thread *t = &p->threads[p->thread_id];
3636 struct instruction *ip = t->ip;
3639 __instr_alu_or_hh_exec(p, t, ip);
3646 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3648 struct thread *t = &p->threads[p->thread_id];
3649 struct instruction *ip = t->ip;
3652 __instr_alu_or_i_exec(p, t, ip);
3659 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3661 struct thread *t = &p->threads[p->thread_id];
3662 struct instruction *ip = t->ip;
3665 __instr_alu_xor_exec(p, t, ip);
3672 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3674 struct thread *t = &p->threads[p->thread_id];
3675 struct instruction *ip = t->ip;
3678 __instr_alu_xor_mh_exec(p, t, ip);
3685 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3687 struct thread *t = &p->threads[p->thread_id];
3688 struct instruction *ip = t->ip;
3691 __instr_alu_xor_hm_exec(p, t, ip);
3698 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3700 struct thread *t = &p->threads[p->thread_id];
3701 struct instruction *ip = t->ip;
3704 __instr_alu_xor_hh_exec(p, t, ip);
3711 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3713 struct thread *t = &p->threads[p->thread_id];
3714 struct instruction *ip = t->ip;
3717 __instr_alu_xor_i_exec(p, t, ip);
3724 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3726 struct thread *t = &p->threads[p->thread_id];
3727 struct instruction *ip = t->ip;
3730 __instr_alu_ckadd_field_exec(p, t, ip);
3737 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3739 struct thread *t = &p->threads[p->thread_id];
3740 struct instruction *ip = t->ip;
3743 __instr_alu_cksub_field_exec(p, t, ip);
3750 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3752 struct thread *t = &p->threads[p->thread_id];
3753 struct instruction *ip = t->ip;
3756 __instr_alu_ckadd_struct20_exec(p, t, ip);
3763 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3765 struct thread *t = &p->threads[p->thread_id];
3766 struct instruction *ip = t->ip;
3769 __instr_alu_ckadd_struct_exec(p, t, ip);
3778 static struct regarray *
3779 regarray_find(struct rte_swx_pipeline *p, const char *name);
3782 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3783 struct action *action,
3786 struct instruction *instr,
3787 struct instruction_data *data __rte_unused)
3789 char *regarray = tokens[1], *idx = tokens[2];
3792 uint32_t idx_struct_id, idx_val;
3794 CHECK(n_tokens == 3, EINVAL);
3796 r = regarray_find(p, regarray);
3799 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3800 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3802 CHECK(!fidx->var_size, EINVAL);
3804 instr->type = INSTR_REGPREFETCH_RM;
3806 instr->type = INSTR_REGPREFETCH_RH;
3808 instr->regarray.regarray_id = r->id;
3809 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3810 instr->regarray.idx.n_bits = fidx->n_bits;
3811 instr->regarray.idx.offset = fidx->offset / 8;
3812 instr->regarray.dstsrc_val = 0; /* Unused. */
3816 /* REGPREFETCH_RI. */
3817 idx_val = strtoul(idx, &idx, 0);
3818 CHECK(!idx[0], EINVAL);
3820 instr->type = INSTR_REGPREFETCH_RI;
3821 instr->regarray.regarray_id = r->id;
3822 instr->regarray.idx_val = idx_val;
3823 instr->regarray.dstsrc_val = 0; /* Unused. */
3828 instr_regrd_translate(struct rte_swx_pipeline *p,
3829 struct action *action,
3832 struct instruction *instr,
3833 struct instruction_data *data __rte_unused)
3835 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3837 struct field *fdst, *fidx;
3838 uint32_t dst_struct_id, idx_struct_id, idx_val;
3840 CHECK(n_tokens == 4, EINVAL);
3842 r = regarray_find(p, regarray);
3845 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3846 CHECK(fdst, EINVAL);
3847 CHECK(!fdst->var_size, EINVAL);
3849 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3850 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3852 CHECK(!fidx->var_size, EINVAL);
3854 instr->type = INSTR_REGRD_MRM;
3855 if (dst[0] == 'h' && idx[0] != 'h')
3856 instr->type = INSTR_REGRD_HRM;
3857 if (dst[0] != 'h' && idx[0] == 'h')
3858 instr->type = INSTR_REGRD_MRH;
3859 if (dst[0] == 'h' && idx[0] == 'h')
3860 instr->type = INSTR_REGRD_HRH;
3862 instr->regarray.regarray_id = r->id;
3863 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3864 instr->regarray.idx.n_bits = fidx->n_bits;
3865 instr->regarray.idx.offset = fidx->offset / 8;
3866 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3867 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3868 instr->regarray.dstsrc.offset = fdst->offset / 8;
3872 /* REGRD_MRI, REGRD_HRI. */
3873 idx_val = strtoul(idx, &idx, 0);
3874 CHECK(!idx[0], EINVAL);
3876 instr->type = INSTR_REGRD_MRI;
3878 instr->type = INSTR_REGRD_HRI;
3880 instr->regarray.regarray_id = r->id;
3881 instr->regarray.idx_val = idx_val;
3882 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3883 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3884 instr->regarray.dstsrc.offset = fdst->offset / 8;
3889 instr_regwr_translate(struct rte_swx_pipeline *p,
3890 struct action *action,
3893 struct instruction *instr,
3894 struct instruction_data *data __rte_unused)
3896 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3898 struct field *fidx, *fsrc;
3900 uint32_t idx_struct_id, idx_val, src_struct_id;
3902 CHECK(n_tokens == 4, EINVAL);
3904 r = regarray_find(p, regarray);
3907 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3908 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3909 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3911 CHECK(!fidx->var_size, EINVAL);
3912 CHECK(!fsrc->var_size, EINVAL);
3914 instr->type = INSTR_REGWR_RMM;
3915 if (idx[0] == 'h' && src[0] != 'h')
3916 instr->type = INSTR_REGWR_RHM;
3917 if (idx[0] != 'h' && src[0] == 'h')
3918 instr->type = INSTR_REGWR_RMH;
3919 if (idx[0] == 'h' && src[0] == 'h')
3920 instr->type = INSTR_REGWR_RHH;
3922 instr->regarray.regarray_id = r->id;
3923 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3924 instr->regarray.idx.n_bits = fidx->n_bits;
3925 instr->regarray.idx.offset = fidx->offset / 8;
3926 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3927 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3928 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3932 /* REGWR_RHI, REGWR_RMI. */
3933 if (fidx && !fsrc) {
3934 CHECK(!fidx->var_size, EINVAL);
3936 src_val = strtoull(src, &src, 0);
3937 CHECK(!src[0], EINVAL);
3939 instr->type = INSTR_REGWR_RMI;
3941 instr->type = INSTR_REGWR_RHI;
3943 instr->regarray.regarray_id = r->id;
3944 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3945 instr->regarray.idx.n_bits = fidx->n_bits;
3946 instr->regarray.idx.offset = fidx->offset / 8;
3947 instr->regarray.dstsrc_val = src_val;
3951 /* REGWR_RIH, REGWR_RIM. */
3952 if (!fidx && fsrc) {
3953 idx_val = strtoul(idx, &idx, 0);
3954 CHECK(!idx[0], EINVAL);
3956 CHECK(!fsrc->var_size, EINVAL);
3958 instr->type = INSTR_REGWR_RIM;
3960 instr->type = INSTR_REGWR_RIH;
3962 instr->regarray.regarray_id = r->id;
3963 instr->regarray.idx_val = idx_val;
3964 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3965 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3966 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3971 src_val = strtoull(src, &src, 0);
3972 CHECK(!src[0], EINVAL);
3974 idx_val = strtoul(idx, &idx, 0);
3975 CHECK(!idx[0], EINVAL);
3977 instr->type = INSTR_REGWR_RII;
3978 instr->regarray.idx_val = idx_val;
3979 instr->regarray.dstsrc_val = src_val;
3985 instr_regadd_translate(struct rte_swx_pipeline *p,
3986 struct action *action,
3989 struct instruction *instr,
3990 struct instruction_data *data __rte_unused)
3992 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3994 struct field *fidx, *fsrc;
3996 uint32_t idx_struct_id, idx_val, src_struct_id;
3998 CHECK(n_tokens == 4, EINVAL);
4000 r = regarray_find(p, regarray);
4003 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4004 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4005 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4007 CHECK(!fidx->var_size, EINVAL);
4008 CHECK(!fsrc->var_size, EINVAL);
4010 instr->type = INSTR_REGADD_RMM;
4011 if (idx[0] == 'h' && src[0] != 'h')
4012 instr->type = INSTR_REGADD_RHM;
4013 if (idx[0] != 'h' && src[0] == 'h')
4014 instr->type = INSTR_REGADD_RMH;
4015 if (idx[0] == 'h' && src[0] == 'h')
4016 instr->type = INSTR_REGADD_RHH;
4018 instr->regarray.regarray_id = r->id;
4019 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4020 instr->regarray.idx.n_bits = fidx->n_bits;
4021 instr->regarray.idx.offset = fidx->offset / 8;
4022 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4023 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4024 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4028 /* REGADD_RHI, REGADD_RMI. */
4029 if (fidx && !fsrc) {
4030 CHECK(!fidx->var_size, EINVAL);
4032 src_val = strtoull(src, &src, 0);
4033 CHECK(!src[0], EINVAL);
4035 instr->type = INSTR_REGADD_RMI;
4037 instr->type = INSTR_REGADD_RHI;
4039 instr->regarray.regarray_id = r->id;
4040 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4041 instr->regarray.idx.n_bits = fidx->n_bits;
4042 instr->regarray.idx.offset = fidx->offset / 8;
4043 instr->regarray.dstsrc_val = src_val;
4047 /* REGADD_RIH, REGADD_RIM. */
4048 if (!fidx && fsrc) {
4049 idx_val = strtoul(idx, &idx, 0);
4050 CHECK(!idx[0], EINVAL);
4052 CHECK(!fsrc->var_size, EINVAL);
4054 instr->type = INSTR_REGADD_RIM;
4056 instr->type = INSTR_REGADD_RIH;
4058 instr->regarray.regarray_id = r->id;
4059 instr->regarray.idx_val = idx_val;
4060 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4061 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4062 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4067 src_val = strtoull(src, &src, 0);
4068 CHECK(!src[0], EINVAL);
4070 idx_val = strtoul(idx, &idx, 0);
4071 CHECK(!idx[0], EINVAL);
4073 instr->type = INSTR_REGADD_RII;
4074 instr->regarray.idx_val = idx_val;
4075 instr->regarray.dstsrc_val = src_val;
4080 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4082 struct thread *t = &p->threads[p->thread_id];
4083 struct instruction *ip = t->ip;
4086 __instr_regprefetch_rh_exec(p, t, ip);
4093 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4095 struct thread *t = &p->threads[p->thread_id];
4096 struct instruction *ip = t->ip;
4099 __instr_regprefetch_rm_exec(p, t, ip);
4106 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4108 struct thread *t = &p->threads[p->thread_id];
4109 struct instruction *ip = t->ip;
4112 __instr_regprefetch_ri_exec(p, t, ip);
4119 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4121 struct thread *t = &p->threads[p->thread_id];
4122 struct instruction *ip = t->ip;
4125 __instr_regrd_hrh_exec(p, t, ip);
4132 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4134 struct thread *t = &p->threads[p->thread_id];
4135 struct instruction *ip = t->ip;
4138 __instr_regrd_hrm_exec(p, t, ip);
4145 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4147 struct thread *t = &p->threads[p->thread_id];
4148 struct instruction *ip = t->ip;
4151 __instr_regrd_mrh_exec(p, t, ip);
4158 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4160 struct thread *t = &p->threads[p->thread_id];
4161 struct instruction *ip = t->ip;
4164 __instr_regrd_mrm_exec(p, t, ip);
4171 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4173 struct thread *t = &p->threads[p->thread_id];
4174 struct instruction *ip = t->ip;
4177 __instr_regrd_hri_exec(p, t, ip);
4184 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4186 struct thread *t = &p->threads[p->thread_id];
4187 struct instruction *ip = t->ip;
4190 __instr_regrd_mri_exec(p, t, ip);
4197 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4199 struct thread *t = &p->threads[p->thread_id];
4200 struct instruction *ip = t->ip;
4203 __instr_regwr_rhh_exec(p, t, ip);
4210 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4212 struct thread *t = &p->threads[p->thread_id];
4213 struct instruction *ip = t->ip;
4216 __instr_regwr_rhm_exec(p, t, ip);
4223 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4225 struct thread *t = &p->threads[p->thread_id];
4226 struct instruction *ip = t->ip;
4229 __instr_regwr_rmh_exec(p, t, ip);
4236 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4238 struct thread *t = &p->threads[p->thread_id];
4239 struct instruction *ip = t->ip;
4242 __instr_regwr_rmm_exec(p, t, ip);
4249 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4251 struct thread *t = &p->threads[p->thread_id];
4252 struct instruction *ip = t->ip;
4255 __instr_regwr_rhi_exec(p, t, ip);
4262 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4264 struct thread *t = &p->threads[p->thread_id];
4265 struct instruction *ip = t->ip;
4268 __instr_regwr_rmi_exec(p, t, ip);
4275 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4277 struct thread *t = &p->threads[p->thread_id];
4278 struct instruction *ip = t->ip;
4281 __instr_regwr_rih_exec(p, t, ip);
4288 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4290 struct thread *t = &p->threads[p->thread_id];
4291 struct instruction *ip = t->ip;
4294 __instr_regwr_rim_exec(p, t, ip);
4301 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4303 struct thread *t = &p->threads[p->thread_id];
4304 struct instruction *ip = t->ip;
4307 __instr_regwr_rii_exec(p, t, ip);
4314 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4316 struct thread *t = &p->threads[p->thread_id];
4317 struct instruction *ip = t->ip;
4320 __instr_regadd_rhh_exec(p, t, ip);
4327 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4329 struct thread *t = &p->threads[p->thread_id];
4330 struct instruction *ip = t->ip;
4333 __instr_regadd_rhm_exec(p, t, ip);
4340 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4342 struct thread *t = &p->threads[p->thread_id];
4343 struct instruction *ip = t->ip;
4346 __instr_regadd_rmh_exec(p, t, ip);
4353 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4355 struct thread *t = &p->threads[p->thread_id];
4356 struct instruction *ip = t->ip;
4359 __instr_regadd_rmm_exec(p, t, ip);
4366 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4368 struct thread *t = &p->threads[p->thread_id];
4369 struct instruction *ip = t->ip;
4372 __instr_regadd_rhi_exec(p, t, ip);
4379 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4381 struct thread *t = &p->threads[p->thread_id];
4382 struct instruction *ip = t->ip;
4385 __instr_regadd_rmi_exec(p, t, ip);
4392 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4394 struct thread *t = &p->threads[p->thread_id];
4395 struct instruction *ip = t->ip;
4398 __instr_regadd_rih_exec(p, t, ip);
4405 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4407 struct thread *t = &p->threads[p->thread_id];
4408 struct instruction *ip = t->ip;
4411 __instr_regadd_rim_exec(p, t, ip);
4418 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4420 struct thread *t = &p->threads[p->thread_id];
4421 struct instruction *ip = t->ip;
4424 __instr_regadd_rii_exec(p, t, ip);
4433 static struct metarray *
4434 metarray_find(struct rte_swx_pipeline *p, const char *name);
4437 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4438 struct action *action,
4441 struct instruction *instr,
4442 struct instruction_data *data __rte_unused)
4444 char *metarray = tokens[1], *idx = tokens[2];
4447 uint32_t idx_struct_id, idx_val;
4449 CHECK(n_tokens == 3, EINVAL);
4451 m = metarray_find(p, metarray);
4454 /* METPREFETCH_H, METPREFETCH_M. */
4455 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4457 CHECK(!fidx->var_size, EINVAL);
4459 instr->type = INSTR_METPREFETCH_M;
4461 instr->type = INSTR_METPREFETCH_H;
4463 instr->meter.metarray_id = m->id;
4464 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4465 instr->meter.idx.n_bits = fidx->n_bits;
4466 instr->meter.idx.offset = fidx->offset / 8;
4470 /* METPREFETCH_I. */
4471 idx_val = strtoul(idx, &idx, 0);
4472 CHECK(!idx[0], EINVAL);
4474 instr->type = INSTR_METPREFETCH_I;
4475 instr->meter.metarray_id = m->id;
4476 instr->meter.idx_val = idx_val;
4481 instr_meter_translate(struct rte_swx_pipeline *p,
4482 struct action *action,
4485 struct instruction *instr,
4486 struct instruction_data *data __rte_unused)
4488 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4489 char *color_in = tokens[4], *color_out = tokens[5];
4491 struct field *fidx, *flength, *fcin, *fcout;
4492 uint32_t idx_struct_id, length_struct_id;
4493 uint32_t color_in_struct_id, color_out_struct_id;
4495 CHECK(n_tokens == 6, EINVAL);
4497 m = metarray_find(p, metarray);
4500 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4502 flength = struct_field_parse(p, action, length, &length_struct_id);
4503 CHECK(flength, EINVAL);
4504 CHECK(!flength->var_size, EINVAL);
4506 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4508 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4509 CHECK(fcout, EINVAL);
4510 CHECK(!fcout->var_size, EINVAL);
4512 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4514 CHECK(!fidx->var_size, EINVAL);
4515 CHECK(!fcin->var_size, EINVAL);
4517 instr->type = INSTR_METER_MMM;
4518 if (idx[0] == 'h' && length[0] == 'h')
4519 instr->type = INSTR_METER_HHM;
4520 if (idx[0] == 'h' && length[0] != 'h')
4521 instr->type = INSTR_METER_HMM;
4522 if (idx[0] != 'h' && length[0] == 'h')
4523 instr->type = INSTR_METER_MHM;
4525 instr->meter.metarray_id = m->id;
4527 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4528 instr->meter.idx.n_bits = fidx->n_bits;
4529 instr->meter.idx.offset = fidx->offset / 8;
4531 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4532 instr->meter.length.n_bits = flength->n_bits;
4533 instr->meter.length.offset = flength->offset / 8;
4535 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4536 instr->meter.color_in.n_bits = fcin->n_bits;
4537 instr->meter.color_in.offset = fcin->offset / 8;
4539 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4540 instr->meter.color_out.n_bits = fcout->n_bits;
4541 instr->meter.color_out.offset = fcout->offset / 8;
4546 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4547 if (fidx && !fcin) {
4548 uint32_t color_in_val;
4550 CHECK(!fidx->var_size, EINVAL);
4552 color_in_val = strtoul(color_in, &color_in, 0);
4553 CHECK(!color_in[0], EINVAL);
4555 instr->type = INSTR_METER_MMI;
4556 if (idx[0] == 'h' && length[0] == 'h')
4557 instr->type = INSTR_METER_HHI;
4558 if (idx[0] == 'h' && length[0] != 'h')
4559 instr->type = INSTR_METER_HMI;
4560 if (idx[0] != 'h' && length[0] == 'h')
4561 instr->type = INSTR_METER_MHI;
4563 instr->meter.metarray_id = m->id;
4565 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4566 instr->meter.idx.n_bits = fidx->n_bits;
4567 instr->meter.idx.offset = fidx->offset / 8;
4569 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4570 instr->meter.length.n_bits = flength->n_bits;
4571 instr->meter.length.offset = flength->offset / 8;
4573 instr->meter.color_in_val = color_in_val;
4575 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4576 instr->meter.color_out.n_bits = fcout->n_bits;
4577 instr->meter.color_out.offset = fcout->offset / 8;
4582 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4583 if (!fidx && fcin) {
4586 idx_val = strtoul(idx, &idx, 0);
4587 CHECK(!idx[0], EINVAL);
4589 CHECK(!fcin->var_size, EINVAL);
4591 instr->type = INSTR_METER_IMM;
4592 if (length[0] == 'h')
4593 instr->type = INSTR_METER_IHM;
4595 instr->meter.metarray_id = m->id;
4597 instr->meter.idx_val = idx_val;
4599 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4600 instr->meter.length.n_bits = flength->n_bits;
4601 instr->meter.length.offset = flength->offset / 8;
4603 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4604 instr->meter.color_in.n_bits = fcin->n_bits;
4605 instr->meter.color_in.offset = fcin->offset / 8;
4607 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4608 instr->meter.color_out.n_bits = fcout->n_bits;
4609 instr->meter.color_out.offset = fcout->offset / 8;
4614 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4615 if (!fidx && !fcin) {
4616 uint32_t idx_val, color_in_val;
4618 idx_val = strtoul(idx, &idx, 0);
4619 CHECK(!idx[0], EINVAL);
4621 color_in_val = strtoul(color_in, &color_in, 0);
4622 CHECK(!color_in[0], EINVAL);
4624 instr->type = INSTR_METER_IMI;
4625 if (length[0] == 'h')
4626 instr->type = INSTR_METER_IHI;
4628 instr->meter.metarray_id = m->id;
4630 instr->meter.idx_val = idx_val;
4632 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4633 instr->meter.length.n_bits = flength->n_bits;
4634 instr->meter.length.offset = flength->offset / 8;
4636 instr->meter.color_in_val = color_in_val;
4638 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4639 instr->meter.color_out.n_bits = fcout->n_bits;
4640 instr->meter.color_out.offset = fcout->offset / 8;
4649 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4651 struct thread *t = &p->threads[p->thread_id];
4652 struct instruction *ip = t->ip;
4655 __instr_metprefetch_h_exec(p, t, ip);
4662 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4664 struct thread *t = &p->threads[p->thread_id];
4665 struct instruction *ip = t->ip;
4668 __instr_metprefetch_m_exec(p, t, ip);
4675 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4677 struct thread *t = &p->threads[p->thread_id];
4678 struct instruction *ip = t->ip;
4681 __instr_metprefetch_i_exec(p, t, ip);
4688 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4690 struct thread *t = &p->threads[p->thread_id];
4691 struct instruction *ip = t->ip;
4694 __instr_meter_hhm_exec(p, t, ip);
4701 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4703 struct thread *t = &p->threads[p->thread_id];
4704 struct instruction *ip = t->ip;
4707 __instr_meter_hhi_exec(p, t, ip);
4714 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4716 struct thread *t = &p->threads[p->thread_id];
4717 struct instruction *ip = t->ip;
4720 __instr_meter_hmm_exec(p, t, ip);
4727 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4729 struct thread *t = &p->threads[p->thread_id];
4730 struct instruction *ip = t->ip;
4733 __instr_meter_hmi_exec(p, t, ip);
4740 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4742 struct thread *t = &p->threads[p->thread_id];
4743 struct instruction *ip = t->ip;
4746 __instr_meter_mhm_exec(p, t, ip);
4753 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4755 struct thread *t = &p->threads[p->thread_id];
4756 struct instruction *ip = t->ip;
4759 __instr_meter_mhi_exec(p, t, ip);
4766 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4768 struct thread *t = &p->threads[p->thread_id];
4769 struct instruction *ip = t->ip;
4772 __instr_meter_mmm_exec(p, t, ip);
4779 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4781 struct thread *t = &p->threads[p->thread_id];
4782 struct instruction *ip = t->ip;
4785 __instr_meter_mmi_exec(p, t, ip);
4792 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4794 struct thread *t = &p->threads[p->thread_id];
4795 struct instruction *ip = t->ip;
4798 __instr_meter_ihm_exec(p, t, ip);
4805 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4807 struct thread *t = &p->threads[p->thread_id];
4808 struct instruction *ip = t->ip;
4811 __instr_meter_ihi_exec(p, t, ip);
4818 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4820 struct thread *t = &p->threads[p->thread_id];
4821 struct instruction *ip = t->ip;
4824 __instr_meter_imm_exec(p, t, ip);
4831 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4833 struct thread *t = &p->threads[p->thread_id];
4834 struct instruction *ip = t->ip;
4837 __instr_meter_imi_exec(p, t, ip);
4847 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4848 struct action *action __rte_unused,
4851 struct instruction *instr,
4852 struct instruction_data *data)
4854 CHECK(n_tokens == 2, EINVAL);
4856 strcpy(data->jmp_label, tokens[1]);
4858 instr->type = INSTR_JMP;
4859 instr->jmp.ip = NULL; /* Resolved later. */
4864 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4865 struct action *action __rte_unused,
4868 struct instruction *instr,
4869 struct instruction_data *data)
4873 CHECK(n_tokens == 3, EINVAL);
4875 strcpy(data->jmp_label, tokens[1]);
4877 h = header_parse(p, tokens[2]);
4880 instr->type = INSTR_JMP_VALID;
4881 instr->jmp.ip = NULL; /* Resolved later. */
4882 instr->jmp.header_id = h->id;
4887 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4888 struct action *action __rte_unused,
4891 struct instruction *instr,
4892 struct instruction_data *data)
4896 CHECK(n_tokens == 3, EINVAL);
4898 strcpy(data->jmp_label, tokens[1]);
4900 h = header_parse(p, tokens[2]);
4903 instr->type = INSTR_JMP_INVALID;
4904 instr->jmp.ip = NULL; /* Resolved later. */
4905 instr->jmp.header_id = h->id;
4910 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4911 struct action *action,
4914 struct instruction *instr,
4915 struct instruction_data *data)
4917 CHECK(!action, EINVAL);
4918 CHECK(n_tokens == 2, EINVAL);
4920 strcpy(data->jmp_label, tokens[1]);
4922 instr->type = INSTR_JMP_HIT;
4923 instr->jmp.ip = NULL; /* Resolved later. */
4928 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4929 struct action *action,
4932 struct instruction *instr,
4933 struct instruction_data *data)
4935 CHECK(!action, EINVAL);
4936 CHECK(n_tokens == 2, EINVAL);
4938 strcpy(data->jmp_label, tokens[1]);
4940 instr->type = INSTR_JMP_MISS;
4941 instr->jmp.ip = NULL; /* Resolved later. */
4946 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4947 struct action *action,
4950 struct instruction *instr,
4951 struct instruction_data *data)
4955 CHECK(!action, EINVAL);
4956 CHECK(n_tokens == 3, EINVAL);
4958 strcpy(data->jmp_label, tokens[1]);
4960 a = action_find(p, tokens[2]);
4963 instr->type = INSTR_JMP_ACTION_HIT;
4964 instr->jmp.ip = NULL; /* Resolved later. */
4965 instr->jmp.action_id = a->id;
4970 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4971 struct action *action,
4974 struct instruction *instr,
4975 struct instruction_data *data)
4979 CHECK(!action, EINVAL);
4980 CHECK(n_tokens == 3, EINVAL);
4982 strcpy(data->jmp_label, tokens[1]);
4984 a = action_find(p, tokens[2]);
4987 instr->type = INSTR_JMP_ACTION_MISS;
4988 instr->jmp.ip = NULL; /* Resolved later. */
4989 instr->jmp.action_id = a->id;
4994 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4995 struct action *action,
4998 struct instruction *instr,
4999 struct instruction_data *data)
5001 char *a = tokens[2], *b = tokens[3];
5002 struct field *fa, *fb;
5004 uint32_t a_struct_id, b_struct_id;
5006 CHECK(n_tokens == 4, EINVAL);
5008 strcpy(data->jmp_label, tokens[1]);
5010 fa = struct_field_parse(p, action, a, &a_struct_id);
5012 CHECK(!fa->var_size, EINVAL);
5014 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5015 fb = struct_field_parse(p, action, b, &b_struct_id);
5017 CHECK(!fb->var_size, EINVAL);
5019 instr->type = INSTR_JMP_EQ;
5020 if (a[0] != 'h' && b[0] == 'h')
5021 instr->type = INSTR_JMP_EQ_MH;
5022 if (a[0] == 'h' && b[0] != 'h')
5023 instr->type = INSTR_JMP_EQ_HM;
5024 if (a[0] == 'h' && b[0] == 'h')
5025 instr->type = INSTR_JMP_EQ_HH;
5026 instr->jmp.ip = NULL; /* Resolved later. */
5028 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5029 instr->jmp.a.n_bits = fa->n_bits;
5030 instr->jmp.a.offset = fa->offset / 8;
5031 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5032 instr->jmp.b.n_bits = fb->n_bits;
5033 instr->jmp.b.offset = fb->offset / 8;
5038 b_val = strtoull(b, &b, 0);
5039 CHECK(!b[0], EINVAL);
5042 b_val = hton64(b_val) >> (64 - fa->n_bits);
5044 instr->type = INSTR_JMP_EQ_I;
5045 instr->jmp.ip = NULL; /* Resolved later. */
5046 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5047 instr->jmp.a.n_bits = fa->n_bits;
5048 instr->jmp.a.offset = fa->offset / 8;
5049 instr->jmp.b_val = b_val;
5054 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5055 struct action *action,
5058 struct instruction *instr,
5059 struct instruction_data *data)
5061 char *a = tokens[2], *b = tokens[3];
5062 struct field *fa, *fb;
5064 uint32_t a_struct_id, b_struct_id;
5066 CHECK(n_tokens == 4, EINVAL);
5068 strcpy(data->jmp_label, tokens[1]);
5070 fa = struct_field_parse(p, action, a, &a_struct_id);
5072 CHECK(!fa->var_size, EINVAL);
5074 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5075 fb = struct_field_parse(p, action, b, &b_struct_id);
5077 CHECK(!fb->var_size, EINVAL);
5079 instr->type = INSTR_JMP_NEQ;
5080 if (a[0] != 'h' && b[0] == 'h')
5081 instr->type = INSTR_JMP_NEQ_MH;
5082 if (a[0] == 'h' && b[0] != 'h')
5083 instr->type = INSTR_JMP_NEQ_HM;
5084 if (a[0] == 'h' && b[0] == 'h')
5085 instr->type = INSTR_JMP_NEQ_HH;
5086 instr->jmp.ip = NULL; /* Resolved later. */
5088 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5089 instr->jmp.a.n_bits = fa->n_bits;
5090 instr->jmp.a.offset = fa->offset / 8;
5091 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5092 instr->jmp.b.n_bits = fb->n_bits;
5093 instr->jmp.b.offset = fb->offset / 8;
5098 b_val = strtoull(b, &b, 0);
5099 CHECK(!b[0], EINVAL);
5102 b_val = hton64(b_val) >> (64 - fa->n_bits);
5104 instr->type = INSTR_JMP_NEQ_I;
5105 instr->jmp.ip = NULL; /* Resolved later. */
5106 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5107 instr->jmp.a.n_bits = fa->n_bits;
5108 instr->jmp.a.offset = fa->offset / 8;
5109 instr->jmp.b_val = b_val;
5114 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5115 struct action *action,
5118 struct instruction *instr,
5119 struct instruction_data *data)
5121 char *a = tokens[2], *b = tokens[3];
5122 struct field *fa, *fb;
5124 uint32_t a_struct_id, b_struct_id;
5126 CHECK(n_tokens == 4, EINVAL);
5128 strcpy(data->jmp_label, tokens[1]);
5130 fa = struct_field_parse(p, action, a, &a_struct_id);
5132 CHECK(!fa->var_size, EINVAL);
5134 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5135 fb = struct_field_parse(p, action, b, &b_struct_id);
5137 CHECK(!fb->var_size, EINVAL);
5139 instr->type = INSTR_JMP_LT;
5140 if (a[0] == 'h' && b[0] != 'h')
5141 instr->type = INSTR_JMP_LT_HM;
5142 if (a[0] != 'h' && b[0] == 'h')
5143 instr->type = INSTR_JMP_LT_MH;
5144 if (a[0] == 'h' && b[0] == 'h')
5145 instr->type = INSTR_JMP_LT_HH;
5146 instr->jmp.ip = NULL; /* Resolved later. */
5148 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5149 instr->jmp.a.n_bits = fa->n_bits;
5150 instr->jmp.a.offset = fa->offset / 8;
5151 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5152 instr->jmp.b.n_bits = fb->n_bits;
5153 instr->jmp.b.offset = fb->offset / 8;
5157 /* JMP_LT_MI, JMP_LT_HI. */
5158 b_val = strtoull(b, &b, 0);
5159 CHECK(!b[0], EINVAL);
5161 instr->type = INSTR_JMP_LT_MI;
5163 instr->type = INSTR_JMP_LT_HI;
5164 instr->jmp.ip = NULL; /* Resolved later. */
5166 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5167 instr->jmp.a.n_bits = fa->n_bits;
5168 instr->jmp.a.offset = fa->offset / 8;
5169 instr->jmp.b_val = b_val;
5174 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5175 struct action *action,
5178 struct instruction *instr,
5179 struct instruction_data *data)
5181 char *a = tokens[2], *b = tokens[3];
5182 struct field *fa, *fb;
5184 uint32_t a_struct_id, b_struct_id;
5186 CHECK(n_tokens == 4, EINVAL);
5188 strcpy(data->jmp_label, tokens[1]);
5190 fa = struct_field_parse(p, action, a, &a_struct_id);
5192 CHECK(!fa->var_size, EINVAL);
5194 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5195 fb = struct_field_parse(p, action, b, &b_struct_id);
5197 CHECK(!fb->var_size, EINVAL);
5199 instr->type = INSTR_JMP_GT;
5200 if (a[0] == 'h' && b[0] != 'h')
5201 instr->type = INSTR_JMP_GT_HM;
5202 if (a[0] != 'h' && b[0] == 'h')
5203 instr->type = INSTR_JMP_GT_MH;
5204 if (a[0] == 'h' && b[0] == 'h')
5205 instr->type = INSTR_JMP_GT_HH;
5206 instr->jmp.ip = NULL; /* Resolved later. */
5208 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5209 instr->jmp.a.n_bits = fa->n_bits;
5210 instr->jmp.a.offset = fa->offset / 8;
5211 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5212 instr->jmp.b.n_bits = fb->n_bits;
5213 instr->jmp.b.offset = fb->offset / 8;
5217 /* JMP_GT_MI, JMP_GT_HI. */
5218 b_val = strtoull(b, &b, 0);
5219 CHECK(!b[0], EINVAL);
5221 instr->type = INSTR_JMP_GT_MI;
5223 instr->type = INSTR_JMP_GT_HI;
5224 instr->jmp.ip = NULL; /* Resolved later. */
5226 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5227 instr->jmp.a.n_bits = fa->n_bits;
5228 instr->jmp.a.offset = fa->offset / 8;
5229 instr->jmp.b_val = b_val;
5234 instr_jmp_exec(struct rte_swx_pipeline *p)
5236 struct thread *t = &p->threads[p->thread_id];
5237 struct instruction *ip = t->ip;
5239 TRACE("[Thread %2u] jmp\n", p->thread_id);
5241 thread_ip_set(t, ip->jmp.ip);
5245 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5247 struct thread *t = &p->threads[p->thread_id];
5248 struct instruction *ip = t->ip;
5249 uint32_t header_id = ip->jmp.header_id;
5251 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5253 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5257 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5259 struct thread *t = &p->threads[p->thread_id];
5260 struct instruction *ip = t->ip;
5261 uint32_t header_id = ip->jmp.header_id;
5263 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5265 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5269 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5271 struct thread *t = &p->threads[p->thread_id];
5272 struct instruction *ip = t->ip;
5273 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5275 TRACE("[Thread %2u] jmph\n", p->thread_id);
5277 t->ip = ip_next[t->hit];
5281 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5283 struct thread *t = &p->threads[p->thread_id];
5284 struct instruction *ip = t->ip;
5285 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5287 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5289 t->ip = ip_next[t->hit];
5293 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5295 struct thread *t = &p->threads[p->thread_id];
5296 struct instruction *ip = t->ip;
5298 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5300 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5304 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5306 struct thread *t = &p->threads[p->thread_id];
5307 struct instruction *ip = t->ip;
5309 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5311 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5315 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5317 struct thread *t = &p->threads[p->thread_id];
5318 struct instruction *ip = t->ip;
5320 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5326 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5328 struct thread *t = &p->threads[p->thread_id];
5329 struct instruction *ip = t->ip;
5331 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5333 JMP_CMP_MH(t, ip, ==);
5337 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5339 struct thread *t = &p->threads[p->thread_id];
5340 struct instruction *ip = t->ip;
5342 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5344 JMP_CMP_HM(t, ip, ==);
5348 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5350 struct thread *t = &p->threads[p->thread_id];
5351 struct instruction *ip = t->ip;
5353 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5355 JMP_CMP_HH_FAST(t, ip, ==);
5359 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5361 struct thread *t = &p->threads[p->thread_id];
5362 struct instruction *ip = t->ip;
5364 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5366 JMP_CMP_I(t, ip, ==);
5370 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5372 struct thread *t = &p->threads[p->thread_id];
5373 struct instruction *ip = t->ip;
5375 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5381 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5383 struct thread *t = &p->threads[p->thread_id];
5384 struct instruction *ip = t->ip;
5386 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5388 JMP_CMP_MH(t, ip, !=);
5392 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5394 struct thread *t = &p->threads[p->thread_id];
5395 struct instruction *ip = t->ip;
5397 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5399 JMP_CMP_HM(t, ip, !=);
5403 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5405 struct thread *t = &p->threads[p->thread_id];
5406 struct instruction *ip = t->ip;
5408 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5410 JMP_CMP_HH_FAST(t, ip, !=);
5414 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5416 struct thread *t = &p->threads[p->thread_id];
5417 struct instruction *ip = t->ip;
5419 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5421 JMP_CMP_I(t, ip, !=);
5425 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5427 struct thread *t = &p->threads[p->thread_id];
5428 struct instruction *ip = t->ip;
5430 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5436 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5438 struct thread *t = &p->threads[p->thread_id];
5439 struct instruction *ip = t->ip;
5441 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5443 JMP_CMP_MH(t, ip, <);
5447 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5449 struct thread *t = &p->threads[p->thread_id];
5450 struct instruction *ip = t->ip;
5452 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5454 JMP_CMP_HM(t, ip, <);
5458 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5460 struct thread *t = &p->threads[p->thread_id];
5461 struct instruction *ip = t->ip;
5463 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5465 JMP_CMP_HH(t, ip, <);
5469 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5471 struct thread *t = &p->threads[p->thread_id];
5472 struct instruction *ip = t->ip;
5474 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5476 JMP_CMP_MI(t, ip, <);
5480 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5482 struct thread *t = &p->threads[p->thread_id];
5483 struct instruction *ip = t->ip;
5485 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5487 JMP_CMP_HI(t, ip, <);
5491 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5493 struct thread *t = &p->threads[p->thread_id];
5494 struct instruction *ip = t->ip;
5496 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5502 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5504 struct thread *t = &p->threads[p->thread_id];
5505 struct instruction *ip = t->ip;
5507 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5509 JMP_CMP_MH(t, ip, >);
5513 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5515 struct thread *t = &p->threads[p->thread_id];
5516 struct instruction *ip = t->ip;
5518 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5520 JMP_CMP_HM(t, ip, >);
5524 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5526 struct thread *t = &p->threads[p->thread_id];
5527 struct instruction *ip = t->ip;
5529 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5531 JMP_CMP_HH(t, ip, >);
5535 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5537 struct thread *t = &p->threads[p->thread_id];
5538 struct instruction *ip = t->ip;
5540 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5542 JMP_CMP_MI(t, ip, >);
5546 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5548 struct thread *t = &p->threads[p->thread_id];
5549 struct instruction *ip = t->ip;
5551 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5553 JMP_CMP_HI(t, ip, >);
5560 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5561 struct action *action,
5562 char **tokens __rte_unused,
5564 struct instruction *instr,
5565 struct instruction_data *data __rte_unused)
5567 CHECK(action, EINVAL);
5568 CHECK(n_tokens == 1, EINVAL);
5570 instr->type = INSTR_RETURN;
5575 instr_return_exec(struct rte_swx_pipeline *p)
5577 struct thread *t = &p->threads[p->thread_id];
5579 TRACE("[Thread %2u] return\n", p->thread_id);
5585 instr_translate(struct rte_swx_pipeline *p,
5586 struct action *action,
5588 struct instruction *instr,
5589 struct instruction_data *data)
5591 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5592 int n_tokens = 0, tpos = 0;
5594 /* Parse the instruction string into tokens. */
5598 token = strtok_r(string, " \t\v", &string);
5602 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5603 CHECK_NAME(token, EINVAL);
5605 tokens[n_tokens] = token;
5609 CHECK(n_tokens, EINVAL);
5611 /* Handle the optional instruction label. */
5612 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5613 strcpy(data->label, tokens[0]);
5616 CHECK(n_tokens - tpos, EINVAL);
5619 /* Identify the instruction type. */
5620 if (!strcmp(tokens[tpos], "rx"))
5621 return instr_rx_translate(p,
5628 if (!strcmp(tokens[tpos], "tx"))
5629 return instr_tx_translate(p,
5636 if (!strcmp(tokens[tpos], "drop"))
5637 return instr_drop_translate(p,
5644 if (!strcmp(tokens[tpos], "extract"))
5645 return instr_hdr_extract_translate(p,
5652 if (!strcmp(tokens[tpos], "lookahead"))
5653 return instr_hdr_lookahead_translate(p,
5660 if (!strcmp(tokens[tpos], "emit"))
5661 return instr_hdr_emit_translate(p,
5668 if (!strcmp(tokens[tpos], "validate"))
5669 return instr_hdr_validate_translate(p,
5676 if (!strcmp(tokens[tpos], "invalidate"))
5677 return instr_hdr_invalidate_translate(p,
5684 if (!strcmp(tokens[tpos], "mov"))
5685 return instr_mov_translate(p,
5692 if (!strcmp(tokens[tpos], "add"))
5693 return instr_alu_add_translate(p,
5700 if (!strcmp(tokens[tpos], "sub"))
5701 return instr_alu_sub_translate(p,
5708 if (!strcmp(tokens[tpos], "ckadd"))
5709 return instr_alu_ckadd_translate(p,
5716 if (!strcmp(tokens[tpos], "cksub"))
5717 return instr_alu_cksub_translate(p,
5724 if (!strcmp(tokens[tpos], "and"))
5725 return instr_alu_and_translate(p,
5732 if (!strcmp(tokens[tpos], "or"))
5733 return instr_alu_or_translate(p,
5740 if (!strcmp(tokens[tpos], "xor"))
5741 return instr_alu_xor_translate(p,
5748 if (!strcmp(tokens[tpos], "shl"))
5749 return instr_alu_shl_translate(p,
5756 if (!strcmp(tokens[tpos], "shr"))
5757 return instr_alu_shr_translate(p,
5764 if (!strcmp(tokens[tpos], "regprefetch"))
5765 return instr_regprefetch_translate(p,
5772 if (!strcmp(tokens[tpos], "regrd"))
5773 return instr_regrd_translate(p,
5780 if (!strcmp(tokens[tpos], "regwr"))
5781 return instr_regwr_translate(p,
5788 if (!strcmp(tokens[tpos], "regadd"))
5789 return instr_regadd_translate(p,
5796 if (!strcmp(tokens[tpos], "metprefetch"))
5797 return instr_metprefetch_translate(p,
5804 if (!strcmp(tokens[tpos], "meter"))
5805 return instr_meter_translate(p,
5812 if (!strcmp(tokens[tpos], "table"))
5813 return instr_table_translate(p,
5820 if (!strcmp(tokens[tpos], "learn"))
5821 return instr_learn_translate(p,
5828 if (!strcmp(tokens[tpos], "forget"))
5829 return instr_forget_translate(p,
5836 if (!strcmp(tokens[tpos], "extern"))
5837 return instr_extern_translate(p,
5844 if (!strcmp(tokens[tpos], "jmp"))
5845 return instr_jmp_translate(p,
5852 if (!strcmp(tokens[tpos], "jmpv"))
5853 return instr_jmp_valid_translate(p,
5860 if (!strcmp(tokens[tpos], "jmpnv"))
5861 return instr_jmp_invalid_translate(p,
5868 if (!strcmp(tokens[tpos], "jmph"))
5869 return instr_jmp_hit_translate(p,
5876 if (!strcmp(tokens[tpos], "jmpnh"))
5877 return instr_jmp_miss_translate(p,
5884 if (!strcmp(tokens[tpos], "jmpa"))
5885 return instr_jmp_action_hit_translate(p,
5892 if (!strcmp(tokens[tpos], "jmpna"))
5893 return instr_jmp_action_miss_translate(p,
5900 if (!strcmp(tokens[tpos], "jmpeq"))
5901 return instr_jmp_eq_translate(p,
5908 if (!strcmp(tokens[tpos], "jmpneq"))
5909 return instr_jmp_neq_translate(p,
5916 if (!strcmp(tokens[tpos], "jmplt"))
5917 return instr_jmp_lt_translate(p,
5924 if (!strcmp(tokens[tpos], "jmpgt"))
5925 return instr_jmp_gt_translate(p,
5932 if (!strcmp(tokens[tpos], "return"))
5933 return instr_return_translate(p,
5943 static struct instruction_data *
5944 label_find(struct instruction_data *data, uint32_t n, const char *label)
5948 for (i = 0; i < n; i++)
5949 if (!strcmp(label, data[i].label))
5956 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5958 uint32_t count = 0, i;
5963 for (i = 0; i < n; i++)
5964 if (!strcmp(label, data[i].jmp_label))
5971 instr_label_check(struct instruction_data *instruction_data,
5972 uint32_t n_instructions)
5976 /* Check that all instruction labels are unique. */
5977 for (i = 0; i < n_instructions; i++) {
5978 struct instruction_data *data = &instruction_data[i];
5979 char *label = data->label;
5985 for (j = i + 1; j < n_instructions; j++)
5986 CHECK(strcmp(label, data[j].label), EINVAL);
5989 /* Get users for each instruction label. */
5990 for (i = 0; i < n_instructions; i++) {
5991 struct instruction_data *data = &instruction_data[i];
5992 char *label = data->label;
5994 data->n_users = label_is_used(instruction_data,
6003 instr_jmp_resolve(struct instruction *instructions,
6004 struct instruction_data *instruction_data,
6005 uint32_t n_instructions)
6009 for (i = 0; i < n_instructions; i++) {
6010 struct instruction *instr = &instructions[i];
6011 struct instruction_data *data = &instruction_data[i];
6012 struct instruction_data *found;
6014 if (!instruction_is_jmp(instr))
6017 found = label_find(instruction_data,
6020 CHECK(found, EINVAL);
6022 instr->jmp.ip = &instructions[found - instruction_data];
6029 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6031 struct instruction *instr,
6032 struct instruction_data *data __rte_unused,
6033 uint32_t n_instructions)
6036 enum instruction_type type;
6039 /* Check that the first instruction is rx. */
6040 CHECK(instr[0].type == INSTR_RX, EINVAL);
6042 /* Check that there is at least one tx instruction. */
6043 for (i = 0; i < n_instructions; i++) {
6044 type = instr[i].type;
6046 if (instruction_is_tx(type))
6049 CHECK(i < n_instructions, EINVAL);
6051 /* Check that the last instruction is either tx or unconditional
6054 type = instr[n_instructions - 1].type;
6055 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6059 enum instruction_type type;
6062 /* Check that there is at least one return or tx instruction. */
6063 for (i = 0; i < n_instructions; i++) {
6064 type = instr[i].type;
6066 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6069 CHECK(i < n_instructions, EINVAL);
6076 instr_compact(struct instruction *instructions,
6077 struct instruction_data *instruction_data,
6078 uint32_t n_instructions)
6080 uint32_t i, pos = 0;
6082 /* Eliminate the invalid instructions that have been optimized out. */
6083 for (i = 0; i < n_instructions; i++) {
6084 struct instruction *instr = &instructions[i];
6085 struct instruction_data *data = &instruction_data[i];
6091 memcpy(&instructions[pos], instr, sizeof(*instr));
6092 memcpy(&instruction_data[pos], data, sizeof(*data));
6102 instr_pattern_extract_many_search(struct instruction *instr,
6103 struct instruction_data *data,
6105 uint32_t *n_pattern_instr)
6109 for (i = 0; i < n_instr; i++) {
6110 if (data[i].invalid)
6113 if (instr[i].type != INSTR_HDR_EXTRACT)
6116 if (i == RTE_DIM(instr->io.hdr.header_id))
6119 if (i && data[i].n_users)
6126 *n_pattern_instr = i;
6131 instr_pattern_extract_many_replace(struct instruction *instr,
6132 struct instruction_data *data,
6137 for (i = 1; i < n_instr; i++) {
6139 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6140 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6141 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6143 data[i].invalid = 1;
6148 instr_pattern_extract_many_optimize(struct instruction *instructions,
6149 struct instruction_data *instruction_data,
6150 uint32_t n_instructions)
6154 for (i = 0; i < n_instructions; ) {
6155 struct instruction *instr = &instructions[i];
6156 struct instruction_data *data = &instruction_data[i];
6157 uint32_t n_instr = 0;
6161 detected = instr_pattern_extract_many_search(instr,
6166 instr_pattern_extract_many_replace(instr,
6173 /* No pattern starting at the current instruction. */
6177 /* Eliminate the invalid instructions that have been optimized out. */
6178 n_instructions = instr_compact(instructions,
6182 return n_instructions;
6186 instr_pattern_emit_many_tx_search(struct instruction *instr,
6187 struct instruction_data *data,
6189 uint32_t *n_pattern_instr)
6193 for (i = 0; i < n_instr; i++) {
6194 if (data[i].invalid)
6197 if (instr[i].type != INSTR_HDR_EMIT)
6200 if (i == RTE_DIM(instr->io.hdr.header_id))
6203 if (i && data[i].n_users)
6210 if (!instruction_is_tx(instr[i].type))
6213 if (data[i].n_users)
6218 *n_pattern_instr = i;
6223 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6224 struct instruction_data *data,
6229 /* Any emit instruction in addition to the first one. */
6230 for (i = 1; i < n_instr - 1; i++) {
6232 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6233 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6234 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6236 data[i].invalid = 1;
6239 /* The TX instruction is the last one in the pattern. */
6241 instr[0].io.io.offset = instr[i].io.io.offset;
6242 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6243 data[i].invalid = 1;
6247 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6248 struct instruction_data *instruction_data,
6249 uint32_t n_instructions)
6253 for (i = 0; i < n_instructions; ) {
6254 struct instruction *instr = &instructions[i];
6255 struct instruction_data *data = &instruction_data[i];
6256 uint32_t n_instr = 0;
6259 /* Emit many + TX. */
6260 detected = instr_pattern_emit_many_tx_search(instr,
6265 instr_pattern_emit_many_tx_replace(instr,
6272 /* No pattern starting at the current instruction. */
6276 /* Eliminate the invalid instructions that have been optimized out. */
6277 n_instructions = instr_compact(instructions,
6281 return n_instructions;
6285 action_arg_src_mov_count(struct action *a,
6287 struct instruction *instructions,
6288 struct instruction_data *instruction_data,
6289 uint32_t n_instructions);
6292 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6294 struct instruction *instr,
6295 struct instruction_data *data,
6297 struct instruction *instructions,
6298 struct instruction_data *instruction_data,
6299 uint32_t n_instructions,
6300 uint32_t *n_pattern_instr)
6303 uint32_t src_field_id, i, j;
6305 /* Prerequisites. */
6309 /* First instruction: MOV_HM. */
6310 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6313 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6314 if (!h || h->st->var_size)
6317 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6318 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6321 if (src_field_id == a->st->n_fields)
6324 if (instr[0].mov.dst.offset ||
6325 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6326 instr[0].mov.src.struct_id ||
6327 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6328 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6331 if ((n_instr < h->st->n_fields + 1) ||
6332 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6335 /* Subsequent instructions: MOV_HM. */
6336 for (i = 1; i < h->st->n_fields; i++)
6337 if (data[i].invalid ||
6339 (instr[i].type != INSTR_MOV_HM) ||
6340 (instr[i].mov.dst.struct_id != h->struct_id) ||
6341 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6342 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6343 instr[i].mov.src.struct_id ||
6344 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6345 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6346 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6349 /* Last instruction: HDR_VALIDATE. */
6350 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6351 (instr[i].valid.header_id != h->id))
6354 /* Check that none of the action args that are used as source for this
6355 * DMA transfer are not used as source in any other mov instruction.
6357 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6360 n_users = action_arg_src_mov_count(a,
6369 *n_pattern_instr = 1 + i;
6374 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6376 struct instruction *instr,
6377 struct instruction_data *data,
6381 uint32_t src_field_id, src_offset, i;
6383 /* Read from the instructions before they are modified. */
6384 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6388 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6389 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6392 if (src_field_id == a->st->n_fields)
6395 src_offset = instr[0].mov.src.offset;
6397 /* Modify the instructions. */
6398 instr[0].type = INSTR_DMA_HT;
6399 instr[0].dma.dst.header_id[0] = h->id;
6400 instr[0].dma.dst.struct_id[0] = h->struct_id;
6401 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6402 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6404 for (i = 1; i < n_instr; i++)
6405 data[i].invalid = 1;
6407 /* Update the endianness of the action arguments to header endianness. */
6408 for (i = 0; i < h->st->n_fields; i++)
6409 a->args_endianness[src_field_id + i] = 1;
6413 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6415 struct instruction *instructions,
6416 struct instruction_data *instruction_data,
6417 uint32_t n_instructions)
6422 return n_instructions;
6424 for (i = 0; i < n_instructions; ) {
6425 struct instruction *instr = &instructions[i];
6426 struct instruction_data *data = &instruction_data[i];
6427 uint32_t n_instr = 0;
6430 /* Mov all + validate. */
6431 detected = instr_pattern_mov_all_validate_search(p,
6441 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6446 /* No pattern starting at the current instruction. */
6450 /* Eliminate the invalid instructions that have been optimized out. */
6451 n_instructions = instr_compact(instructions,
6455 return n_instructions;
6459 instr_pattern_dma_many_search(struct instruction *instr,
6460 struct instruction_data *data,
6462 uint32_t *n_pattern_instr)
6466 for (i = 0; i < n_instr; i++) {
6467 if (data[i].invalid)
6470 if (instr[i].type != INSTR_DMA_HT)
6473 if (i == RTE_DIM(instr->dma.dst.header_id))
6476 if (i && data[i].n_users)
6483 *n_pattern_instr = i;
6488 instr_pattern_dma_many_replace(struct instruction *instr,
6489 struct instruction_data *data,
6494 for (i = 1; i < n_instr; i++) {
6496 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6497 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6498 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6499 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6501 data[i].invalid = 1;
6506 instr_pattern_dma_many_optimize(struct instruction *instructions,
6507 struct instruction_data *instruction_data,
6508 uint32_t n_instructions)
6512 for (i = 0; i < n_instructions; ) {
6513 struct instruction *instr = &instructions[i];
6514 struct instruction_data *data = &instruction_data[i];
6515 uint32_t n_instr = 0;
6519 detected = instr_pattern_dma_many_search(instr,
6524 instr_pattern_dma_many_replace(instr, data, n_instr);
6529 /* No pattern starting at the current instruction. */
6533 /* Eliminate the invalid instructions that have been optimized out. */
6534 n_instructions = instr_compact(instructions,
6538 return n_instructions;
6542 instr_optimize(struct rte_swx_pipeline *p,
6544 struct instruction *instructions,
6545 struct instruction_data *instruction_data,
6546 uint32_t n_instructions)
6549 n_instructions = instr_pattern_extract_many_optimize(instructions,
6553 /* Emit many + TX. */
6554 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6558 /* Mov all + validate. */
6559 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6566 n_instructions = instr_pattern_dma_many_optimize(instructions,
6570 return n_instructions;
6574 instruction_config(struct rte_swx_pipeline *p,
6576 const char **instructions,
6577 uint32_t n_instructions)
6579 struct instruction *instr = NULL;
6580 struct instruction_data *data = NULL;
6584 CHECK(n_instructions, EINVAL);
6585 CHECK(instructions, EINVAL);
6586 for (i = 0; i < n_instructions; i++)
6587 CHECK_INSTRUCTION(instructions[i], EINVAL);
6589 /* Memory allocation. */
6590 instr = calloc(n_instructions, sizeof(struct instruction));
6596 data = calloc(n_instructions, sizeof(struct instruction_data));
6602 for (i = 0; i < n_instructions; i++) {
6603 char *string = strdup(instructions[i]);
6609 err = instr_translate(p, a, string, &instr[i], &data[i]);
6618 err = instr_label_check(data, n_instructions);
6622 err = instr_verify(p, a, instr, data, n_instructions);
6626 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6628 err = instr_jmp_resolve(instr, data, n_instructions);
6633 a->instructions = instr;
6634 a->instruction_data = data;
6635 a->n_instructions = n_instructions;
6637 p->instructions = instr;
6638 p->instruction_data = data;
6639 p->n_instructions = n_instructions;
6650 static instr_exec_t instruction_table[] = {
6651 [INSTR_RX] = instr_rx_exec,
6652 [INSTR_TX] = instr_tx_exec,
6653 [INSTR_TX_I] = instr_tx_i_exec,
6655 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6656 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6657 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6658 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6659 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6660 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6661 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6662 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6663 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6664 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6666 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6667 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6668 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6669 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6670 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6671 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6672 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6673 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6674 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6676 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6677 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6679 [INSTR_MOV] = instr_mov_exec,
6680 [INSTR_MOV_MH] = instr_mov_mh_exec,
6681 [INSTR_MOV_HM] = instr_mov_hm_exec,
6682 [INSTR_MOV_HH] = instr_mov_hh_exec,
6683 [INSTR_MOV_I] = instr_mov_i_exec,
6685 [INSTR_DMA_HT] = instr_dma_ht_exec,
6686 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6687 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6688 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6689 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6690 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6691 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6692 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6694 [INSTR_ALU_ADD] = instr_alu_add_exec,
6695 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6696 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6697 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6698 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6699 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6701 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6702 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6703 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6704 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6705 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6706 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6708 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6709 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6710 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6711 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6713 [INSTR_ALU_AND] = instr_alu_and_exec,
6714 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6715 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6716 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6717 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6719 [INSTR_ALU_OR] = instr_alu_or_exec,
6720 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6721 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6722 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6723 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6725 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6726 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6727 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6728 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6729 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6731 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6732 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6733 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6734 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6735 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6736 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6738 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6739 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6740 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6741 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6742 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6743 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6745 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6746 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6747 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6749 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6750 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6751 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6752 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6753 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6754 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6756 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6757 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6758 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6759 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6760 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6761 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6762 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6763 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6764 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6766 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6767 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6768 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6769 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6770 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6771 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6772 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6773 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6774 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6776 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6777 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6778 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6780 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6781 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6782 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6783 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6784 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6785 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6786 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6787 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6788 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6789 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6790 [INSTR_METER_IMM] = instr_meter_imm_exec,
6791 [INSTR_METER_IMI] = instr_meter_imi_exec,
6793 [INSTR_TABLE] = instr_table_exec,
6794 [INSTR_TABLE_AF] = instr_table_af_exec,
6795 [INSTR_SELECTOR] = instr_selector_exec,
6796 [INSTR_LEARNER] = instr_learner_exec,
6797 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6798 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6799 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6800 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6801 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6803 [INSTR_JMP] = instr_jmp_exec,
6804 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6805 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6806 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6807 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6808 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6809 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6811 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6812 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6813 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6814 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6815 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6817 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6818 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6819 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6820 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6821 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6823 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6824 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6825 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6826 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6827 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6828 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6830 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6831 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6832 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6833 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6834 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6835 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6837 [INSTR_RETURN] = instr_return_exec,
6841 instruction_table_build(struct rte_swx_pipeline *p)
6843 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6844 sizeof(struct instr_exec_t *));
6845 if (!p->instruction_table)
6848 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6854 instruction_table_build_free(struct rte_swx_pipeline *p)
6856 if (!p->instruction_table)
6859 free(p->instruction_table);
6860 p->instruction_table = NULL;
6864 instruction_table_free(struct rte_swx_pipeline *p)
6866 instruction_table_build_free(p);
6870 instr_exec(struct rte_swx_pipeline *p)
6872 struct thread *t = &p->threads[p->thread_id];
6873 struct instruction *ip = t->ip;
6874 instr_exec_t instr = p->instruction_table[ip->type];
6882 static struct action *
6883 action_find(struct rte_swx_pipeline *p, const char *name)
6885 struct action *elem;
6890 TAILQ_FOREACH(elem, &p->actions, node)
6891 if (strcmp(elem->name, name) == 0)
6897 static struct action *
6898 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6900 struct action *action = NULL;
6902 TAILQ_FOREACH(action, &p->actions, node)
6903 if (action->id == id)
6909 static struct field *
6910 action_field_find(struct action *a, const char *name)
6912 return a->st ? struct_type_field_find(a->st, name) : NULL;
6915 static struct field *
6916 action_field_parse(struct action *action, const char *name)
6918 if (name[0] != 't' || name[1] != '.')
6921 return action_field_find(action, &name[2]);
6925 action_has_nbo_args(struct action *a)
6929 /* Return if the action does not have any args. */
6931 return 0; /* FALSE */
6933 for (i = 0; i < a->st->n_fields; i++)
6934 if (a->args_endianness[i])
6935 return 1; /* TRUE */
6937 return 0; /* FALSE */
6941 action_does_learning(struct action *a)
6945 for (i = 0; i < a->n_instructions; i++)
6946 switch (a->instructions[i].type) {
6947 case INSTR_LEARNER_LEARN:
6948 return 1; /* TRUE */
6950 case INSTR_LEARNER_FORGET:
6951 return 1; /* TRUE */
6957 return 0; /* FALSE */
6961 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6963 const char *args_struct_type_name,
6964 const char **instructions,
6965 uint32_t n_instructions)
6967 struct struct_type *args_struct_type = NULL;
6973 CHECK_NAME(name, EINVAL);
6974 CHECK(!action_find(p, name), EEXIST);
6976 if (args_struct_type_name) {
6977 CHECK_NAME(args_struct_type_name, EINVAL);
6978 args_struct_type = struct_type_find(p, args_struct_type_name);
6979 CHECK(args_struct_type, EINVAL);
6980 CHECK(!args_struct_type->var_size, EINVAL);
6983 /* Node allocation. */
6984 a = calloc(1, sizeof(struct action));
6986 if (args_struct_type) {
6987 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6988 if (!a->args_endianness) {
6994 /* Node initialization. */
6995 strcpy(a->name, name);
6996 a->st = args_struct_type;
6997 a->id = p->n_actions;
6999 /* Instruction translation. */
7000 err = instruction_config(p, a, instructions, n_instructions);
7002 free(a->args_endianness);
7007 /* Node add to tailq. */
7008 TAILQ_INSERT_TAIL(&p->actions, a, node);
7015 action_build(struct rte_swx_pipeline *p)
7017 struct action *action;
7019 /* p->action_instructions. */
7020 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7021 CHECK(p->action_instructions, ENOMEM);
7023 TAILQ_FOREACH(action, &p->actions, node)
7024 p->action_instructions[action->id] = action->instructions;
7026 /* p->action_funcs. */
7027 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7028 CHECK(p->action_funcs, ENOMEM);
7034 action_build_free(struct rte_swx_pipeline *p)
7036 free(p->action_funcs);
7037 p->action_funcs = NULL;
7039 free(p->action_instructions);
7040 p->action_instructions = NULL;
7044 action_free(struct rte_swx_pipeline *p)
7046 action_build_free(p);
7049 struct action *action;
7051 action = TAILQ_FIRST(&p->actions);
7055 TAILQ_REMOVE(&p->actions, action, node);
7056 free(action->instruction_data);
7057 free(action->instructions);
7063 action_arg_src_mov_count(struct action *a,
7065 struct instruction *instructions,
7066 struct instruction_data *instruction_data,
7067 uint32_t n_instructions)
7069 uint32_t offset, n_users = 0, i;
7072 (arg_id >= a->st->n_fields) ||
7074 !instruction_data ||
7078 offset = a->st->fields[arg_id].offset / 8;
7080 for (i = 0; i < n_instructions; i++) {
7081 struct instruction *instr = &instructions[i];
7082 struct instruction_data *data = &instruction_data[i];
7084 if (data->invalid ||
7085 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7086 instr->mov.src.struct_id ||
7087 (instr->mov.src.offset != offset))
7099 static struct table_type *
7100 table_type_find(struct rte_swx_pipeline *p, const char *name)
7102 struct table_type *elem;
7104 TAILQ_FOREACH(elem, &p->table_types, node)
7105 if (strcmp(elem->name, name) == 0)
7111 static struct table_type *
7112 table_type_resolve(struct rte_swx_pipeline *p,
7113 const char *recommended_type_name,
7114 enum rte_swx_table_match_type match_type)
7116 struct table_type *elem;
7118 /* Only consider the recommended type if the match type is correct. */
7119 if (recommended_type_name)
7120 TAILQ_FOREACH(elem, &p->table_types, node)
7121 if (!strcmp(elem->name, recommended_type_name) &&
7122 (elem->match_type == match_type))
7125 /* Ignore the recommended type and get the first element with this match
7128 TAILQ_FOREACH(elem, &p->table_types, node)
7129 if (elem->match_type == match_type)
7135 static struct table *
7136 table_find(struct rte_swx_pipeline *p, const char *name)
7140 TAILQ_FOREACH(elem, &p->tables, node)
7141 if (strcmp(elem->name, name) == 0)
7147 static struct table *
7148 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7150 struct table *table = NULL;
7152 TAILQ_FOREACH(table, &p->tables, node)
7153 if (table->id == id)
7160 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7162 enum rte_swx_table_match_type match_type,
7163 struct rte_swx_table_ops *ops)
7165 struct table_type *elem;
7169 CHECK_NAME(name, EINVAL);
7170 CHECK(!table_type_find(p, name), EEXIST);
7173 CHECK(ops->create, EINVAL);
7174 CHECK(ops->lkp, EINVAL);
7175 CHECK(ops->free, EINVAL);
7177 /* Node allocation. */
7178 elem = calloc(1, sizeof(struct table_type));
7179 CHECK(elem, ENOMEM);
7181 /* Node initialization. */
7182 strcpy(elem->name, name);
7183 elem->match_type = match_type;
7184 memcpy(&elem->ops, ops, sizeof(*ops));
7186 /* Node add to tailq. */
7187 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7193 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7195 enum rte_swx_table_match_type *match_type)
7197 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7199 for (i = 0; i < n_fields; i++) {
7200 struct rte_swx_match_field_params *f = &fields[i];
7202 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7205 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7209 if ((n_fields_lpm > 1) ||
7210 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7213 *match_type = (n_fields_em == n_fields) ?
7214 RTE_SWX_TABLE_MATCH_EXACT :
7215 RTE_SWX_TABLE_MATCH_WILDCARD;
7221 table_match_fields_check(struct rte_swx_pipeline *p,
7222 struct rte_swx_pipeline_table_params *params,
7223 struct header **header)
7225 struct header *h0 = NULL;
7226 struct field *hf, *mf;
7227 uint32_t *offset = NULL, i;
7230 /* Return if no match fields. */
7231 if (!params->n_fields) {
7232 if (params->fields) {
7243 /* Memory allocation. */
7244 offset = calloc(params->n_fields, sizeof(uint32_t));
7250 /* Check that all the match fields belong to either the same header or
7253 hf = header_field_parse(p, params->fields[0].name, &h0);
7254 mf = metadata_field_parse(p, params->fields[0].name);
7255 if ((!hf && !mf) || (hf && hf->var_size)) {
7260 offset[0] = h0 ? hf->offset : mf->offset;
7262 for (i = 1; i < params->n_fields; i++)
7266 hf = header_field_parse(p, params->fields[i].name, &h);
7267 if (!hf || (h->id != h0->id) || hf->var_size) {
7272 offset[i] = hf->offset;
7274 mf = metadata_field_parse(p, params->fields[i].name);
7280 offset[i] = mf->offset;
7283 /* Check that there are no duplicated match fields. */
7284 for (i = 0; i < params->n_fields; i++) {
7287 for (j = 0; j < i; j++)
7288 if (offset[j] == offset[i]) {
7304 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7306 struct rte_swx_pipeline_table_params *params,
7307 const char *recommended_table_type_name,
7311 struct table_type *type;
7313 struct action *default_action;
7314 struct header *header = NULL;
7315 uint32_t action_data_size_max = 0, i;
7320 CHECK_NAME(name, EINVAL);
7321 CHECK(!table_find(p, name), EEXIST);
7322 CHECK(!selector_find(p, name), EEXIST);
7323 CHECK(!learner_find(p, name), EEXIST);
7325 CHECK(params, EINVAL);
7328 status = table_match_fields_check(p, params, &header);
7332 /* Action checks. */
7333 CHECK(params->n_actions, EINVAL);
7334 CHECK(params->action_names, EINVAL);
7335 for (i = 0; i < params->n_actions; i++) {
7336 const char *action_name = params->action_names[i];
7338 uint32_t action_data_size;
7340 CHECK_NAME(action_name, EINVAL);
7342 a = action_find(p, action_name);
7344 CHECK(!action_does_learning(a), EINVAL);
7346 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7347 if (action_data_size > action_data_size_max)
7348 action_data_size_max = action_data_size;
7351 CHECK_NAME(params->default_action_name, EINVAL);
7352 for (i = 0; i < p->n_actions; i++)
7353 if (!strcmp(params->action_names[i],
7354 params->default_action_name))
7356 CHECK(i < params->n_actions, EINVAL);
7357 default_action = action_find(p, params->default_action_name);
7358 CHECK((default_action->st && params->default_action_data) ||
7359 !params->default_action_data, EINVAL);
7361 /* Table type checks. */
7362 if (recommended_table_type_name)
7363 CHECK_NAME(recommended_table_type_name, EINVAL);
7365 if (params->n_fields) {
7366 enum rte_swx_table_match_type match_type;
7368 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7372 type = table_type_resolve(p, recommended_table_type_name, match_type);
7373 CHECK(type, EINVAL);
7378 /* Memory allocation. */
7379 t = calloc(1, sizeof(struct table));
7382 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7388 t->actions = calloc(params->n_actions, sizeof(struct action *));
7395 if (action_data_size_max) {
7396 t->default_action_data = calloc(1, action_data_size_max);
7397 if (!t->default_action_data) {
7405 /* Node initialization. */
7406 strcpy(t->name, name);
7407 if (args && args[0])
7408 strcpy(t->args, args);
7411 for (i = 0; i < params->n_fields; i++) {
7412 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7413 struct match_field *f = &t->fields[i];
7415 f->match_type = field->match_type;
7417 header_field_parse(p, field->name, NULL) :
7418 metadata_field_parse(p, field->name);
7420 t->n_fields = params->n_fields;
7423 for (i = 0; i < params->n_actions; i++)
7424 t->actions[i] = action_find(p, params->action_names[i]);
7425 t->default_action = default_action;
7426 if (default_action->st)
7427 memcpy(t->default_action_data,
7428 params->default_action_data,
7429 default_action->st->n_bits / 8);
7430 t->n_actions = params->n_actions;
7431 t->default_action_is_const = params->default_action_is_const;
7432 t->action_data_size_max = action_data_size_max;
7435 t->id = p->n_tables;
7437 /* Node add to tailq. */
7438 TAILQ_INSERT_TAIL(&p->tables, t, node);
7444 static struct rte_swx_table_params *
7445 table_params_get(struct table *table)
7447 struct rte_swx_table_params *params;
7448 struct field *first, *last;
7450 uint32_t key_size, key_offset, action_data_size, i;
7452 /* Memory allocation. */
7453 params = calloc(1, sizeof(struct rte_swx_table_params));
7457 /* Find first (smallest offset) and last (biggest offset) match fields. */
7458 first = table->fields[0].field;
7459 last = table->fields[0].field;
7461 for (i = 0; i < table->n_fields; i++) {
7462 struct field *f = table->fields[i].field;
7464 if (f->offset < first->offset)
7467 if (f->offset > last->offset)
7471 /* Key offset and size. */
7472 key_offset = first->offset / 8;
7473 key_size = (last->offset + last->n_bits - first->offset) / 8;
7475 /* Memory allocation. */
7476 key_mask = calloc(1, key_size);
7483 for (i = 0; i < table->n_fields; i++) {
7484 struct field *f = table->fields[i].field;
7485 uint32_t start = (f->offset - first->offset) / 8;
7486 size_t size = f->n_bits / 8;
7488 memset(&key_mask[start], 0xFF, size);
7491 /* Action data size. */
7492 action_data_size = 0;
7493 for (i = 0; i < table->n_actions; i++) {
7494 struct action *action = table->actions[i];
7495 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7497 if (ads > action_data_size)
7498 action_data_size = ads;
7502 params->match_type = table->type->match_type;
7503 params->key_size = key_size;
7504 params->key_offset = key_offset;
7505 params->key_mask0 = key_mask;
7506 params->action_data_size = action_data_size;
7507 params->n_keys_max = table->size;
7513 table_params_free(struct rte_swx_table_params *params)
7518 free(params->key_mask0);
7523 table_stub_lkp(void *table __rte_unused,
7524 void *mailbox __rte_unused,
7525 uint8_t **key __rte_unused,
7526 uint64_t *action_id __rte_unused,
7527 uint8_t **action_data __rte_unused,
7531 return 1; /* DONE. */
7535 table_build(struct rte_swx_pipeline *p)
7539 /* Per pipeline: table statistics. */
7540 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7541 CHECK(p->table_stats, ENOMEM);
7543 for (i = 0; i < p->n_tables; i++) {
7544 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7545 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7548 /* Per thread: table runt-time. */
7549 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7550 struct thread *t = &p->threads[i];
7551 struct table *table;
7553 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7554 CHECK(t->tables, ENOMEM);
7556 TAILQ_FOREACH(table, &p->tables, node) {
7557 struct table_runtime *r = &t->tables[table->id];
7562 size = table->type->ops.mailbox_size_get();
7565 r->func = table->type->ops.lkp;
7569 r->mailbox = calloc(1, size);
7570 CHECK(r->mailbox, ENOMEM);
7574 r->key = table->header ?
7575 &t->structs[table->header->struct_id] :
7576 &t->structs[p->metadata_struct_id];
7578 r->func = table_stub_lkp;
7587 table_build_free(struct rte_swx_pipeline *p)
7591 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7592 struct thread *t = &p->threads[i];
7598 for (j = 0; j < p->n_tables; j++) {
7599 struct table_runtime *r = &t->tables[j];
7608 if (p->table_stats) {
7609 for (i = 0; i < p->n_tables; i++)
7610 free(p->table_stats[i].n_pkts_action);
7612 free(p->table_stats);
7617 table_free(struct rte_swx_pipeline *p)
7619 table_build_free(p);
7625 elem = TAILQ_FIRST(&p->tables);
7629 TAILQ_REMOVE(&p->tables, elem, node);
7631 free(elem->actions);
7632 free(elem->default_action_data);
7638 struct table_type *elem;
7640 elem = TAILQ_FIRST(&p->table_types);
7644 TAILQ_REMOVE(&p->table_types, elem, node);
7652 static struct selector *
7653 selector_find(struct rte_swx_pipeline *p, const char *name)
7657 TAILQ_FOREACH(s, &p->selectors, node)
7658 if (strcmp(s->name, name) == 0)
7664 static struct selector *
7665 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7667 struct selector *s = NULL;
7669 TAILQ_FOREACH(s, &p->selectors, node)
7677 selector_fields_check(struct rte_swx_pipeline *p,
7678 struct rte_swx_pipeline_selector_params *params,
7679 struct header **header)
7681 struct header *h0 = NULL;
7682 struct field *hf, *mf;
7685 /* Return if no selector fields. */
7686 if (!params->n_selector_fields || !params->selector_field_names)
7689 /* Check that all the selector fields either belong to the same header
7690 * or are all meta-data fields.
7692 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7693 mf = metadata_field_parse(p, params->selector_field_names[0]);
7697 for (i = 1; i < params->n_selector_fields; i++)
7701 hf = header_field_parse(p, params->selector_field_names[i], &h);
7702 if (!hf || (h->id != h0->id))
7705 mf = metadata_field_parse(p, params->selector_field_names[i]);
7710 /* Check that there are no duplicated match fields. */
7711 for (i = 0; i < params->n_selector_fields; i++) {
7712 const char *field_name = params->selector_field_names[i];
7715 for (j = i + 1; j < params->n_selector_fields; j++)
7716 if (!strcmp(params->selector_field_names[j], field_name))
7728 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7730 struct rte_swx_pipeline_selector_params *params)
7733 struct header *selector_header = NULL;
7734 struct field *group_id_field, *member_id_field;
7740 CHECK_NAME(name, EINVAL);
7741 CHECK(!table_find(p, name), EEXIST);
7742 CHECK(!selector_find(p, name), EEXIST);
7743 CHECK(!learner_find(p, name), EEXIST);
7745 CHECK(params, EINVAL);
7747 CHECK_NAME(params->group_id_field_name, EINVAL);
7748 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7749 CHECK(group_id_field, EINVAL);
7751 for (i = 0; i < params->n_selector_fields; i++) {
7752 const char *field_name = params->selector_field_names[i];
7754 CHECK_NAME(field_name, EINVAL);
7756 status = selector_fields_check(p, params, &selector_header);
7760 CHECK_NAME(params->member_id_field_name, EINVAL);
7761 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7762 CHECK(member_id_field, EINVAL);
7764 CHECK(params->n_groups_max, EINVAL);
7766 CHECK(params->n_members_per_group_max, EINVAL);
7768 /* Memory allocation. */
7769 s = calloc(1, sizeof(struct selector));
7775 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7776 if (!s->selector_fields) {
7781 /* Node initialization. */
7782 strcpy(s->name, name);
7784 s->group_id_field = group_id_field;
7786 for (i = 0; i < params->n_selector_fields; i++) {
7787 const char *field_name = params->selector_field_names[i];
7789 s->selector_fields[i] = selector_header ?
7790 header_field_parse(p, field_name, NULL) :
7791 metadata_field_parse(p, field_name);
7794 s->n_selector_fields = params->n_selector_fields;
7796 s->selector_header = selector_header;
7798 s->member_id_field = member_id_field;
7800 s->n_groups_max = params->n_groups_max;
7802 s->n_members_per_group_max = params->n_members_per_group_max;
7804 s->id = p->n_selectors;
7806 /* Node add to tailq. */
7807 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7816 free(s->selector_fields);
7824 selector_params_free(struct rte_swx_table_selector_params *params)
7829 free(params->selector_mask);
7834 static struct rte_swx_table_selector_params *
7835 selector_table_params_get(struct selector *s)
7837 struct rte_swx_table_selector_params *params = NULL;
7838 struct field *first, *last;
7841 /* Memory allocation. */
7842 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7847 params->group_id_offset = s->group_id_field->offset / 8;
7849 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7850 first = s->selector_fields[0];
7851 last = s->selector_fields[0];
7853 for (i = 0; i < s->n_selector_fields; i++) {
7854 struct field *f = s->selector_fields[i];
7856 if (f->offset < first->offset)
7859 if (f->offset > last->offset)
7863 /* Selector offset and size. */
7864 params->selector_offset = first->offset / 8;
7865 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7867 /* Memory allocation. */
7868 params->selector_mask = calloc(1, params->selector_size);
7869 if (!params->selector_mask)
7872 /* Selector mask. */
7873 for (i = 0; i < s->n_selector_fields; i++) {
7874 struct field *f = s->selector_fields[i];
7875 uint32_t start = (f->offset - first->offset) / 8;
7876 size_t size = f->n_bits / 8;
7878 memset(¶ms->selector_mask[start], 0xFF, size);
7882 params->member_id_offset = s->member_id_field->offset / 8;
7884 /* Maximum number of groups. */
7885 params->n_groups_max = s->n_groups_max;
7887 /* Maximum number of members per group. */
7888 params->n_members_per_group_max = s->n_members_per_group_max;
7893 selector_params_free(params);
7898 selector_build_free(struct rte_swx_pipeline *p)
7902 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7903 struct thread *t = &p->threads[i];
7909 for (j = 0; j < p->n_selectors; j++) {
7910 struct selector_runtime *r = &t->selectors[j];
7916 t->selectors = NULL;
7919 free(p->selector_stats);
7920 p->selector_stats = NULL;
7924 selector_build(struct rte_swx_pipeline *p)
7929 /* Per pipeline: selector statistics. */
7930 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7931 if (!p->selector_stats) {
7936 /* Per thread: selector run-time. */
7937 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7938 struct thread *t = &p->threads[i];
7941 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7942 if (!t->selectors) {
7947 TAILQ_FOREACH(s, &p->selectors, node) {
7948 struct selector_runtime *r = &t->selectors[s->id];
7952 size = rte_swx_table_selector_mailbox_size_get();
7954 r->mailbox = calloc(1, size);
7961 /* r->group_id_buffer. */
7962 r->group_id_buffer = &t->structs[p->metadata_struct_id];
7964 /* r->selector_buffer. */
7965 r->selector_buffer = s->selector_header ?
7966 &t->structs[s->selector_header->struct_id] :
7967 &t->structs[p->metadata_struct_id];
7969 /* r->member_id_buffer. */
7970 r->member_id_buffer = &t->structs[p->metadata_struct_id];
7977 selector_build_free(p);
7982 selector_free(struct rte_swx_pipeline *p)
7984 selector_build_free(p);
7986 /* Selector tables. */
7988 struct selector *elem;
7990 elem = TAILQ_FIRST(&p->selectors);
7994 TAILQ_REMOVE(&p->selectors, elem, node);
7995 free(elem->selector_fields);
8003 static struct learner *
8004 learner_find(struct rte_swx_pipeline *p, const char *name)
8008 TAILQ_FOREACH(l, &p->learners, node)
8009 if (!strcmp(l->name, name))
8015 static struct learner *
8016 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8018 struct learner *l = NULL;
8020 TAILQ_FOREACH(l, &p->learners, node)
8028 learner_match_fields_check(struct rte_swx_pipeline *p,
8029 struct rte_swx_pipeline_learner_params *params,
8030 struct header **header)
8032 struct header *h0 = NULL;
8033 struct field *hf, *mf;
8036 /* Return if no match fields. */
8037 if (!params->n_fields || !params->field_names)
8040 /* Check that all the match fields either belong to the same header
8041 * or are all meta-data fields.
8043 hf = header_field_parse(p, params->field_names[0], &h0);
8044 mf = metadata_field_parse(p, params->field_names[0]);
8048 for (i = 1; i < params->n_fields; i++)
8052 hf = header_field_parse(p, params->field_names[i], &h);
8053 if (!hf || (h->id != h0->id))
8056 mf = metadata_field_parse(p, params->field_names[i]);
8061 /* Check that there are no duplicated match fields. */
8062 for (i = 0; i < params->n_fields; i++) {
8063 const char *field_name = params->field_names[i];
8066 for (j = i + 1; j < params->n_fields; j++)
8067 if (!strcmp(params->field_names[j], field_name))
8079 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8081 struct struct_type *mst = p->metadata_st, *ast = a->st;
8082 struct field *mf, *af;
8092 /* Check that mf_name is the name of a valid meta-data field. */
8093 CHECK_NAME(mf_name, EINVAL);
8094 mf = metadata_field_parse(p, mf_name);
8097 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8098 * all the action arguments.
8100 mf_pos = mf - mst->fields;
8101 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8103 /* Check that the size of each of the identified meta-data fields matches exactly the size
8104 * of the corresponding action argument.
8106 for (i = 0; i < ast->n_fields; i++) {
8107 mf = &mst->fields[mf_pos + i];
8108 af = &ast->fields[i];
8110 CHECK(mf->n_bits == af->n_bits, EINVAL);
8117 learner_action_learning_check(struct rte_swx_pipeline *p,
8118 struct action *action,
8119 const char **action_names,
8124 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8125 * the action passed as argument to the "learn" instruction) is also enabled for the
8126 * current learner table.
8128 for (i = 0; i < action->n_instructions; i++) {
8129 struct instruction *instr = &action->instructions[i];
8130 uint32_t found = 0, j;
8132 if (instr->type != INSTR_LEARNER_LEARN)
8135 for (j = 0; j < n_actions; j++) {
8138 a = action_find(p, action_names[j]);
8142 if (a->id == instr->learn.action_id)
8154 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8156 struct rte_swx_pipeline_learner_params *params,
8160 struct learner *l = NULL;
8161 struct action *default_action;
8162 struct header *header = NULL;
8163 uint32_t action_data_size_max = 0, i;
8168 CHECK_NAME(name, EINVAL);
8169 CHECK(!table_find(p, name), EEXIST);
8170 CHECK(!selector_find(p, name), EEXIST);
8171 CHECK(!learner_find(p, name), EEXIST);
8173 CHECK(params, EINVAL);
8176 status = learner_match_fields_check(p, params, &header);
8180 /* Action checks. */
8181 CHECK(params->n_actions, EINVAL);
8183 CHECK(params->action_names, EINVAL);
8184 for (i = 0; i < params->n_actions; i++) {
8185 const char *action_name = params->action_names[i];
8187 uint32_t action_data_size;
8189 CHECK_NAME(action_name, EINVAL);
8191 a = action_find(p, action_name);
8194 status = learner_action_learning_check(p,
8196 params->action_names,
8201 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8202 if (action_data_size > action_data_size_max)
8203 action_data_size_max = action_data_size;
8206 CHECK_NAME(params->default_action_name, EINVAL);
8207 for (i = 0; i < p->n_actions; i++)
8208 if (!strcmp(params->action_names[i],
8209 params->default_action_name))
8211 CHECK(i < params->n_actions, EINVAL);
8213 default_action = action_find(p, params->default_action_name);
8214 CHECK((default_action->st && params->default_action_data) ||
8215 !params->default_action_data, EINVAL);
8217 /* Any other checks. */
8218 CHECK(size, EINVAL);
8219 CHECK(timeout, EINVAL);
8221 /* Memory allocation. */
8222 l = calloc(1, sizeof(struct learner));
8226 l->fields = calloc(params->n_fields, sizeof(struct field *));
8230 l->actions = calloc(params->n_actions, sizeof(struct action *));
8234 if (action_data_size_max) {
8235 l->default_action_data = calloc(1, action_data_size_max);
8236 if (!l->default_action_data)
8240 /* Node initialization. */
8241 strcpy(l->name, name);
8243 for (i = 0; i < params->n_fields; i++) {
8244 const char *field_name = params->field_names[i];
8246 l->fields[i] = header ?
8247 header_field_parse(p, field_name, NULL) :
8248 metadata_field_parse(p, field_name);
8251 l->n_fields = params->n_fields;
8255 for (i = 0; i < params->n_actions; i++)
8256 l->actions[i] = action_find(p, params->action_names[i]);
8258 l->default_action = default_action;
8260 if (default_action->st)
8261 memcpy(l->default_action_data,
8262 params->default_action_data,
8263 default_action->st->n_bits / 8);
8265 l->n_actions = params->n_actions;
8267 l->default_action_is_const = params->default_action_is_const;
8269 l->action_data_size_max = action_data_size_max;
8273 l->timeout = timeout;
8275 l->id = p->n_learners;
8277 /* Node add to tailq. */
8278 TAILQ_INSERT_TAIL(&p->learners, l, node);
8295 learner_params_free(struct rte_swx_table_learner_params *params)
8300 free(params->key_mask0);
8305 static struct rte_swx_table_learner_params *
8306 learner_params_get(struct learner *l)
8308 struct rte_swx_table_learner_params *params = NULL;
8309 struct field *first, *last;
8312 /* Memory allocation. */
8313 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8317 /* Find first (smallest offset) and last (biggest offset) match fields. */
8318 first = l->fields[0];
8319 last = l->fields[0];
8321 for (i = 0; i < l->n_fields; i++) {
8322 struct field *f = l->fields[i];
8324 if (f->offset < first->offset)
8327 if (f->offset > last->offset)
8331 /* Key offset and size. */
8332 params->key_offset = first->offset / 8;
8333 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8335 /* Memory allocation. */
8336 params->key_mask0 = calloc(1, params->key_size);
8337 if (!params->key_mask0)
8341 for (i = 0; i < l->n_fields; i++) {
8342 struct field *f = l->fields[i];
8343 uint32_t start = (f->offset - first->offset) / 8;
8344 size_t size = f->n_bits / 8;
8346 memset(¶ms->key_mask0[start], 0xFF, size);
8349 /* Action data size. */
8350 params->action_data_size = l->action_data_size_max;
8352 /* Maximum number of keys. */
8353 params->n_keys_max = l->size;
8356 params->key_timeout = l->timeout;
8361 learner_params_free(params);
8366 learner_build_free(struct rte_swx_pipeline *p)
8370 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8371 struct thread *t = &p->threads[i];
8377 for (j = 0; j < p->n_learners; j++) {
8378 struct learner_runtime *r = &t->learners[j];
8387 if (p->learner_stats) {
8388 for (i = 0; i < p->n_learners; i++)
8389 free(p->learner_stats[i].n_pkts_action);
8391 free(p->learner_stats);
8396 learner_build(struct rte_swx_pipeline *p)
8401 /* Per pipeline: learner statistics. */
8402 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8403 CHECK(p->learner_stats, ENOMEM);
8405 for (i = 0; i < p->n_learners; i++) {
8406 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8407 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8410 /* Per thread: learner run-time. */
8411 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8412 struct thread *t = &p->threads[i];
8415 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8421 TAILQ_FOREACH(l, &p->learners, node) {
8422 struct learner_runtime *r = &t->learners[l->id];
8426 size = rte_swx_table_learner_mailbox_size_get();
8428 r->mailbox = calloc(1, size);
8436 r->key = l->header ?
8437 &t->structs[l->header->struct_id] :
8438 &t->structs[p->metadata_struct_id];
8445 learner_build_free(p);
8450 learner_free(struct rte_swx_pipeline *p)
8452 learner_build_free(p);
8454 /* Learner tables. */
8458 l = TAILQ_FIRST(&p->learners);
8462 TAILQ_REMOVE(&p->learners, l, node);
8465 free(l->default_action_data);
8474 table_state_build(struct rte_swx_pipeline *p)
8476 struct table *table;
8480 p->table_state = calloc(p->n_tables + p->n_selectors,
8481 sizeof(struct rte_swx_table_state));
8482 CHECK(p->table_state, ENOMEM);
8484 TAILQ_FOREACH(table, &p->tables, node) {
8485 struct rte_swx_table_state *ts = &p->table_state[table->id];
8488 struct rte_swx_table_params *params;
8491 params = table_params_get(table);
8492 CHECK(params, ENOMEM);
8494 ts->obj = table->type->ops.create(params,
8499 table_params_free(params);
8500 CHECK(ts->obj, ENODEV);
8503 /* ts->default_action_data. */
8504 if (table->action_data_size_max) {
8505 ts->default_action_data =
8506 malloc(table->action_data_size_max);
8507 CHECK(ts->default_action_data, ENOMEM);
8509 memcpy(ts->default_action_data,
8510 table->default_action_data,
8511 table->action_data_size_max);
8514 /* ts->default_action_id. */
8515 ts->default_action_id = table->default_action->id;
8518 TAILQ_FOREACH(s, &p->selectors, node) {
8519 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8520 struct rte_swx_table_selector_params *params;
8523 params = selector_table_params_get(s);
8524 CHECK(params, ENOMEM);
8526 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8528 selector_params_free(params);
8529 CHECK(ts->obj, ENODEV);
8532 TAILQ_FOREACH(l, &p->learners, node) {
8533 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8534 p->n_selectors + l->id];
8535 struct rte_swx_table_learner_params *params;
8538 params = learner_params_get(l);
8539 CHECK(params, ENOMEM);
8541 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8542 learner_params_free(params);
8543 CHECK(ts->obj, ENODEV);
8545 /* ts->default_action_data. */
8546 if (l->action_data_size_max) {
8547 ts->default_action_data = malloc(l->action_data_size_max);
8548 CHECK(ts->default_action_data, ENOMEM);
8550 memcpy(ts->default_action_data,
8551 l->default_action_data,
8552 l->action_data_size_max);
8555 /* ts->default_action_id. */
8556 ts->default_action_id = l->default_action->id;
8563 table_state_build_free(struct rte_swx_pipeline *p)
8567 if (!p->table_state)
8570 for (i = 0; i < p->n_tables; i++) {
8571 struct rte_swx_table_state *ts = &p->table_state[i];
8572 struct table *table = table_find_by_id(p, i);
8575 if (table->type && ts->obj)
8576 table->type->ops.free(ts->obj);
8578 /* ts->default_action_data. */
8579 free(ts->default_action_data);
8582 for (i = 0; i < p->n_selectors; i++) {
8583 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8587 rte_swx_table_selector_free(ts->obj);
8590 for (i = 0; i < p->n_learners; i++) {
8591 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8595 rte_swx_table_learner_free(ts->obj);
8597 /* ts->default_action_data. */
8598 free(ts->default_action_data);
8601 free(p->table_state);
8602 p->table_state = NULL;
8606 table_state_free(struct rte_swx_pipeline *p)
8608 table_state_build_free(p);
8614 static struct regarray *
8615 regarray_find(struct rte_swx_pipeline *p, const char *name)
8617 struct regarray *elem;
8619 TAILQ_FOREACH(elem, &p->regarrays, node)
8620 if (!strcmp(elem->name, name))
8626 static struct regarray *
8627 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8629 struct regarray *elem = NULL;
8631 TAILQ_FOREACH(elem, &p->regarrays, node)
8639 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8648 CHECK_NAME(name, EINVAL);
8649 CHECK(!regarray_find(p, name), EEXIST);
8651 CHECK(size, EINVAL);
8652 size = rte_align32pow2(size);
8654 /* Memory allocation. */
8655 r = calloc(1, sizeof(struct regarray));
8658 /* Node initialization. */
8659 strcpy(r->name, name);
8660 r->init_val = init_val;
8662 r->id = p->n_regarrays;
8664 /* Node add to tailq. */
8665 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8672 regarray_build(struct rte_swx_pipeline *p)
8674 struct regarray *regarray;
8676 if (!p->n_regarrays)
8679 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8680 CHECK(p->regarray_runtime, ENOMEM);
8682 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8683 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8686 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8687 RTE_CACHE_LINE_SIZE,
8689 CHECK(r->regarray, ENOMEM);
8691 if (regarray->init_val)
8692 for (i = 0; i < regarray->size; i++)
8693 r->regarray[i] = regarray->init_val;
8695 r->size_mask = regarray->size - 1;
8702 regarray_build_free(struct rte_swx_pipeline *p)
8706 if (!p->regarray_runtime)
8709 for (i = 0; i < p->n_regarrays; i++) {
8710 struct regarray *regarray = regarray_find_by_id(p, i);
8711 struct regarray_runtime *r = &p->regarray_runtime[i];
8713 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8716 free(p->regarray_runtime);
8717 p->regarray_runtime = NULL;
8721 regarray_free(struct rte_swx_pipeline *p)
8723 regarray_build_free(p);
8726 struct regarray *elem;
8728 elem = TAILQ_FIRST(&p->regarrays);
8732 TAILQ_REMOVE(&p->regarrays, elem, node);
8740 static struct meter_profile *
8741 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8743 struct meter_profile *elem;
8745 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8746 if (!strcmp(elem->name, name))
8752 static struct metarray *
8753 metarray_find(struct rte_swx_pipeline *p, const char *name)
8755 struct metarray *elem;
8757 TAILQ_FOREACH(elem, &p->metarrays, node)
8758 if (!strcmp(elem->name, name))
8764 static struct metarray *
8765 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8767 struct metarray *elem = NULL;
8769 TAILQ_FOREACH(elem, &p->metarrays, node)
8777 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8785 CHECK_NAME(name, EINVAL);
8786 CHECK(!metarray_find(p, name), EEXIST);
8788 CHECK(size, EINVAL);
8789 size = rte_align32pow2(size);
8791 /* Memory allocation. */
8792 m = calloc(1, sizeof(struct metarray));
8795 /* Node initialization. */
8796 strcpy(m->name, name);
8798 m->id = p->n_metarrays;
8800 /* Node add to tailq. */
8801 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8807 struct meter_profile meter_profile_default = {
8816 .cir_bytes_per_period = 1,
8818 .pir_bytes_per_period = 1,
8825 meter_init(struct meter *m)
8827 memset(m, 0, sizeof(struct meter));
8828 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8829 m->profile = &meter_profile_default;
8830 m->color_mask = RTE_COLOR_GREEN;
8832 meter_profile_default.n_users++;
8836 metarray_build(struct rte_swx_pipeline *p)
8840 if (!p->n_metarrays)
8843 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8844 CHECK(p->metarray_runtime, ENOMEM);
8846 TAILQ_FOREACH(m, &p->metarrays, node) {
8847 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8850 r->metarray = env_malloc(m->size * sizeof(struct meter),
8851 RTE_CACHE_LINE_SIZE,
8853 CHECK(r->metarray, ENOMEM);
8855 for (i = 0; i < m->size; i++)
8856 meter_init(&r->metarray[i]);
8858 r->size_mask = m->size - 1;
8865 metarray_build_free(struct rte_swx_pipeline *p)
8869 if (!p->metarray_runtime)
8872 for (i = 0; i < p->n_metarrays; i++) {
8873 struct metarray *m = metarray_find_by_id(p, i);
8874 struct metarray_runtime *r = &p->metarray_runtime[i];
8876 env_free(r->metarray, m->size * sizeof(struct meter));
8879 free(p->metarray_runtime);
8880 p->metarray_runtime = NULL;
8884 metarray_free(struct rte_swx_pipeline *p)
8886 metarray_build_free(p);
8890 struct metarray *elem;
8892 elem = TAILQ_FIRST(&p->metarrays);
8896 TAILQ_REMOVE(&p->metarrays, elem, node);
8900 /* Meter profiles. */
8902 struct meter_profile *elem;
8904 elem = TAILQ_FIRST(&p->meter_profiles);
8908 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8917 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8919 struct rte_swx_pipeline *pipeline;
8921 /* Check input parameters. */
8924 /* Memory allocation. */
8925 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8926 CHECK(pipeline, ENOMEM);
8928 /* Initialization. */
8929 TAILQ_INIT(&pipeline->struct_types);
8930 TAILQ_INIT(&pipeline->port_in_types);
8931 TAILQ_INIT(&pipeline->ports_in);
8932 TAILQ_INIT(&pipeline->port_out_types);
8933 TAILQ_INIT(&pipeline->ports_out);
8934 TAILQ_INIT(&pipeline->extern_types);
8935 TAILQ_INIT(&pipeline->extern_objs);
8936 TAILQ_INIT(&pipeline->extern_funcs);
8937 TAILQ_INIT(&pipeline->headers);
8938 TAILQ_INIT(&pipeline->actions);
8939 TAILQ_INIT(&pipeline->table_types);
8940 TAILQ_INIT(&pipeline->tables);
8941 TAILQ_INIT(&pipeline->selectors);
8942 TAILQ_INIT(&pipeline->learners);
8943 TAILQ_INIT(&pipeline->regarrays);
8944 TAILQ_INIT(&pipeline->meter_profiles);
8945 TAILQ_INIT(&pipeline->metarrays);
8947 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8948 pipeline->numa_node = numa_node;
8955 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8964 free(p->instruction_data);
8965 free(p->instructions);
8969 table_state_free(p);
8974 instruction_table_free(p);
8977 extern_func_free(p);
8990 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8991 const char **instructions,
8992 uint32_t n_instructions)
8997 err = instruction_config(p, NULL, instructions, n_instructions);
9001 /* Thread instruction pointer reset. */
9002 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9003 struct thread *t = &p->threads[i];
9005 thread_ip_reset(p, t);
9012 pipeline_compile(struct rte_swx_pipeline *p);
9015 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9020 CHECK(p->build_done == 0, EEXIST);
9022 status = port_in_build(p);
9026 status = port_out_build(p);
9030 status = struct_build(p);
9034 status = extern_obj_build(p);
9038 status = extern_func_build(p);
9042 status = header_build(p);
9046 status = metadata_build(p);
9050 status = instruction_table_build(p);
9054 status = action_build(p);
9058 status = table_build(p);
9062 status = selector_build(p);
9066 status = learner_build(p);
9070 status = table_state_build(p);
9074 status = regarray_build(p);
9078 status = metarray_build(p);
9084 pipeline_compile(p);
9089 metarray_build_free(p);
9090 regarray_build_free(p);
9091 table_state_build_free(p);
9092 learner_build_free(p);
9093 selector_build_free(p);
9094 table_build_free(p);
9095 action_build_free(p);
9096 instruction_table_build_free(p);
9097 metadata_build_free(p);
9098 header_build_free(p);
9099 extern_func_build_free(p);
9100 extern_obj_build_free(p);
9101 port_out_build_free(p);
9102 port_in_build_free(p);
9103 struct_build_free(p);
9109 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9113 for (i = 0; i < n_instructions; i++)
9118 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9122 for (i = 0; i < p->n_ports_out; i++) {
9123 struct port_out_runtime *port = &p->out[i];
9126 port->flush(port->obj);
9134 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9135 struct rte_swx_ctl_pipeline_info *pipeline)
9137 struct action *action;
9138 struct table *table;
9139 uint32_t n_actions = 0, n_tables = 0;
9141 if (!p || !pipeline)
9144 TAILQ_FOREACH(action, &p->actions, node)
9147 TAILQ_FOREACH(table, &p->tables, node)
9150 pipeline->n_ports_in = p->n_ports_in;
9151 pipeline->n_ports_out = p->n_ports_out;
9152 pipeline->n_actions = n_actions;
9153 pipeline->n_tables = n_tables;
9154 pipeline->n_selectors = p->n_selectors;
9155 pipeline->n_learners = p->n_learners;
9156 pipeline->n_regarrays = p->n_regarrays;
9157 pipeline->n_metarrays = p->n_metarrays;
9163 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9165 if (!p || !numa_node)
9168 *numa_node = p->numa_node;
9173 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9175 struct rte_swx_ctl_action_info *action)
9177 struct action *a = NULL;
9179 if (!p || (action_id >= p->n_actions) || !action)
9182 a = action_find_by_id(p, action_id);
9186 strcpy(action->name, a->name);
9187 action->n_args = a->st ? a->st->n_fields : 0;
9192 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9194 uint32_t action_arg_id,
9195 struct rte_swx_ctl_action_arg_info *action_arg)
9197 struct action *a = NULL;
9198 struct field *arg = NULL;
9200 if (!p || (action_id >= p->n_actions) || !action_arg)
9203 a = action_find_by_id(p, action_id);
9204 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9207 arg = &a->st->fields[action_arg_id];
9208 strcpy(action_arg->name, arg->name);
9209 action_arg->n_bits = arg->n_bits;
9210 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9216 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9218 struct rte_swx_ctl_table_info *table)
9220 struct table *t = NULL;
9225 t = table_find_by_id(p, table_id);
9229 strcpy(table->name, t->name);
9230 strcpy(table->args, t->args);
9231 table->n_match_fields = t->n_fields;
9232 table->n_actions = t->n_actions;
9233 table->default_action_is_const = t->default_action_is_const;
9234 table->size = t->size;
9239 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9241 uint32_t match_field_id,
9242 struct rte_swx_ctl_table_match_field_info *match_field)
9245 struct match_field *f;
9247 if (!p || (table_id >= p->n_tables) || !match_field)
9250 t = table_find_by_id(p, table_id);
9251 if (!t || (match_field_id >= t->n_fields))
9254 f = &t->fields[match_field_id];
9255 match_field->match_type = f->match_type;
9256 match_field->is_header = t->header ? 1 : 0;
9257 match_field->n_bits = f->field->n_bits;
9258 match_field->offset = f->field->offset;
9264 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9266 uint32_t table_action_id,
9267 struct rte_swx_ctl_table_action_info *table_action)
9271 if (!p || (table_id >= p->n_tables) || !table_action)
9274 t = table_find_by_id(p, table_id);
9275 if (!t || (table_action_id >= t->n_actions))
9278 table_action->action_id = t->actions[table_action_id]->id;
9284 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9286 struct rte_swx_table_ops *table_ops,
9291 if (!p || (table_id >= p->n_tables))
9294 t = table_find_by_id(p, table_id);
9300 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9310 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9311 uint32_t selector_id,
9312 struct rte_swx_ctl_selector_info *selector)
9314 struct selector *s = NULL;
9316 if (!p || !selector)
9319 s = selector_find_by_id(p, selector_id);
9323 strcpy(selector->name, s->name);
9325 selector->n_selector_fields = s->n_selector_fields;
9326 selector->n_groups_max = s->n_groups_max;
9327 selector->n_members_per_group_max = s->n_members_per_group_max;
9333 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9334 uint32_t selector_id,
9335 struct rte_swx_ctl_table_match_field_info *field)
9339 if (!p || (selector_id >= p->n_selectors) || !field)
9342 s = selector_find_by_id(p, selector_id);
9346 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9347 field->is_header = 0;
9348 field->n_bits = s->group_id_field->n_bits;
9349 field->offset = s->group_id_field->offset;
9355 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9356 uint32_t selector_id,
9357 uint32_t selector_field_id,
9358 struct rte_swx_ctl_table_match_field_info *field)
9363 if (!p || (selector_id >= p->n_selectors) || !field)
9366 s = selector_find_by_id(p, selector_id);
9367 if (!s || (selector_field_id >= s->n_selector_fields))
9370 f = s->selector_fields[selector_field_id];
9371 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9372 field->is_header = s->selector_header ? 1 : 0;
9373 field->n_bits = f->n_bits;
9374 field->offset = f->offset;
9380 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9381 uint32_t selector_id,
9382 struct rte_swx_ctl_table_match_field_info *field)
9386 if (!p || (selector_id >= p->n_selectors) || !field)
9389 s = selector_find_by_id(p, selector_id);
9393 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9394 field->is_header = 0;
9395 field->n_bits = s->member_id_field->n_bits;
9396 field->offset = s->member_id_field->offset;
9402 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9403 uint32_t learner_id,
9404 struct rte_swx_ctl_learner_info *learner)
9406 struct learner *l = NULL;
9411 l = learner_find_by_id(p, learner_id);
9415 strcpy(learner->name, l->name);
9417 learner->n_match_fields = l->n_fields;
9418 learner->n_actions = l->n_actions;
9419 learner->default_action_is_const = l->default_action_is_const;
9420 learner->size = l->size;
9426 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9427 uint32_t learner_id,
9428 uint32_t match_field_id,
9429 struct rte_swx_ctl_table_match_field_info *match_field)
9434 if (!p || (learner_id >= p->n_learners) || !match_field)
9437 l = learner_find_by_id(p, learner_id);
9438 if (!l || (match_field_id >= l->n_fields))
9441 f = l->fields[match_field_id];
9442 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9443 match_field->is_header = l->header ? 1 : 0;
9444 match_field->n_bits = f->n_bits;
9445 match_field->offset = f->offset;
9451 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9452 uint32_t learner_id,
9453 uint32_t learner_action_id,
9454 struct rte_swx_ctl_table_action_info *learner_action)
9458 if (!p || (learner_id >= p->n_learners) || !learner_action)
9461 l = learner_find_by_id(p, learner_id);
9462 if (!l || (learner_action_id >= l->n_actions))
9465 learner_action->action_id = l->actions[learner_action_id]->id;
9471 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9472 struct rte_swx_table_state **table_state)
9474 if (!p || !table_state || !p->build_done)
9477 *table_state = p->table_state;
9482 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9483 struct rte_swx_table_state *table_state)
9485 if (!p || !table_state || !p->build_done)
9488 p->table_state = table_state;
9493 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9495 struct rte_swx_port_in_stats *stats)
9497 struct port_in *port;
9502 port = port_in_find(p, port_id);
9506 port->type->ops.stats_read(port->obj, stats);
9511 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9513 struct rte_swx_port_out_stats *stats)
9515 struct port_out *port;
9520 port = port_out_find(p, port_id);
9524 port->type->ops.stats_read(port->obj, stats);
9529 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9530 const char *table_name,
9531 struct rte_swx_table_stats *stats)
9533 struct table *table;
9534 struct table_statistics *table_stats;
9536 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9539 table = table_find(p, table_name);
9543 table_stats = &p->table_stats[table->id];
9545 memcpy(stats->n_pkts_action,
9546 table_stats->n_pkts_action,
9547 p->n_actions * sizeof(uint64_t));
9549 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9550 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9556 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9557 const char *selector_name,
9558 struct rte_swx_pipeline_selector_stats *stats)
9562 if (!p || !selector_name || !selector_name[0] || !stats)
9565 s = selector_find(p, selector_name);
9569 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9575 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9576 const char *learner_name,
9577 struct rte_swx_learner_stats *stats)
9580 struct learner_statistics *learner_stats;
9582 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9585 l = learner_find(p, learner_name);
9589 learner_stats = &p->learner_stats[l->id];
9591 memcpy(stats->n_pkts_action,
9592 learner_stats->n_pkts_action,
9593 p->n_actions * sizeof(uint64_t));
9595 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9596 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9598 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9599 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9601 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9607 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9608 uint32_t regarray_id,
9609 struct rte_swx_ctl_regarray_info *regarray)
9613 if (!p || !regarray)
9616 r = regarray_find_by_id(p, regarray_id);
9620 strcpy(regarray->name, r->name);
9621 regarray->size = r->size;
9626 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9627 const char *regarray_name,
9628 uint32_t regarray_index,
9631 struct regarray *regarray;
9632 struct regarray_runtime *r;
9634 if (!p || !regarray_name || !value)
9637 regarray = regarray_find(p, regarray_name);
9638 if (!regarray || (regarray_index >= regarray->size))
9641 r = &p->regarray_runtime[regarray->id];
9642 *value = r->regarray[regarray_index];
9647 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9648 const char *regarray_name,
9649 uint32_t regarray_index,
9652 struct regarray *regarray;
9653 struct regarray_runtime *r;
9655 if (!p || !regarray_name)
9658 regarray = regarray_find(p, regarray_name);
9659 if (!regarray || (regarray_index >= regarray->size))
9662 r = &p->regarray_runtime[regarray->id];
9663 r->regarray[regarray_index] = value;
9668 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9669 uint32_t metarray_id,
9670 struct rte_swx_ctl_metarray_info *metarray)
9674 if (!p || !metarray)
9677 m = metarray_find_by_id(p, metarray_id);
9681 strcpy(metarray->name, m->name);
9682 metarray->size = m->size;
9687 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9689 struct rte_meter_trtcm_params *params)
9691 struct meter_profile *mp;
9695 CHECK_NAME(name, EINVAL);
9696 CHECK(params, EINVAL);
9697 CHECK(!meter_profile_find(p, name), EEXIST);
9699 /* Node allocation. */
9700 mp = calloc(1, sizeof(struct meter_profile));
9703 /* Node initialization. */
9704 strcpy(mp->name, name);
9705 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9706 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9712 /* Node add to tailq. */
9713 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9719 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9722 struct meter_profile *mp;
9725 CHECK_NAME(name, EINVAL);
9727 mp = meter_profile_find(p, name);
9729 CHECK(!mp->n_users, EBUSY);
9731 /* Remove node from tailq. */
9732 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9739 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9740 const char *metarray_name,
9741 uint32_t metarray_index)
9743 struct meter_profile *mp_old;
9744 struct metarray *metarray;
9745 struct metarray_runtime *metarray_runtime;
9749 CHECK_NAME(metarray_name, EINVAL);
9751 metarray = metarray_find(p, metarray_name);
9752 CHECK(metarray, EINVAL);
9753 CHECK(metarray_index < metarray->size, EINVAL);
9755 metarray_runtime = &p->metarray_runtime[metarray->id];
9756 m = &metarray_runtime->metarray[metarray_index];
9757 mp_old = m->profile;
9767 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9768 const char *metarray_name,
9769 uint32_t metarray_index,
9770 const char *profile_name)
9772 struct meter_profile *mp, *mp_old;
9773 struct metarray *metarray;
9774 struct metarray_runtime *metarray_runtime;
9778 CHECK_NAME(metarray_name, EINVAL);
9780 metarray = metarray_find(p, metarray_name);
9781 CHECK(metarray, EINVAL);
9782 CHECK(metarray_index < metarray->size, EINVAL);
9784 mp = meter_profile_find(p, profile_name);
9787 metarray_runtime = &p->metarray_runtime[metarray->id];
9788 m = &metarray_runtime->metarray[metarray_index];
9789 mp_old = m->profile;
9791 memset(m, 0, sizeof(struct meter));
9792 rte_meter_trtcm_config(&m->m, &mp->profile);
9794 m->color_mask = RTE_COLORS;
9803 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9804 const char *metarray_name,
9805 uint32_t metarray_index,
9806 struct rte_swx_ctl_meter_stats *stats)
9808 struct metarray *metarray;
9809 struct metarray_runtime *metarray_runtime;
9813 CHECK_NAME(metarray_name, EINVAL);
9815 metarray = metarray_find(p, metarray_name);
9816 CHECK(metarray, EINVAL);
9817 CHECK(metarray_index < metarray->size, EINVAL);
9819 CHECK(stats, EINVAL);
9821 metarray_runtime = &p->metarray_runtime[metarray->id];
9822 m = &metarray_runtime->metarray[metarray_index];
9824 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9825 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9831 * Pipeline compilation.
9834 instr_type_to_name(struct instruction *instr)
9836 switch (instr->type) {
9837 case INSTR_RX: return "INSTR_RX";
9839 case INSTR_TX: return "INSTR_TX";
9840 case INSTR_TX_I: return "INSTR_TX_I";
9842 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9843 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9844 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9845 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9846 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9847 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9848 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9849 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9851 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9853 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9855 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9856 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9857 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9858 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9859 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9860 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9861 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9862 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9863 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9865 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9866 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9868 case INSTR_MOV: return "INSTR_MOV";
9869 case INSTR_MOV_MH: return "INSTR_MOV_MH";
9870 case INSTR_MOV_HM: return "INSTR_MOV_HM";
9871 case INSTR_MOV_HH: return "INSTR_MOV_HH";
9872 case INSTR_MOV_I: return "INSTR_MOV_I";
9874 case INSTR_DMA_HT: return "INSTR_DMA_HT";
9875 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9876 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9877 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9878 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9879 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9880 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9881 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9883 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9884 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9885 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9886 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9887 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9888 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9890 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9891 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9892 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9893 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9894 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9895 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9897 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9898 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9899 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9900 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9902 case INSTR_ALU_AND: return "INSTR_ALU_AND";
9903 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9904 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9905 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9906 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9908 case INSTR_ALU_OR: return "INSTR_ALU_OR";
9909 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9910 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9911 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9912 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9914 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9915 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9916 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9917 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9918 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9920 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9921 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9922 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9923 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9924 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9925 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9927 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9928 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9929 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9930 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9931 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9932 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9934 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9935 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9936 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
9938 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
9939 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
9940 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
9941 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
9942 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
9943 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
9945 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
9946 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
9947 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
9948 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
9949 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
9950 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
9951 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
9952 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
9953 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
9955 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
9956 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
9957 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
9958 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
9959 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
9960 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
9961 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
9962 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
9963 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
9965 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
9966 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
9967 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
9969 case INSTR_METER_HHM: return "INSTR_METER_HHM";
9970 case INSTR_METER_HHI: return "INSTR_METER_HHI";
9971 case INSTR_METER_HMM: return "INSTR_METER_HMM";
9972 case INSTR_METER_HMI: return "INSTR_METER_HMI";
9973 case INSTR_METER_MHM: return "INSTR_METER_MHM";
9974 case INSTR_METER_MHI: return "INSTR_METER_MHI";
9975 case INSTR_METER_MMM: return "INSTR_METER_MMM";
9976 case INSTR_METER_MMI: return "INSTR_METER_MMI";
9977 case INSTR_METER_IHM: return "INSTR_METER_IHM";
9978 case INSTR_METER_IHI: return "INSTR_METER_IHI";
9979 case INSTR_METER_IMM: return "INSTR_METER_IMM";
9980 case INSTR_METER_IMI: return "INSTR_METER_IMI";
9982 case INSTR_TABLE: return "INSTR_TABLE";
9983 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
9984 case INSTR_SELECTOR: return "INSTR_SELECTOR";
9985 case INSTR_LEARNER: return "INSTR_LEARNER";
9986 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
9988 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
9989 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
9991 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
9992 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
9994 case INSTR_JMP: return "INSTR_JMP";
9995 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
9996 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
9997 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
9998 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
9999 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10000 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10001 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10002 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10003 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10004 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10005 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10006 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10007 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10008 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10009 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10010 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10011 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10012 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10013 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10014 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10015 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10016 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10017 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10018 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10019 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10020 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10021 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10022 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10024 case INSTR_RETURN: return "INSTR_RETURN";
10026 default: return "INSTR_UNKNOWN";
10031 (*instruction_export_t)(struct instruction *, FILE *);
10034 instr_io_export(struct instruction *instr, FILE *f)
10036 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10038 /* n_io, n_io_imm, n_hdrs. */
10039 if (instr->type == INSTR_RX ||
10040 instr->type == INSTR_TX ||
10041 instr->type == INSTR_HDR_EXTRACT_M ||
10042 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10045 if (instr->type == INSTR_TX_I)
10048 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10049 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10051 if (instr->type == INSTR_HDR_EXTRACT_M ||
10052 instr->type == INSTR_HDR_LOOKAHEAD ||
10053 instr->type == INSTR_HDR_EMIT)
10056 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10057 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10062 "\t\t.type = %s,\n",
10063 instr_type_to_name(instr));
10073 "\t\t\t\t.offset = %u,\n"
10074 "\t\t\t\t.n_bits = %u,\n"
10076 instr->io.io.offset,
10077 instr->io.io.n_bits);
10082 "\t\t\t\t.val = %u,\n"
10086 /* instr.io.hdr. */
10091 /* instr.io.hdr.header_id. */
10093 "\t\t\t.header_id = {");
10095 for (i = 0; i < n_hdrs; i++)
10098 instr->io.hdr.header_id[i]);
10103 /* instr.io.hdr.struct_id. */
10105 "\t\t\t.struct_id = {");
10107 for (i = 0; i < n_hdrs; i++)
10110 instr->io.hdr.struct_id[i]);
10115 /* instr.io.hdr.n_bytes. */
10117 "\t\t\t.n_bytes = {");
10119 for (i = 0; i < n_hdrs; i++)
10122 instr->io.hdr.n_bytes[i]);
10127 /* instr.io.hdr - closing curly brace. */
10132 /* instr.io - closing curly brace. */
10136 /* instr - closing curly brace. */
10142 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10146 "\t\t.type = %s,\n"
10148 "\t\t\t.header_id = %u,\n"
10151 instr_type_to_name(instr),
10152 instr->valid.header_id);
10156 instr_mov_export(struct instruction *instr, FILE *f)
10158 if (instr->type != INSTR_MOV_I)
10161 "\t\t.type = %s,\n"
10164 "\t\t\t\t.struct_id = %u,\n"
10165 "\t\t\t\t.n_bits = %u,\n"
10166 "\t\t\t\t.offset = %u,\n"
10169 "\t\t\t\t.struct_id = %u,\n"
10170 "\t\t\t\t.n_bits = %u,\n"
10171 "\t\t\t\t.offset = %u,\n"
10175 instr_type_to_name(instr),
10176 instr->mov.dst.struct_id,
10177 instr->mov.dst.n_bits,
10178 instr->mov.dst.offset,
10179 instr->mov.src.struct_id,
10180 instr->mov.src.n_bits,
10181 instr->mov.src.offset);
10185 "\t\t.type = %s,\n"
10188 "\t\t\t\t.struct_id = %u,\n"
10189 "\t\t\t\t.n_bits = %u,\n"
10190 "\t\t\t\t.offset = %u,\n"
10192 "\t\t\t.src_val = %" PRIu64 ",\n"
10195 instr_type_to_name(instr),
10196 instr->mov.dst.struct_id,
10197 instr->mov.dst.n_bits,
10198 instr->mov.dst.offset,
10199 instr->mov.src_val);
10203 instr_dma_ht_export(struct instruction *instr, FILE *f)
10205 uint32_t n_dma = 0, i;
10208 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10213 "\t\t.type = %s,\n",
10214 instr_type_to_name(instr));
10220 /* instr.dma.dst. */
10222 "\t\t\t.dst = {\n");
10224 /* instr.dma.dst.header_id. */
10226 "\t\t\t\t.header_id = {");
10228 for (i = 0; i < n_dma; i++)
10231 instr->dma.dst.header_id[i]);
10236 /* instr.dma.dst.struct_id. */
10238 "\t\t\t\t.struct_id = {");
10240 for (i = 0; i < n_dma; i++)
10243 instr->dma.dst.struct_id[i]);
10248 /* instr.dma.dst - closing curly brace. */
10252 /* instr.dma.src. */
10254 "\t\t\t.src = {\n");
10256 /* instr.dma.src.offset. */
10258 "\t\t\t\t.offset = {");
10260 for (i = 0; i < n_dma; i++)
10263 instr->dma.src.offset[i]);
10268 /* instr.dma.src - closing curly brace. */
10272 /* instr.dma.n_bytes. */
10274 "\t\t\t.n_bytes = {");
10276 for (i = 0; i < n_dma; i++)
10279 instr->dma.n_bytes[i]);
10284 /* instr.dma - closing curly brace. */
10288 /* instr - closing curly brace. */
10294 instr_alu_export(struct instruction *instr, FILE *f)
10298 if (instr->type == INSTR_ALU_ADD_MI ||
10299 instr->type == INSTR_ALU_ADD_HI ||
10300 instr->type == INSTR_ALU_SUB_MI ||
10301 instr->type == INSTR_ALU_SUB_HI ||
10302 instr->type == INSTR_ALU_SHL_MI ||
10303 instr->type == INSTR_ALU_SHL_HI ||
10304 instr->type == INSTR_ALU_SHR_MI ||
10305 instr->type == INSTR_ALU_SHR_HI ||
10306 instr->type == INSTR_ALU_AND_I ||
10307 instr->type == INSTR_ALU_OR_I ||
10308 instr->type == INSTR_ALU_XOR_I)
10314 "\t\t.type = %s,\n"
10317 "\t\t\t\t.struct_id = %u,\n"
10318 "\t\t\t\t.n_bits = %u,\n"
10319 "\t\t\t\t.offset = %u,\n"
10322 "\t\t\t\t.struct_id = %u,\n"
10323 "\t\t\t\t.n_bits = %u,\n"
10324 "\t\t\t\t.offset = %u,\n"
10328 instr_type_to_name(instr),
10329 instr->alu.dst.struct_id,
10330 instr->alu.dst.n_bits,
10331 instr->alu.dst.offset,
10332 instr->alu.src.struct_id,
10333 instr->alu.src.n_bits,
10334 instr->alu.src.offset);
10338 "\t\t.type = %s,\n"
10341 "\t\t\t\t.struct_id = %u,\n"
10342 "\t\t\t\t.n_bits = %u,\n"
10343 "\t\t\t\t.offset = %u,\n"
10345 "\t\t\t.src_val = %" PRIu64 ",\n"
10348 instr_type_to_name(instr),
10349 instr->alu.dst.struct_id,
10350 instr->alu.dst.n_bits,
10351 instr->alu.dst.offset,
10352 instr->alu.src_val);
10356 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10358 int prefetch = 0, idx_imm = 0, src_imm = 0;
10360 if (instr->type == INSTR_REGPREFETCH_RH ||
10361 instr->type == INSTR_REGPREFETCH_RM ||
10362 instr->type == INSTR_REGPREFETCH_RI)
10365 /* index is the 3rd operand for the regrd instruction and the 2nd
10366 * operand for the regwr and regadd instructions.
10368 if (instr->type == INSTR_REGPREFETCH_RI ||
10369 instr->type == INSTR_REGRD_HRI ||
10370 instr->type == INSTR_REGRD_MRI ||
10371 instr->type == INSTR_REGWR_RIH ||
10372 instr->type == INSTR_REGWR_RIM ||
10373 instr->type == INSTR_REGWR_RII ||
10374 instr->type == INSTR_REGADD_RIH ||
10375 instr->type == INSTR_REGADD_RIM ||
10376 instr->type == INSTR_REGADD_RII)
10379 /* src is the 3rd operand for the regwr and regadd instructions. */
10380 if (instr->type == INSTR_REGWR_RHI ||
10381 instr->type == INSTR_REGWR_RMI ||
10382 instr->type == INSTR_REGWR_RII ||
10383 instr->type == INSTR_REGADD_RHI ||
10384 instr->type == INSTR_REGADD_RMI ||
10385 instr->type == INSTR_REGADD_RII)
10388 /* instr.regarray.regarray_id. */
10391 "\t\t.type = %s,\n"
10392 "\t\t.regarray = {\n"
10393 "\t\t\t.regarray_id = %u,\n",
10394 instr_type_to_name(instr),
10395 instr->regarray.regarray_id);
10397 /* instr.regarray.idx / instr.regarray.idx_val. */
10400 "\t\t\t\t.idx = {\n"
10401 "\t\t\t\t\t.struct_id = %u,\n"
10402 "\t\t\t\t\t.n_bits = %u,\n"
10403 "\t\t\t\t\t.offset = %u,\n"
10405 instr->regarray.idx.struct_id,
10406 instr->regarray.idx.n_bits,
10407 instr->regarray.idx.offset);
10410 "\t\t\t\t.idx_val = %u,\n",
10411 instr->regarray.idx_val);
10413 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10417 "\t\t\t\t.dstsrc = {\n"
10418 "\t\t\t\t\t.struct_id = %u,\n"
10419 "\t\t\t\t\t.n_bits = %u,\n"
10420 "\t\t\t\t\t.offset = %u,\n"
10422 instr->regarray.dstsrc.struct_id,
10423 instr->regarray.dstsrc.n_bits,
10424 instr->regarray.dstsrc.offset);
10427 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10428 instr->regarray.dstsrc_val);
10431 /* instr.regarray and instr - closing curly braces. */
10438 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10440 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10442 if (instr->type == INSTR_METPREFETCH_H ||
10443 instr->type == INSTR_METPREFETCH_M ||
10444 instr->type == INSTR_METPREFETCH_I)
10448 if (instr->type == INSTR_METPREFETCH_I ||
10449 instr->type == INSTR_METER_IHM ||
10450 instr->type == INSTR_METER_IHI ||
10451 instr->type == INSTR_METER_IMM ||
10452 instr->type == INSTR_METER_IMI)
10455 /* color_in_imm. */
10456 if (instr->type == INSTR_METER_HHI ||
10457 instr->type == INSTR_METER_HMI ||
10458 instr->type == INSTR_METER_MHI ||
10459 instr->type == INSTR_METER_MMI ||
10460 instr->type == INSTR_METER_IHI ||
10461 instr->type == INSTR_METER_IMI)
10464 /* instr.meter.metarray_id. */
10467 "\t\t.type = %s,\n"
10469 "\t\t\t.metarray_id = %u,\n",
10470 instr_type_to_name(instr),
10471 instr->meter.metarray_id);
10473 /* instr.meter.idx / instr.meter.idx_val. */
10477 "\t\t\t\t.struct_id = %u,\n"
10478 "\t\t\t\t.n_bits = %u,\n"
10479 "\t\t\t\t.offset = %u,\n"
10481 instr->meter.idx.struct_id,
10482 instr->meter.idx.n_bits,
10483 instr->meter.idx.offset);
10486 "\t\t\t.idx_val = %u,\n",
10487 instr->meter.idx_val);
10490 /* instr.meter.length. */
10492 "\t\t\t.length = {\n"
10493 "\t\t\t\t.struct_id = %u,\n"
10494 "\t\t\t\t.n_bits = %u,\n"
10495 "\t\t\t\t.offset = %u,\n"
10497 instr->meter.length.struct_id,
10498 instr->meter.length.n_bits,
10499 instr->meter.length.offset);
10501 /* instr.meter.color_in / instr.meter.color_in_val. */
10504 "\t\t\t.color_in = {\n"
10505 "\t\t\t\t.struct_id = %u,\n"
10506 "\t\t\t\t.n_bits = %u,\n"
10507 "\t\t\t\t.offset = %u,\n"
10509 instr->meter.color_in.struct_id,
10510 instr->meter.color_in.n_bits,
10511 instr->meter.color_in.offset);
10514 "\t\t\t.color_in_val = %u,\n",
10515 (uint32_t)instr->meter.color_in_val);
10517 /* instr.meter.color_out. */
10519 "\t\t\t.color_out = {\n"
10520 "\t\t\t\t.struct_id = %u,\n"
10521 "\t\t\t\t.n_bits = %u,\n"
10522 "\t\t\t\t.offset = %u,\n"
10524 instr->meter.color_out.struct_id,
10525 instr->meter.color_out.n_bits,
10526 instr->meter.color_out.offset);
10529 /* instr.meter and instr - closing curly braces. */
10536 instr_table_export(struct instruction *instr,
10541 "\t\t.type = %s,\n"
10543 "\t\t\t.table_id = %u,\n"
10546 instr_type_to_name(instr),
10547 instr->table.table_id);
10551 instr_learn_export(struct instruction *instr, FILE *f)
10555 "\t\t.type = %s,\n"
10557 "\t\t\t\t.action_id = %u,\n"
10560 instr_type_to_name(instr),
10561 instr->learn.action_id);
10565 instr_forget_export(struct instruction *instr, FILE *f)
10569 "\t\t.type = %s,\n"
10571 instr_type_to_name(instr));
10575 instr_extern_export(struct instruction *instr, FILE *f)
10577 if (instr->type == INSTR_EXTERN_OBJ)
10580 "\t\t.type = %s,\n"
10581 "\t\t.ext_obj = {\n"
10582 "\t\t\t.ext_obj_id = %u,\n"
10583 "\t\t\t.func_id = %u,\n"
10586 instr_type_to_name(instr),
10587 instr->ext_obj.ext_obj_id,
10588 instr->ext_obj.func_id);
10592 "\t\t.type = %s,\n"
10593 "\t\t.ext_func = {\n"
10594 "\t\t\t.ext_func_id = %u,\n"
10597 instr_type_to_name(instr),
10598 instr->ext_func.ext_func_id);
10602 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10606 "\t\t.type = %s,\n"
10608 "\t\t\t.ip = NULL,\n",
10609 instr_type_to_name(instr));
10611 switch (instr->type) {
10612 case INSTR_JMP_VALID:
10613 case INSTR_JMP_INVALID:
10615 "\t\t\t.header_id = %u,\n",
10616 instr->jmp.header_id);
10619 case INSTR_JMP_ACTION_HIT:
10620 case INSTR_JMP_ACTION_MISS:
10622 "\t\t\t.action_id = %u,\n",
10623 instr->jmp.action_id);
10627 case INSTR_JMP_EQ_MH:
10628 case INSTR_JMP_EQ_HM:
10629 case INSTR_JMP_EQ_HH:
10630 case INSTR_JMP_NEQ:
10631 case INSTR_JMP_NEQ_MH:
10632 case INSTR_JMP_NEQ_HM:
10633 case INSTR_JMP_NEQ_HH:
10635 case INSTR_JMP_LT_MH:
10636 case INSTR_JMP_LT_HM:
10637 case INSTR_JMP_LT_HH:
10639 case INSTR_JMP_GT_MH:
10640 case INSTR_JMP_GT_HM:
10641 case INSTR_JMP_GT_HH:
10644 "\t\t\t\t.struct_id = %u,\n"
10645 "\t\t\t\t.n_bits = %u,\n"
10646 "\t\t\t\t.offset = %u,\n"
10649 "\t\t\t\t.struct_id = %u,\n"
10650 "\t\t\t\t.n_bits = %u,\n"
10651 "\t\t\t\t.offset = %u,\n"
10653 instr->jmp.a.struct_id,
10654 instr->jmp.a.n_bits,
10655 instr->jmp.a.offset,
10656 instr->jmp.b.struct_id,
10657 instr->jmp.b.n_bits,
10658 instr->jmp.b.offset);
10661 case INSTR_JMP_EQ_I:
10662 case INSTR_JMP_NEQ_I:
10663 case INSTR_JMP_LT_MI:
10664 case INSTR_JMP_LT_HI:
10665 case INSTR_JMP_GT_MI:
10666 case INSTR_JMP_GT_HI:
10669 "\t\t\t\t.struct_id = %u,\n"
10670 "\t\t\t\t.n_bits = %u,\n"
10671 "\t\t\t\t.offset = %u,\n"
10673 "\t\t\t.b_val = %" PRIu64 ",\n",
10674 instr->jmp.a.struct_id,
10675 instr->jmp.a.n_bits,
10676 instr->jmp.a.offset,
10690 instr_return_export(struct instruction *instr,
10695 "\t\t.type = %s,\n",
10696 instr_type_to_name(instr));
10702 static instruction_export_t export_table[] = {
10703 [INSTR_RX] = instr_io_export,
10705 [INSTR_TX] = instr_io_export,
10706 [INSTR_TX_I] = instr_io_export,
10708 [INSTR_HDR_EXTRACT] = instr_io_export,
10709 [INSTR_HDR_EXTRACT2] = instr_io_export,
10710 [INSTR_HDR_EXTRACT3] = instr_io_export,
10711 [INSTR_HDR_EXTRACT4] = instr_io_export,
10712 [INSTR_HDR_EXTRACT5] = instr_io_export,
10713 [INSTR_HDR_EXTRACT6] = instr_io_export,
10714 [INSTR_HDR_EXTRACT7] = instr_io_export,
10715 [INSTR_HDR_EXTRACT8] = instr_io_export,
10717 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10719 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10721 [INSTR_HDR_EMIT] = instr_io_export,
10722 [INSTR_HDR_EMIT_TX] = instr_io_export,
10723 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10724 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10725 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10726 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10727 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10728 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10729 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10731 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10732 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10734 [INSTR_MOV] = instr_mov_export,
10735 [INSTR_MOV_MH] = instr_mov_export,
10736 [INSTR_MOV_HM] = instr_mov_export,
10737 [INSTR_MOV_HH] = instr_mov_export,
10738 [INSTR_MOV_I] = instr_mov_export,
10740 [INSTR_DMA_HT] = instr_dma_ht_export,
10741 [INSTR_DMA_HT2] = instr_dma_ht_export,
10742 [INSTR_DMA_HT3] = instr_dma_ht_export,
10743 [INSTR_DMA_HT4] = instr_dma_ht_export,
10744 [INSTR_DMA_HT5] = instr_dma_ht_export,
10745 [INSTR_DMA_HT6] = instr_dma_ht_export,
10746 [INSTR_DMA_HT7] = instr_dma_ht_export,
10747 [INSTR_DMA_HT8] = instr_dma_ht_export,
10749 [INSTR_ALU_ADD] = instr_alu_export,
10750 [INSTR_ALU_ADD_MH] = instr_alu_export,
10751 [INSTR_ALU_ADD_HM] = instr_alu_export,
10752 [INSTR_ALU_ADD_HH] = instr_alu_export,
10753 [INSTR_ALU_ADD_MI] = instr_alu_export,
10754 [INSTR_ALU_ADD_HI] = instr_alu_export,
10756 [INSTR_ALU_SUB] = instr_alu_export,
10757 [INSTR_ALU_SUB_MH] = instr_alu_export,
10758 [INSTR_ALU_SUB_HM] = instr_alu_export,
10759 [INSTR_ALU_SUB_HH] = instr_alu_export,
10760 [INSTR_ALU_SUB_MI] = instr_alu_export,
10761 [INSTR_ALU_SUB_HI] = instr_alu_export,
10763 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10764 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10765 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10766 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10768 [INSTR_ALU_AND] = instr_alu_export,
10769 [INSTR_ALU_AND_MH] = instr_alu_export,
10770 [INSTR_ALU_AND_HM] = instr_alu_export,
10771 [INSTR_ALU_AND_HH] = instr_alu_export,
10772 [INSTR_ALU_AND_I] = instr_alu_export,
10774 [INSTR_ALU_OR] = instr_alu_export,
10775 [INSTR_ALU_OR_MH] = instr_alu_export,
10776 [INSTR_ALU_OR_HM] = instr_alu_export,
10777 [INSTR_ALU_OR_HH] = instr_alu_export,
10778 [INSTR_ALU_OR_I] = instr_alu_export,
10780 [INSTR_ALU_XOR] = instr_alu_export,
10781 [INSTR_ALU_XOR_MH] = instr_alu_export,
10782 [INSTR_ALU_XOR_HM] = instr_alu_export,
10783 [INSTR_ALU_XOR_HH] = instr_alu_export,
10784 [INSTR_ALU_XOR_I] = instr_alu_export,
10786 [INSTR_ALU_SHL] = instr_alu_export,
10787 [INSTR_ALU_SHL_MH] = instr_alu_export,
10788 [INSTR_ALU_SHL_HM] = instr_alu_export,
10789 [INSTR_ALU_SHL_HH] = instr_alu_export,
10790 [INSTR_ALU_SHL_MI] = instr_alu_export,
10791 [INSTR_ALU_SHL_HI] = instr_alu_export,
10793 [INSTR_ALU_SHR] = instr_alu_export,
10794 [INSTR_ALU_SHR_MH] = instr_alu_export,
10795 [INSTR_ALU_SHR_HM] = instr_alu_export,
10796 [INSTR_ALU_SHR_HH] = instr_alu_export,
10797 [INSTR_ALU_SHR_MI] = instr_alu_export,
10798 [INSTR_ALU_SHR_HI] = instr_alu_export,
10800 [INSTR_REGPREFETCH_RH] = instr_reg_export,
10801 [INSTR_REGPREFETCH_RM] = instr_reg_export,
10802 [INSTR_REGPREFETCH_RI] = instr_reg_export,
10804 [INSTR_REGRD_HRH] = instr_reg_export,
10805 [INSTR_REGRD_HRM] = instr_reg_export,
10806 [INSTR_REGRD_MRH] = instr_reg_export,
10807 [INSTR_REGRD_MRM] = instr_reg_export,
10808 [INSTR_REGRD_HRI] = instr_reg_export,
10809 [INSTR_REGRD_MRI] = instr_reg_export,
10811 [INSTR_REGWR_RHH] = instr_reg_export,
10812 [INSTR_REGWR_RHM] = instr_reg_export,
10813 [INSTR_REGWR_RMH] = instr_reg_export,
10814 [INSTR_REGWR_RMM] = instr_reg_export,
10815 [INSTR_REGWR_RHI] = instr_reg_export,
10816 [INSTR_REGWR_RMI] = instr_reg_export,
10817 [INSTR_REGWR_RIH] = instr_reg_export,
10818 [INSTR_REGWR_RIM] = instr_reg_export,
10819 [INSTR_REGWR_RII] = instr_reg_export,
10821 [INSTR_REGADD_RHH] = instr_reg_export,
10822 [INSTR_REGADD_RHM] = instr_reg_export,
10823 [INSTR_REGADD_RMH] = instr_reg_export,
10824 [INSTR_REGADD_RMM] = instr_reg_export,
10825 [INSTR_REGADD_RHI] = instr_reg_export,
10826 [INSTR_REGADD_RMI] = instr_reg_export,
10827 [INSTR_REGADD_RIH] = instr_reg_export,
10828 [INSTR_REGADD_RIM] = instr_reg_export,
10829 [INSTR_REGADD_RII] = instr_reg_export,
10831 [INSTR_METPREFETCH_H] = instr_meter_export,
10832 [INSTR_METPREFETCH_M] = instr_meter_export,
10833 [INSTR_METPREFETCH_I] = instr_meter_export,
10835 [INSTR_METER_HHM] = instr_meter_export,
10836 [INSTR_METER_HHI] = instr_meter_export,
10837 [INSTR_METER_HMM] = instr_meter_export,
10838 [INSTR_METER_HMI] = instr_meter_export,
10839 [INSTR_METER_MHM] = instr_meter_export,
10840 [INSTR_METER_MHI] = instr_meter_export,
10841 [INSTR_METER_MMM] = instr_meter_export,
10842 [INSTR_METER_MMI] = instr_meter_export,
10843 [INSTR_METER_IHM] = instr_meter_export,
10844 [INSTR_METER_IHI] = instr_meter_export,
10845 [INSTR_METER_IMM] = instr_meter_export,
10846 [INSTR_METER_IMI] = instr_meter_export,
10848 [INSTR_TABLE] = instr_table_export,
10849 [INSTR_TABLE_AF] = instr_table_export,
10850 [INSTR_SELECTOR] = instr_table_export,
10851 [INSTR_LEARNER] = instr_table_export,
10852 [INSTR_LEARNER_AF] = instr_table_export,
10854 [INSTR_LEARNER_LEARN] = instr_learn_export,
10855 [INSTR_LEARNER_FORGET] = instr_forget_export,
10857 [INSTR_EXTERN_OBJ] = instr_extern_export,
10858 [INSTR_EXTERN_FUNC] = instr_extern_export,
10860 [INSTR_JMP] = instr_jmp_export,
10861 [INSTR_JMP_VALID] = instr_jmp_export,
10862 [INSTR_JMP_INVALID] = instr_jmp_export,
10863 [INSTR_JMP_HIT] = instr_jmp_export,
10864 [INSTR_JMP_MISS] = instr_jmp_export,
10865 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10866 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10868 [INSTR_JMP_EQ] = instr_jmp_export,
10869 [INSTR_JMP_EQ_MH] = instr_jmp_export,
10870 [INSTR_JMP_EQ_HM] = instr_jmp_export,
10871 [INSTR_JMP_EQ_HH] = instr_jmp_export,
10872 [INSTR_JMP_EQ_I] = instr_jmp_export,
10874 [INSTR_JMP_NEQ] = instr_jmp_export,
10875 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10876 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10877 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10878 [INSTR_JMP_NEQ_I] = instr_jmp_export,
10880 [INSTR_JMP_LT] = instr_jmp_export,
10881 [INSTR_JMP_LT_MH] = instr_jmp_export,
10882 [INSTR_JMP_LT_HM] = instr_jmp_export,
10883 [INSTR_JMP_LT_HH] = instr_jmp_export,
10884 [INSTR_JMP_LT_MI] = instr_jmp_export,
10885 [INSTR_JMP_LT_HI] = instr_jmp_export,
10887 [INSTR_JMP_GT] = instr_jmp_export,
10888 [INSTR_JMP_GT_MH] = instr_jmp_export,
10889 [INSTR_JMP_GT_HM] = instr_jmp_export,
10890 [INSTR_JMP_GT_HH] = instr_jmp_export,
10891 [INSTR_JMP_GT_MI] = instr_jmp_export,
10892 [INSTR_JMP_GT_HI] = instr_jmp_export,
10894 [INSTR_RETURN] = instr_return_export,
10898 action_data_codegen(struct action *a, FILE *f)
10903 "static const struct instruction action_%s_instructions[] = {\n",
10906 for (i = 0; i < a->n_instructions; i++) {
10907 struct instruction *instr = &a->instructions[i];
10908 instruction_export_t func = export_table[instr->type];
10913 fprintf(f, "};\n");
10916 static const char *
10917 instr_type_to_func(struct instruction *instr)
10919 switch (instr->type) {
10920 case INSTR_RX: return NULL;
10922 case INSTR_TX: return "__instr_tx_exec";
10923 case INSTR_TX_I: return "__instr_tx_i_exec";
10925 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10926 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10927 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10928 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10929 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10930 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10931 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10932 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10934 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10936 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
10938 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
10939 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
10940 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
10941 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
10942 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
10943 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
10944 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
10945 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
10946 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
10948 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
10949 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
10951 case INSTR_MOV: return "__instr_mov_exec";
10952 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
10953 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
10954 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
10955 case INSTR_MOV_I: return "__instr_mov_i_exec";
10957 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
10958 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
10959 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
10960 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
10961 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
10962 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
10963 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
10964 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
10966 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
10967 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
10968 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
10969 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
10970 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
10971 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
10973 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
10974 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
10975 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
10976 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
10977 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
10978 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
10980 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
10981 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
10982 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
10983 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
10985 case INSTR_ALU_AND: return "__instr_alu_and_exec";
10986 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
10987 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
10988 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
10989 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
10991 case INSTR_ALU_OR: return "__instr_alu_or_exec";
10992 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
10993 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
10994 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
10995 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
10997 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
10998 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
10999 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11000 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11001 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11003 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11004 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11005 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11006 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11007 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11008 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11010 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11011 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11012 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11013 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11014 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11015 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11017 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11018 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11019 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11021 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11022 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11023 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11024 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11025 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11026 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11028 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11029 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11030 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11031 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11032 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11033 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11034 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11035 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11036 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11038 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11039 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11040 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11041 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11042 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11043 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11044 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11045 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11046 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11048 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11049 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11050 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11052 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11053 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11054 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11055 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11056 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11057 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11058 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11059 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11060 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11061 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11062 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11063 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11065 case INSTR_TABLE: return NULL;
11066 case INSTR_TABLE_AF: return NULL;
11067 case INSTR_SELECTOR: return NULL;
11068 case INSTR_LEARNER: return NULL;
11069 case INSTR_LEARNER_AF: return NULL;
11071 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11072 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11074 case INSTR_EXTERN_OBJ: return NULL;
11075 case INSTR_EXTERN_FUNC: return NULL;
11077 case INSTR_JMP: return NULL;
11078 case INSTR_JMP_VALID: return NULL;
11079 case INSTR_JMP_INVALID: return NULL;
11080 case INSTR_JMP_HIT: return NULL;
11081 case INSTR_JMP_MISS: return NULL;
11082 case INSTR_JMP_ACTION_HIT: return NULL;
11083 case INSTR_JMP_ACTION_MISS: return NULL;
11084 case INSTR_JMP_EQ: return NULL;
11085 case INSTR_JMP_EQ_MH: return NULL;
11086 case INSTR_JMP_EQ_HM: return NULL;
11087 case INSTR_JMP_EQ_HH: return NULL;
11088 case INSTR_JMP_EQ_I: return NULL;
11089 case INSTR_JMP_NEQ: return NULL;
11090 case INSTR_JMP_NEQ_MH: return NULL;
11091 case INSTR_JMP_NEQ_HM: return NULL;
11092 case INSTR_JMP_NEQ_HH: return NULL;
11093 case INSTR_JMP_NEQ_I: return NULL;
11094 case INSTR_JMP_LT: return NULL;
11095 case INSTR_JMP_LT_MH: return NULL;
11096 case INSTR_JMP_LT_HM: return NULL;
11097 case INSTR_JMP_LT_HH: return NULL;
11098 case INSTR_JMP_LT_MI: return NULL;
11099 case INSTR_JMP_LT_HI: return NULL;
11100 case INSTR_JMP_GT: return NULL;
11101 case INSTR_JMP_GT_MH: return NULL;
11102 case INSTR_JMP_GT_HM: return NULL;
11103 case INSTR_JMP_GT_HH: return NULL;
11104 case INSTR_JMP_GT_MI: return NULL;
11105 case INSTR_JMP_GT_HI: return NULL;
11107 case INSTR_RETURN: return NULL;
11109 default: return NULL;
11114 action_instr_does_tx_codegen(struct action *a,
11115 uint32_t instr_pos,
11116 struct instruction *instr,
11120 "%s(p, t, &action_%s_instructions[%u]);\n"
11121 "\tthread_ip_reset(p, t);\n"
11122 "\tinstr_rx_exec(p);\n"
11124 instr_type_to_func(instr),
11130 action_instr_extern_obj_codegen(struct action *a,
11131 uint32_t instr_pos,
11135 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11141 action_instr_extern_func_codegen(struct action *a,
11142 uint32_t instr_pos,
11146 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11152 action_instr_jmp_codegen(struct action *a,
11153 uint32_t instr_pos,
11154 struct instruction *instr,
11155 struct instruction_data *data,
11158 switch (instr->type) {
11165 case INSTR_JMP_VALID:
11167 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11174 case INSTR_JMP_INVALID:
11176 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11183 case INSTR_JMP_HIT:
11190 case INSTR_JMP_MISS:
11197 case INSTR_JMP_ACTION_HIT:
11199 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11206 case INSTR_JMP_ACTION_MISS:
11208 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11217 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11218 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11227 case INSTR_JMP_EQ_MH:
11229 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11230 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11239 case INSTR_JMP_EQ_HM:
11241 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11242 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11251 case INSTR_JMP_EQ_HH:
11253 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11254 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11263 case INSTR_JMP_EQ_I:
11265 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11266 "action_%s_instructions[%u].jmp.b_val)\n"
11275 case INSTR_JMP_NEQ:
11277 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11278 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11287 case INSTR_JMP_NEQ_MH:
11289 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11290 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11299 case INSTR_JMP_NEQ_HM:
11301 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11302 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11311 case INSTR_JMP_NEQ_HH:
11313 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11314 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11323 case INSTR_JMP_NEQ_I:
11325 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11326 "action_%s_instructions[%u].jmp.b_val)\n"
11337 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11338 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11347 case INSTR_JMP_LT_MH:
11349 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11350 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11359 case INSTR_JMP_LT_HM:
11361 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11362 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11371 case INSTR_JMP_LT_HH:
11373 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11374 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11383 case INSTR_JMP_LT_MI:
11385 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11386 "action_%s_instructions[%u].jmp.b_val)\n"
11395 case INSTR_JMP_LT_HI:
11397 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11398 "action_%s_instructions[%u].jmp.b_val)\n"
11409 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11410 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11419 case INSTR_JMP_GT_MH:
11421 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11422 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11431 case INSTR_JMP_GT_HM:
11433 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11434 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11443 case INSTR_JMP_GT_HH:
11445 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11446 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11455 case INSTR_JMP_GT_MI:
11457 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11458 "action_%s_instructions[%u].jmp.b_val)\n"
11467 case INSTR_JMP_GT_HI:
11469 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11470 "action_%s_instructions[%u].jmp.b_val)\n"
11485 action_instr_return_codegen(FILE *f)
11492 action_instr_codegen(struct action *a, FILE *f)
11498 "action_%s_run(struct rte_swx_pipeline *p)\n"
11500 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11504 for (i = 0; i < a->n_instructions; i++) {
11505 struct instruction *instr = &a->instructions[i];
11506 struct instruction_data *data = &a->instruction_data[i];
11508 /* Label, if present. */
11509 if (data->label[0])
11510 fprintf(f, "\n%s : ", data->label);
11512 fprintf(f, "\n\t");
11514 /* TX instruction type. */
11515 if (instruction_does_tx(instr)) {
11516 action_instr_does_tx_codegen(a, i, instr, f);
11520 /* Extern object/function instruction type. */
11521 if (instr->type == INSTR_EXTERN_OBJ) {
11522 action_instr_extern_obj_codegen(a, i, f);
11526 if (instr->type == INSTR_EXTERN_FUNC) {
11527 action_instr_extern_func_codegen(a, i, f);
11531 /* Jump instruction type. */
11532 if (instruction_is_jmp(instr)) {
11533 action_instr_jmp_codegen(a, i, instr, data, f);
11537 /* Return instruction type. */
11538 if (instr->type == INSTR_RETURN) {
11539 action_instr_return_codegen(f);
11543 /* Any other instruction type. */
11545 "%s(p, t, &action_%s_instructions[%u]);\n",
11546 instr_type_to_func(instr),
11551 fprintf(f, "}\n\n");
11554 struct instruction_group {
11555 TAILQ_ENTRY(instruction_group) node;
11559 uint32_t first_instr_id;
11561 uint32_t last_instr_id;
11566 TAILQ_HEAD(instruction_group_list, instruction_group);
11568 static struct instruction_group *
11569 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11571 struct instruction_group *g;
11573 TAILQ_FOREACH(g, igl, node)
11574 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11581 instruction_group_list_free(struct instruction_group_list *igl)
11587 struct instruction_group *g;
11589 g = TAILQ_FIRST(igl);
11593 TAILQ_REMOVE(igl, g, node);
11600 static struct instruction_group_list *
11601 instruction_group_list_create(struct rte_swx_pipeline *p)
11603 struct instruction_group_list *igl = NULL;
11604 struct instruction_group *g = NULL;
11605 uint32_t n_groups = 0, i;
11607 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11611 igl = calloc(1, sizeof(struct instruction_group_list));
11617 /* Allocate the first group. */
11618 g = calloc(1, sizeof(struct instruction_group));
11622 /* Iteration 1: Separate the instructions into groups based on the thread yield
11623 * instructions. Do not worry about the jump instructions at this point.
11625 for (i = 0; i < p->n_instructions; i++) {
11626 struct instruction *instr = &p->instructions[i];
11628 /* Check for thread yield instructions. */
11629 if (!instruction_does_thread_yield(instr))
11632 /* If the current group contains at least one instruction, then finalize it (with
11633 * the previous instruction), add it to the list and allocate a new group (that
11634 * starts with the current instruction).
11636 if (i - g->first_instr_id) {
11637 /* Finalize the group. */
11638 g->last_instr_id = i - 1;
11640 /* Add the group to the list. Advance the number of groups. */
11641 TAILQ_INSERT_TAIL(igl, g, node);
11644 /* Allocate a new group. */
11645 g = calloc(1, sizeof(struct instruction_group));
11649 /* Initialize the new group. */
11650 g->group_id = n_groups;
11651 g->first_instr_id = i;
11654 /* Finalize the current group (with the current instruction, therefore this group
11655 * contains just the current thread yield instruction), add it to the list and
11656 * allocate a new group (that starts with the next instruction).
11659 /* Finalize the group. */
11660 g->last_instr_id = i;
11662 /* Add the group to the list. Advance the number of groups. */
11663 TAILQ_INSERT_TAIL(igl, g, node);
11666 /* Allocate a new group. */
11667 g = calloc(1, sizeof(struct instruction_group));
11671 /* Initialize the new group. */
11672 g->group_id = n_groups;
11673 g->first_instr_id = i + 1;
11676 /* Handle the last group. */
11677 if (i - g->first_instr_id) {
11678 /* Finalize the group. */
11679 g->last_instr_id = i - 1;
11681 /* Add the group to the list. Advance the number of groups. */
11682 TAILQ_INSERT_TAIL(igl, g, node);
11689 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11690 * the destination of a jump instruction located in a different group ("far jump"), then the
11691 * current group has to be split, so that the instruction representing the far jump
11692 * destination is at the start of its group.
11695 int is_modified = 0;
11697 for (i = 0; i < p->n_instructions; i++) {
11698 struct instruction_data *data = &p->instruction_data[i];
11699 struct instruction_group *g;
11702 /* Continue when the current instruction is not a jump destination. */
11703 if (!data->n_users)
11706 g = instruction_group_list_group_find(igl, i);
11710 /* Find out all the jump instructions with this destination. */
11711 for (j = 0; j < p->n_instructions; j++) {
11712 struct instruction *jmp_instr = &p->instructions[j];
11713 struct instruction_data *jmp_data = &p->instruction_data[j];
11714 struct instruction_group *jmp_g, *new_g;
11716 /* Continue when not a jump instruction. Even when jump instruction,
11717 * continue when the jump destination is not this instruction.
11719 if (!instruction_is_jmp(jmp_instr) ||
11720 strcmp(jmp_data->jmp_label, data->label))
11723 jmp_g = instruction_group_list_group_find(igl, j);
11727 /* Continue when both the jump instruction and the jump destination
11728 * instruction are in the same group. Even when in different groups,
11729 * still continue if the jump destination instruction is already the
11730 * first instruction of its group.
11732 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11735 /* Split the group of the current jump destination instruction to
11736 * make this instruction the first instruction of a new group.
11738 new_g = calloc(1, sizeof(struct instruction_group));
11742 new_g->group_id = n_groups;
11743 new_g->first_instr_id = i;
11744 new_g->last_instr_id = g->last_instr_id;
11746 g->last_instr_id = i - 1;
11748 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11752 /* The decision to split this group (to make the current instruction
11753 * the first instruction of a new group) is already taken and fully
11754 * implemented, so no need to search for more reasons to do it.
11760 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11761 * previously considered local (i.e. the jump destination is in the same group as
11762 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11763 * different group than the jump instruction). Wost case scenario: each instruction
11764 * that is a jump destination ends up as the first instruction of its group.
11770 /* Re-assign the group IDs to be in incremental order. */
11772 TAILQ_FOREACH(g, igl, node) {
11781 instruction_group_list_free(igl);
11789 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11790 uint32_t instr_pos,
11791 struct instruction *instr,
11795 "%s(p, t, &pipeline_instructions[%u]);\n"
11796 "\tthread_ip_reset(p, t);\n"
11797 "\tinstr_rx_exec(p);\n"
11799 instr_type_to_func(instr),
11804 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11805 struct instruction_group_list *igl,
11806 uint32_t jmp_instr_id,
11807 struct instruction *jmp_instr,
11808 struct instruction_data *jmp_data,
11811 struct instruction_group *jmp_g, *g;
11812 struct instruction_data *data;
11815 switch (jmp_instr->type) {
11819 case INSTR_JMP_VALID:
11821 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11825 case INSTR_JMP_INVALID:
11827 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11831 case INSTR_JMP_HIT:
11836 case INSTR_JMP_MISS:
11841 case INSTR_JMP_ACTION_HIT:
11843 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11847 case INSTR_JMP_ACTION_MISS:
11849 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11855 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11856 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11861 case INSTR_JMP_EQ_MH:
11863 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11864 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11869 case INSTR_JMP_EQ_HM:
11871 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11872 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11877 case INSTR_JMP_EQ_HH:
11879 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11880 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11885 case INSTR_JMP_EQ_I:
11887 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11888 "pipeline_instructions[%u].jmp.b_val)",
11893 case INSTR_JMP_NEQ:
11895 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11896 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11901 case INSTR_JMP_NEQ_MH:
11903 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11904 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11909 case INSTR_JMP_NEQ_HM:
11911 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11912 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11917 case INSTR_JMP_NEQ_HH:
11919 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11920 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11925 case INSTR_JMP_NEQ_I:
11927 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11928 "pipeline_instructions[%u].jmp.b_val)",
11935 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11936 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11941 case INSTR_JMP_LT_MH:
11943 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11944 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11949 case INSTR_JMP_LT_HM:
11951 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11952 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11957 case INSTR_JMP_LT_HH:
11959 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11960 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11965 case INSTR_JMP_LT_MI:
11967 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11968 "pipeline_instructions[%u].jmp.b_val)",
11973 case INSTR_JMP_LT_HI:
11975 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11976 "pipeline_instructions[%u].jmp.b_val)",
11983 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
11984 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11989 case INSTR_JMP_GT_MH:
11991 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
11992 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11997 case INSTR_JMP_GT_HM:
11999 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12000 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12005 case INSTR_JMP_GT_HH:
12007 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12008 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12013 case INSTR_JMP_GT_MI:
12015 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12016 "pipeline_instructions[%u].jmp.b_val)",
12021 case INSTR_JMP_GT_HI:
12023 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12024 "pipeline_instructions[%u].jmp.b_val)",
12033 /* Find the instruction group of the jump instruction. */
12034 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12038 /* Find the instruction group of the jump destination instruction. */
12039 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12043 instr_id = data - p->instruction_data;
12045 g = instruction_group_list_group_find(igl, instr_id);
12049 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12050 * instruction group).
12052 if (g->group_id == jmp_g->group_id)
12054 "\n\t\tgoto %s;\n",
12055 jmp_data->jmp_label);
12059 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12068 instruction_group_list_codegen(struct instruction_group_list *igl,
12069 struct rte_swx_pipeline *p,
12072 struct instruction_group *g;
12074 int is_required = 0;
12076 /* Check if code generation is required. */
12077 TAILQ_FOREACH(g, igl, node)
12078 if (g->first_instr_id < g->last_instr_id)
12084 /* Generate the code for the pipeline instruction array. */
12086 "static const struct instruction pipeline_instructions[] = {\n");
12088 for (i = 0; i < p->n_instructions; i++) {
12089 struct instruction *instr = &p->instructions[i];
12090 instruction_export_t func = export_table[instr->type];
12095 fprintf(f, "};\n\n");
12097 /* Generate the code for the pipeline functions: one function for each instruction group
12098 * that contains more than one instruction.
12100 TAILQ_FOREACH(g, igl, node) {
12101 struct instruction *last_instr;
12104 /* Skip if group contains a single instruction. */
12105 if (g->last_instr_id == g->first_instr_id)
12108 /* Generate new pipeline function. */
12111 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12113 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12117 /* Generate the code for each pipeline instruction. */
12118 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12119 struct instruction *instr = &p->instructions[j];
12120 struct instruction_data *data = &p->instruction_data[j];
12122 /* Label, if present. */
12123 if (data->label[0])
12124 fprintf(f, "\n%s : ", data->label);
12126 fprintf(f, "\n\t");
12128 /* TX instruction type. */
12129 if (instruction_does_tx(instr)) {
12130 pipeline_instr_does_tx_codegen(p, j, instr, f);
12134 /* Jump instruction type. */
12135 if (instruction_is_jmp(instr)) {
12136 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12140 /* Any other instruction type. */
12142 "%s(p, t, &pipeline_instructions[%u]);\n",
12143 instr_type_to_func(instr),
12147 /* Finalize the generated pipeline function. For some instructions such as TX,
12148 * emit-many-and-TX and unconditional jump, the next instruction has been already
12149 * decided unconditionally and the instruction pointer of the current thread set
12150 * accordingly; for all the other instructions, the instruction pointer must be
12153 last_instr = &p->instructions[g->last_instr_id];
12155 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12157 "thread_ip_inc(p);\n");
12166 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12168 struct instruction_group *g;
12169 uint32_t n_custom_instr = 0;
12171 /* Groups with a single instruction: no function is generated for this group, the group
12172 * keeps its current instruction. Groups with more than two instructions: one function and
12173 * the associated custom instruction get generated for each such group.
12175 TAILQ_FOREACH(g, igl, node) {
12176 if (g->first_instr_id == g->last_instr_id)
12182 return n_custom_instr;
12186 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12191 /* Create the .c file. */
12192 f = fopen("/tmp/pipeline.c", "w");
12196 /* Include the .h file. */
12197 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12199 /* Add the code for each action. */
12200 TAILQ_FOREACH(a, &p->actions, node) {
12201 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12203 action_data_codegen(a, f);
12207 action_instr_codegen(a, f);
12212 /* Add the pipeline code. */
12213 instruction_group_list_codegen(igl, p, f);
12215 /* Close the .c file. */
12221 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12222 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12226 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12229 struct instruction_group *g;
12230 char *dir_in, *buffer = NULL;
12231 const char *dir_out;
12234 /* Get the environment variables. */
12235 dir_in = getenv("RTE_INSTALL_DIR");
12243 /* Memory allocation for the command buffer. */
12244 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12251 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12252 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12253 "-I %s/lib/pipeline "
12254 "-I %s/lib/eal/include "
12255 "-I %s/lib/eal/x86/include "
12256 "-I %s/lib/eal/include/generic "
12260 "-I %s/lib/pipeline "
12263 "-I %s/lib/eal/linux/include "
12264 ">%s/pipeline.log 2>&1 "
12266 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12267 ">>%s/pipeline.log 2>&1",
12286 /* Build the shared object library. */
12287 status = system(buffer);
12291 /* Open library. */
12293 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12294 "%s/libpipeline.so",
12297 p->lib = dlopen(buffer, RTLD_LAZY);
12303 /* Get the action function symbols. */
12304 TAILQ_FOREACH(a, &p->actions, node) {
12305 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12307 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12308 if (!p->action_funcs[a->id]) {
12314 /* Get the pipeline function symbols. */
12315 TAILQ_FOREACH(g, igl, node) {
12316 if (g->first_instr_id == g->last_instr_id)
12319 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12321 g->func = dlsym(p->lib, buffer);
12329 if (status && p->lib) {
12340 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12341 struct instruction_group_list *igl)
12343 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12345 /* Check that enough space is available within the pipeline instruction table to store all
12346 * the custom instructions.
12348 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12355 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12357 struct instruction_group *g;
12360 /* Pipeline table instructions. */
12361 for (i = 0; i < p->n_instructions; i++) {
12362 struct instruction *instr = &p->instructions[i];
12364 if (instr->type == INSTR_TABLE)
12365 instr->type = INSTR_TABLE_AF;
12367 if (instr->type == INSTR_LEARNER)
12368 instr->type = INSTR_LEARNER_AF;
12371 /* Pipeline custom instructions. */
12373 TAILQ_FOREACH(g, igl, node) {
12374 struct instruction *instr = &p->instructions[g->first_instr_id];
12377 if (g->first_instr_id == g->last_instr_id)
12380 /* Install a new custom instruction. */
12381 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12383 /* First instruction of the group: change its type to the new custom instruction. */
12384 instr->type = INSTR_CUSTOM_0 + i;
12386 /* All the subsequent instructions of the group: invalidate. */
12387 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12388 struct instruction_data *data = &p->instruction_data[j];
12396 /* Remove the invalidated instructions. */
12397 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12399 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12400 * instructions that are the only instruction within their group, so they were left
12403 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12407 pipeline_compile(struct rte_swx_pipeline *p)
12409 struct instruction_group_list *igl = NULL;
12412 igl = instruction_group_list_create(p);
12418 /* Code generation. */
12419 status = pipeline_codegen(p, igl);
12423 /* Build and load the shared object library. */
12424 status = pipeline_libload(p, igl);
12428 /* Adjust instructions. */
12429 status = pipeline_adjust_check(p, igl);
12433 pipeline_adjust(p, igl);
12436 instruction_group_list_free(igl);