1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include "rte_swx_pipeline_internal.h"
12 #define CHECK(condition, err_code) \
18 #define CHECK_NAME(name, err_code) \
21 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
24 #define CHECK_INSTRUCTION(instr, err_code) \
27 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
28 RTE_SWX_INSTRUCTION_SIZE), \
34 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
36 #include <rte_malloc.h>
39 env_malloc(size_t size, size_t alignment, int numa_node)
41 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
45 env_free(void *start, size_t size __rte_unused)
55 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
59 if (numa_available() == -1)
62 start = numa_alloc_onnode(size, numa_node);
66 memset(start, 0, size);
71 env_free(void *start, size_t size)
73 if (numa_available() == -1)
76 numa_free(start, size);
84 static struct struct_type *
85 struct_type_find(struct rte_swx_pipeline *p, const char *name)
87 struct struct_type *elem;
89 TAILQ_FOREACH(elem, &p->struct_types, node)
90 if (strcmp(elem->name, name) == 0)
97 struct_type_field_find(struct struct_type *st, const char *name)
101 for (i = 0; i < st->n_fields; i++) {
102 struct field *f = &st->fields[i];
104 if (strcmp(f->name, name) == 0)
112 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
114 struct rte_swx_field_params *fields,
116 int last_field_has_variable_size)
118 struct struct_type *st;
122 CHECK_NAME(name, EINVAL);
123 CHECK(fields, EINVAL);
124 CHECK(n_fields, EINVAL);
126 for (i = 0; i < n_fields; i++) {
127 struct rte_swx_field_params *f = &fields[i];
128 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
131 CHECK_NAME(f->name, EINVAL);
132 CHECK(f->n_bits, EINVAL);
133 CHECK((f->n_bits <= 64) || var_size, EINVAL);
134 CHECK((f->n_bits & 7) == 0, EINVAL);
136 for (j = 0; j < i; j++) {
137 struct rte_swx_field_params *f_prev = &fields[j];
139 CHECK(strcmp(f->name, f_prev->name), EINVAL);
143 CHECK(!struct_type_find(p, name), EEXIST);
145 /* Node allocation. */
146 st = calloc(1, sizeof(struct struct_type));
149 st->fields = calloc(n_fields, sizeof(struct field));
155 /* Node initialization. */
156 strcpy(st->name, name);
157 for (i = 0; i < n_fields; i++) {
158 struct field *dst = &st->fields[i];
159 struct rte_swx_field_params *src = &fields[i];
160 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
162 strcpy(dst->name, src->name);
163 dst->n_bits = src->n_bits;
164 dst->offset = st->n_bits;
165 dst->var_size = var_size;
167 st->n_bits += src->n_bits;
168 st->n_bits_min += var_size ? 0 : src->n_bits;
170 st->n_fields = n_fields;
171 st->var_size = last_field_has_variable_size;
173 /* Node add to tailq. */
174 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
180 struct_build(struct rte_swx_pipeline *p)
184 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
185 struct thread *t = &p->threads[i];
187 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
188 CHECK(t->structs, ENOMEM);
195 struct_build_free(struct rte_swx_pipeline *p)
199 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
200 struct thread *t = &p->threads[i];
208 struct_free(struct rte_swx_pipeline *p)
210 struct_build_free(p);
214 struct struct_type *elem;
216 elem = TAILQ_FIRST(&p->struct_types);
220 TAILQ_REMOVE(&p->struct_types, elem, node);
229 static struct port_in_type *
230 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
232 struct port_in_type *elem;
237 TAILQ_FOREACH(elem, &p->port_in_types, node)
238 if (strcmp(elem->name, name) == 0)
245 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
247 struct rte_swx_port_in_ops *ops)
249 struct port_in_type *elem;
252 CHECK_NAME(name, EINVAL);
254 CHECK(ops->create, EINVAL);
255 CHECK(ops->free, EINVAL);
256 CHECK(ops->pkt_rx, EINVAL);
257 CHECK(ops->stats_read, EINVAL);
259 CHECK(!port_in_type_find(p, name), EEXIST);
261 /* Node allocation. */
262 elem = calloc(1, sizeof(struct port_in_type));
265 /* Node initialization. */
266 strcpy(elem->name, name);
267 memcpy(&elem->ops, ops, sizeof(*ops));
269 /* Node add to tailq. */
270 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
275 static struct port_in *
276 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
278 struct port_in *port;
280 TAILQ_FOREACH(port, &p->ports_in, node)
281 if (port->id == port_id)
288 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
290 const char *port_type_name,
293 struct port_in_type *type = NULL;
294 struct port_in *port = NULL;
299 CHECK(!port_in_find(p, port_id), EINVAL);
301 CHECK_NAME(port_type_name, EINVAL);
302 type = port_in_type_find(p, port_type_name);
305 obj = type->ops.create(args);
308 /* Node allocation. */
309 port = calloc(1, sizeof(struct port_in));
312 /* Node initialization. */
317 /* Node add to tailq. */
318 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
319 if (p->n_ports_in < port_id + 1)
320 p->n_ports_in = port_id + 1;
326 port_in_build(struct rte_swx_pipeline *p)
328 struct port_in *port;
331 CHECK(p->n_ports_in, EINVAL);
332 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
334 for (i = 0; i < p->n_ports_in; i++)
335 CHECK(port_in_find(p, i), EINVAL);
337 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
338 CHECK(p->in, ENOMEM);
340 TAILQ_FOREACH(port, &p->ports_in, node) {
341 struct port_in_runtime *in = &p->in[port->id];
343 in->pkt_rx = port->type->ops.pkt_rx;
351 port_in_build_free(struct rte_swx_pipeline *p)
358 port_in_free(struct rte_swx_pipeline *p)
360 port_in_build_free(p);
364 struct port_in *port;
366 port = TAILQ_FIRST(&p->ports_in);
370 TAILQ_REMOVE(&p->ports_in, port, node);
371 port->type->ops.free(port->obj);
375 /* Input port types. */
377 struct port_in_type *elem;
379 elem = TAILQ_FIRST(&p->port_in_types);
383 TAILQ_REMOVE(&p->port_in_types, elem, node);
391 static struct port_out_type *
392 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
394 struct port_out_type *elem;
399 TAILQ_FOREACH(elem, &p->port_out_types, node)
400 if (!strcmp(elem->name, name))
407 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
409 struct rte_swx_port_out_ops *ops)
411 struct port_out_type *elem;
414 CHECK_NAME(name, EINVAL);
416 CHECK(ops->create, EINVAL);
417 CHECK(ops->free, EINVAL);
418 CHECK(ops->pkt_tx, EINVAL);
419 CHECK(ops->stats_read, EINVAL);
421 CHECK(!port_out_type_find(p, name), EEXIST);
423 /* Node allocation. */
424 elem = calloc(1, sizeof(struct port_out_type));
427 /* Node initialization. */
428 strcpy(elem->name, name);
429 memcpy(&elem->ops, ops, sizeof(*ops));
431 /* Node add to tailq. */
432 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
437 static struct port_out *
438 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
440 struct port_out *port;
442 TAILQ_FOREACH(port, &p->ports_out, node)
443 if (port->id == port_id)
450 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
452 const char *port_type_name,
455 struct port_out_type *type = NULL;
456 struct port_out *port = NULL;
461 CHECK(!port_out_find(p, port_id), EINVAL);
463 CHECK_NAME(port_type_name, EINVAL);
464 type = port_out_type_find(p, port_type_name);
467 obj = type->ops.create(args);
470 /* Node allocation. */
471 port = calloc(1, sizeof(struct port_out));
474 /* Node initialization. */
479 /* Node add to tailq. */
480 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
481 if (p->n_ports_out < port_id + 1)
482 p->n_ports_out = port_id + 1;
488 port_out_build(struct rte_swx_pipeline *p)
490 struct port_out *port;
493 CHECK(p->n_ports_out, EINVAL);
495 for (i = 0; i < p->n_ports_out; i++)
496 CHECK(port_out_find(p, i), EINVAL);
498 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
499 CHECK(p->out, ENOMEM);
501 TAILQ_FOREACH(port, &p->ports_out, node) {
502 struct port_out_runtime *out = &p->out[port->id];
504 out->pkt_tx = port->type->ops.pkt_tx;
505 out->flush = port->type->ops.flush;
506 out->obj = port->obj;
513 port_out_build_free(struct rte_swx_pipeline *p)
520 port_out_free(struct rte_swx_pipeline *p)
522 port_out_build_free(p);
526 struct port_out *port;
528 port = TAILQ_FIRST(&p->ports_out);
532 TAILQ_REMOVE(&p->ports_out, port, node);
533 port->type->ops.free(port->obj);
537 /* Output port types. */
539 struct port_out_type *elem;
541 elem = TAILQ_FIRST(&p->port_out_types);
545 TAILQ_REMOVE(&p->port_out_types, elem, node);
553 static struct extern_type *
554 extern_type_find(struct rte_swx_pipeline *p, const char *name)
556 struct extern_type *elem;
558 TAILQ_FOREACH(elem, &p->extern_types, node)
559 if (strcmp(elem->name, name) == 0)
565 static struct extern_type_member_func *
566 extern_type_member_func_find(struct extern_type *type, const char *name)
568 struct extern_type_member_func *elem;
570 TAILQ_FOREACH(elem, &type->funcs, node)
571 if (strcmp(elem->name, name) == 0)
577 static struct extern_obj *
578 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
580 struct extern_obj *elem;
582 TAILQ_FOREACH(elem, &p->extern_objs, node)
583 if (strcmp(elem->name, name) == 0)
589 static struct extern_type_member_func *
590 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
592 struct extern_obj **obj)
594 struct extern_obj *object;
595 struct extern_type_member_func *func;
596 char *object_name, *func_name;
598 if (name[0] != 'e' || name[1] != '.')
601 object_name = strdup(&name[2]);
605 func_name = strchr(object_name, '.');
614 object = extern_obj_find(p, object_name);
620 func = extern_type_member_func_find(object->type, func_name);
633 static struct field *
634 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
636 struct extern_obj **object)
638 struct extern_obj *obj;
640 char *obj_name, *field_name;
642 if ((name[0] != 'e') || (name[1] != '.'))
645 obj_name = strdup(&name[2]);
649 field_name = strchr(obj_name, '.');
658 obj = extern_obj_find(p, obj_name);
664 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
678 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
680 const char *mailbox_struct_type_name,
681 rte_swx_extern_type_constructor_t constructor,
682 rte_swx_extern_type_destructor_t destructor)
684 struct extern_type *elem;
685 struct struct_type *mailbox_struct_type;
689 CHECK_NAME(name, EINVAL);
690 CHECK(!extern_type_find(p, name), EEXIST);
692 CHECK_NAME(mailbox_struct_type_name, EINVAL);
693 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
694 CHECK(mailbox_struct_type, EINVAL);
695 CHECK(!mailbox_struct_type->var_size, EINVAL);
697 CHECK(constructor, EINVAL);
698 CHECK(destructor, EINVAL);
700 /* Node allocation. */
701 elem = calloc(1, sizeof(struct extern_type));
704 /* Node initialization. */
705 strcpy(elem->name, name);
706 elem->mailbox_struct_type = mailbox_struct_type;
707 elem->constructor = constructor;
708 elem->destructor = destructor;
709 TAILQ_INIT(&elem->funcs);
711 /* Node add to tailq. */
712 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
718 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
719 const char *extern_type_name,
721 rte_swx_extern_type_member_func_t member_func)
723 struct extern_type *type;
724 struct extern_type_member_func *type_member;
728 CHECK_NAME(extern_type_name, EINVAL);
729 type = extern_type_find(p, extern_type_name);
731 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
733 CHECK_NAME(name, EINVAL);
734 CHECK(!extern_type_member_func_find(type, name), EEXIST);
736 CHECK(member_func, EINVAL);
738 /* Node allocation. */
739 type_member = calloc(1, sizeof(struct extern_type_member_func));
740 CHECK(type_member, ENOMEM);
742 /* Node initialization. */
743 strcpy(type_member->name, name);
744 type_member->func = member_func;
745 type_member->id = type->n_funcs;
747 /* Node add to tailq. */
748 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
755 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
756 const char *extern_type_name,
760 struct extern_type *type;
761 struct extern_obj *obj;
766 CHECK_NAME(extern_type_name, EINVAL);
767 type = extern_type_find(p, extern_type_name);
770 CHECK_NAME(name, EINVAL);
771 CHECK(!extern_obj_find(p, name), EEXIST);
773 /* Node allocation. */
774 obj = calloc(1, sizeof(struct extern_obj));
777 /* Object construction. */
778 obj_handle = type->constructor(args);
784 /* Node initialization. */
785 strcpy(obj->name, name);
787 obj->obj = obj_handle;
788 obj->struct_id = p->n_structs;
789 obj->id = p->n_extern_objs;
791 /* Node add to tailq. */
792 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
800 extern_obj_build(struct rte_swx_pipeline *p)
804 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
805 struct thread *t = &p->threads[i];
806 struct extern_obj *obj;
808 t->extern_objs = calloc(p->n_extern_objs,
809 sizeof(struct extern_obj_runtime));
810 CHECK(t->extern_objs, ENOMEM);
812 TAILQ_FOREACH(obj, &p->extern_objs, node) {
813 struct extern_obj_runtime *r =
814 &t->extern_objs[obj->id];
815 struct extern_type_member_func *func;
816 uint32_t mailbox_size =
817 obj->type->mailbox_struct_type->n_bits / 8;
821 r->mailbox = calloc(1, mailbox_size);
822 CHECK(r->mailbox, ENOMEM);
824 TAILQ_FOREACH(func, &obj->type->funcs, node)
825 r->funcs[func->id] = func->func;
827 t->structs[obj->struct_id] = r->mailbox;
835 extern_obj_build_free(struct rte_swx_pipeline *p)
839 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
840 struct thread *t = &p->threads[i];
846 for (j = 0; j < p->n_extern_objs; j++) {
847 struct extern_obj_runtime *r = &t->extern_objs[j];
852 free(t->extern_objs);
853 t->extern_objs = NULL;
858 extern_obj_free(struct rte_swx_pipeline *p)
860 extern_obj_build_free(p);
862 /* Extern objects. */
864 struct extern_obj *elem;
866 elem = TAILQ_FIRST(&p->extern_objs);
870 TAILQ_REMOVE(&p->extern_objs, elem, node);
872 elem->type->destructor(elem->obj);
878 struct extern_type *elem;
880 elem = TAILQ_FIRST(&p->extern_types);
884 TAILQ_REMOVE(&p->extern_types, elem, node);
887 struct extern_type_member_func *func;
889 func = TAILQ_FIRST(&elem->funcs);
893 TAILQ_REMOVE(&elem->funcs, func, node);
904 static struct extern_func *
905 extern_func_find(struct rte_swx_pipeline *p, const char *name)
907 struct extern_func *elem;
909 TAILQ_FOREACH(elem, &p->extern_funcs, node)
910 if (strcmp(elem->name, name) == 0)
916 static struct extern_func *
917 extern_func_parse(struct rte_swx_pipeline *p,
920 if (name[0] != 'f' || name[1] != '.')
923 return extern_func_find(p, &name[2]);
926 static struct field *
927 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
929 struct extern_func **function)
931 struct extern_func *func;
933 char *func_name, *field_name;
935 if ((name[0] != 'f') || (name[1] != '.'))
938 func_name = strdup(&name[2]);
942 field_name = strchr(func_name, '.');
951 func = extern_func_find(p, func_name);
957 f = struct_type_field_find(func->mailbox_struct_type, field_name);
971 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
973 const char *mailbox_struct_type_name,
974 rte_swx_extern_func_t func)
976 struct extern_func *f;
977 struct struct_type *mailbox_struct_type;
981 CHECK_NAME(name, EINVAL);
982 CHECK(!extern_func_find(p, name), EEXIST);
984 CHECK_NAME(mailbox_struct_type_name, EINVAL);
985 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
986 CHECK(mailbox_struct_type, EINVAL);
987 CHECK(!mailbox_struct_type->var_size, EINVAL);
991 /* Node allocation. */
992 f = calloc(1, sizeof(struct extern_func));
995 /* Node initialization. */
996 strcpy(f->name, name);
997 f->mailbox_struct_type = mailbox_struct_type;
999 f->struct_id = p->n_structs;
1000 f->id = p->n_extern_funcs;
1002 /* Node add to tailq. */
1003 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1004 p->n_extern_funcs++;
1011 extern_func_build(struct rte_swx_pipeline *p)
1015 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1016 struct thread *t = &p->threads[i];
1017 struct extern_func *func;
1019 /* Memory allocation. */
1020 t->extern_funcs = calloc(p->n_extern_funcs,
1021 sizeof(struct extern_func_runtime));
1022 CHECK(t->extern_funcs, ENOMEM);
1024 /* Extern function. */
1025 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1026 struct extern_func_runtime *r =
1027 &t->extern_funcs[func->id];
1028 uint32_t mailbox_size =
1029 func->mailbox_struct_type->n_bits / 8;
1031 r->func = func->func;
1033 r->mailbox = calloc(1, mailbox_size);
1034 CHECK(r->mailbox, ENOMEM);
1036 t->structs[func->struct_id] = r->mailbox;
1044 extern_func_build_free(struct rte_swx_pipeline *p)
1048 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1049 struct thread *t = &p->threads[i];
1052 if (!t->extern_funcs)
1055 for (j = 0; j < p->n_extern_funcs; j++) {
1056 struct extern_func_runtime *r = &t->extern_funcs[j];
1061 free(t->extern_funcs);
1062 t->extern_funcs = NULL;
1067 extern_func_free(struct rte_swx_pipeline *p)
1069 extern_func_build_free(p);
1072 struct extern_func *elem;
1074 elem = TAILQ_FIRST(&p->extern_funcs);
1078 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1086 static struct header *
1087 header_find(struct rte_swx_pipeline *p, const char *name)
1089 struct header *elem;
1091 TAILQ_FOREACH(elem, &p->headers, node)
1092 if (strcmp(elem->name, name) == 0)
1098 static struct header *
1099 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1101 struct header *elem;
1103 TAILQ_FOREACH(elem, &p->headers, node)
1104 if (elem->struct_id == struct_id)
1110 static struct header *
1111 header_parse(struct rte_swx_pipeline *p,
1114 if (name[0] != 'h' || name[1] != '.')
1117 return header_find(p, &name[2]);
1120 static struct field *
1121 header_field_parse(struct rte_swx_pipeline *p,
1123 struct header **header)
1127 char *header_name, *field_name;
1129 if ((name[0] != 'h') || (name[1] != '.'))
1132 header_name = strdup(&name[2]);
1136 field_name = strchr(header_name, '.');
1145 h = header_find(p, header_name);
1151 f = struct_type_field_find(h->st, field_name);
1165 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1167 const char *struct_type_name)
1169 struct struct_type *st;
1171 size_t n_headers_max;
1174 CHECK_NAME(name, EINVAL);
1175 CHECK_NAME(struct_type_name, EINVAL);
1177 CHECK(!header_find(p, name), EEXIST);
1179 st = struct_type_find(p, struct_type_name);
1182 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1183 CHECK(p->n_headers < n_headers_max, ENOSPC);
1185 /* Node allocation. */
1186 h = calloc(1, sizeof(struct header));
1189 /* Node initialization. */
1190 strcpy(h->name, name);
1192 h->struct_id = p->n_structs;
1193 h->id = p->n_headers;
1195 /* Node add to tailq. */
1196 TAILQ_INSERT_TAIL(&p->headers, h, node);
1204 header_build(struct rte_swx_pipeline *p)
1207 uint32_t n_bytes = 0, i;
1209 TAILQ_FOREACH(h, &p->headers, node) {
1210 n_bytes += h->st->n_bits / 8;
1213 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1214 struct thread *t = &p->threads[i];
1215 uint32_t offset = 0;
1217 t->headers = calloc(p->n_headers,
1218 sizeof(struct header_runtime));
1219 CHECK(t->headers, ENOMEM);
1221 t->headers_out = calloc(p->n_headers,
1222 sizeof(struct header_out_runtime));
1223 CHECK(t->headers_out, ENOMEM);
1225 t->header_storage = calloc(1, n_bytes);
1226 CHECK(t->header_storage, ENOMEM);
1228 t->header_out_storage = calloc(1, n_bytes);
1229 CHECK(t->header_out_storage, ENOMEM);
1231 TAILQ_FOREACH(h, &p->headers, node) {
1232 uint8_t *header_storage;
1233 uint32_t n_bytes = h->st->n_bits / 8;
1235 header_storage = &t->header_storage[offset];
1238 t->headers[h->id].ptr0 = header_storage;
1239 t->headers[h->id].n_bytes = n_bytes;
1241 t->structs[h->struct_id] = header_storage;
1249 header_build_free(struct rte_swx_pipeline *p)
1253 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1254 struct thread *t = &p->threads[i];
1256 free(t->headers_out);
1257 t->headers_out = NULL;
1262 free(t->header_out_storage);
1263 t->header_out_storage = NULL;
1265 free(t->header_storage);
1266 t->header_storage = NULL;
1271 header_free(struct rte_swx_pipeline *p)
1273 header_build_free(p);
1276 struct header *elem;
1278 elem = TAILQ_FIRST(&p->headers);
1282 TAILQ_REMOVE(&p->headers, elem, node);
1290 static struct field *
1291 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1293 if (!p->metadata_st)
1296 if (name[0] != 'm' || name[1] != '.')
1299 return struct_type_field_find(p->metadata_st, &name[2]);
1303 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1304 const char *struct_type_name)
1306 struct struct_type *st = NULL;
1310 CHECK_NAME(struct_type_name, EINVAL);
1311 st = struct_type_find(p, struct_type_name);
1313 CHECK(!st->var_size, EINVAL);
1314 CHECK(!p->metadata_st, EINVAL);
1316 p->metadata_st = st;
1317 p->metadata_struct_id = p->n_structs;
1325 metadata_build(struct rte_swx_pipeline *p)
1327 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1330 /* Thread-level initialization. */
1331 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1332 struct thread *t = &p->threads[i];
1335 metadata = calloc(1, n_bytes);
1336 CHECK(metadata, ENOMEM);
1338 t->metadata = metadata;
1339 t->structs[p->metadata_struct_id] = metadata;
1346 metadata_build_free(struct rte_swx_pipeline *p)
1350 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1351 struct thread *t = &p->threads[i];
1359 metadata_free(struct rte_swx_pipeline *p)
1361 metadata_build_free(p);
1368 instruction_is_tx(enum instruction_type type)
1381 instruction_does_tx(struct instruction *instr)
1383 switch (instr->type) {
1386 case INSTR_HDR_EMIT_TX:
1387 case INSTR_HDR_EMIT2_TX:
1388 case INSTR_HDR_EMIT3_TX:
1389 case INSTR_HDR_EMIT4_TX:
1390 case INSTR_HDR_EMIT5_TX:
1391 case INSTR_HDR_EMIT6_TX:
1392 case INSTR_HDR_EMIT7_TX:
1393 case INSTR_HDR_EMIT8_TX:
1401 instruction_is_jmp(struct instruction *instr)
1403 switch (instr->type) {
1405 case INSTR_JMP_VALID:
1406 case INSTR_JMP_INVALID:
1408 case INSTR_JMP_MISS:
1409 case INSTR_JMP_ACTION_HIT:
1410 case INSTR_JMP_ACTION_MISS:
1412 case INSTR_JMP_EQ_MH:
1413 case INSTR_JMP_EQ_HM:
1414 case INSTR_JMP_EQ_HH:
1415 case INSTR_JMP_EQ_I:
1417 case INSTR_JMP_NEQ_MH:
1418 case INSTR_JMP_NEQ_HM:
1419 case INSTR_JMP_NEQ_HH:
1420 case INSTR_JMP_NEQ_I:
1422 case INSTR_JMP_LT_MH:
1423 case INSTR_JMP_LT_HM:
1424 case INSTR_JMP_LT_HH:
1425 case INSTR_JMP_LT_MI:
1426 case INSTR_JMP_LT_HI:
1428 case INSTR_JMP_GT_MH:
1429 case INSTR_JMP_GT_HM:
1430 case INSTR_JMP_GT_HH:
1431 case INSTR_JMP_GT_MI:
1432 case INSTR_JMP_GT_HI:
1441 instruction_does_thread_yield(struct instruction *instr)
1443 switch (instr->type) {
1446 case INSTR_TABLE_AF:
1447 case INSTR_SELECTOR:
1449 case INSTR_LEARNER_AF:
1450 case INSTR_EXTERN_OBJ:
1451 case INSTR_EXTERN_FUNC:
1458 static struct field *
1459 action_field_parse(struct action *action, const char *name);
1461 static struct field *
1462 struct_field_parse(struct rte_swx_pipeline *p,
1463 struct action *action,
1465 uint32_t *struct_id)
1472 struct header *header;
1474 f = header_field_parse(p, name, &header);
1478 *struct_id = header->struct_id;
1484 f = metadata_field_parse(p, name);
1488 *struct_id = p->metadata_struct_id;
1497 f = action_field_parse(action, name);
1507 struct extern_obj *obj;
1509 f = extern_obj_mailbox_field_parse(p, name, &obj);
1513 *struct_id = obj->struct_id;
1519 struct extern_func *func;
1521 f = extern_func_mailbox_field_parse(p, name, &func);
1525 *struct_id = func->struct_id;
1538 instr_rx_translate(struct rte_swx_pipeline *p,
1539 struct action *action,
1542 struct instruction *instr,
1543 struct instruction_data *data __rte_unused)
1547 CHECK(!action, EINVAL);
1548 CHECK(n_tokens == 2, EINVAL);
1550 f = metadata_field_parse(p, tokens[1]);
1553 instr->type = INSTR_RX;
1554 instr->io.io.offset = f->offset / 8;
1555 instr->io.io.n_bits = f->n_bits;
1563 instr_tx_translate(struct rte_swx_pipeline *p,
1564 struct action *action __rte_unused,
1567 struct instruction *instr,
1568 struct instruction_data *data __rte_unused)
1570 char *port = tokens[1];
1574 CHECK(n_tokens == 2, EINVAL);
1576 f = metadata_field_parse(p, port);
1578 instr->type = INSTR_TX;
1579 instr->io.io.offset = f->offset / 8;
1580 instr->io.io.n_bits = f->n_bits;
1585 port_val = strtoul(port, &port, 0);
1586 CHECK(!port[0], EINVAL);
1588 instr->type = INSTR_TX_I;
1589 instr->io.io.val = port_val;
1594 instr_drop_translate(struct rte_swx_pipeline *p,
1595 struct action *action __rte_unused,
1596 char **tokens __rte_unused,
1598 struct instruction *instr,
1599 struct instruction_data *data __rte_unused)
1601 CHECK(n_tokens == 1, EINVAL);
1604 instr->type = INSTR_TX_I;
1605 instr->io.io.val = p->n_ports_out - 1;
1610 instr_tx_exec(struct rte_swx_pipeline *p)
1612 struct thread *t = &p->threads[p->thread_id];
1613 struct instruction *ip = t->ip;
1615 __instr_tx_exec(p, t, ip);
1618 thread_ip_reset(p, t);
1623 instr_tx_i_exec(struct rte_swx_pipeline *p)
1625 struct thread *t = &p->threads[p->thread_id];
1626 struct instruction *ip = t->ip;
1628 __instr_tx_i_exec(p, t, ip);
1631 thread_ip_reset(p, t);
1639 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1640 struct action *action,
1643 struct instruction *instr,
1644 struct instruction_data *data __rte_unused)
1648 CHECK(!action, EINVAL);
1649 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1651 h = header_parse(p, tokens[1]);
1654 if (n_tokens == 2) {
1655 CHECK(!h->st->var_size, EINVAL);
1657 instr->type = INSTR_HDR_EXTRACT;
1658 instr->io.hdr.header_id[0] = h->id;
1659 instr->io.hdr.struct_id[0] = h->struct_id;
1660 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1664 CHECK(h->st->var_size, EINVAL);
1666 mf = metadata_field_parse(p, tokens[2]);
1668 CHECK(!mf->var_size, EINVAL);
1670 instr->type = INSTR_HDR_EXTRACT_M;
1671 instr->io.io.offset = mf->offset / 8;
1672 instr->io.io.n_bits = mf->n_bits;
1673 instr->io.hdr.header_id[0] = h->id;
1674 instr->io.hdr.struct_id[0] = h->struct_id;
1675 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1682 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1683 struct action *action,
1686 struct instruction *instr,
1687 struct instruction_data *data __rte_unused)
1691 CHECK(!action, EINVAL);
1692 CHECK(n_tokens == 2, EINVAL);
1694 h = header_parse(p, tokens[1]);
1696 CHECK(!h->st->var_size, EINVAL);
1698 instr->type = INSTR_HDR_LOOKAHEAD;
1699 instr->io.hdr.header_id[0] = h->id;
1700 instr->io.hdr.struct_id[0] = h->struct_id;
1701 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1707 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1709 struct thread *t = &p->threads[p->thread_id];
1710 struct instruction *ip = t->ip;
1712 __instr_hdr_extract_exec(p, t, ip);
1719 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1721 struct thread *t = &p->threads[p->thread_id];
1722 struct instruction *ip = t->ip;
1724 __instr_hdr_extract2_exec(p, t, ip);
1731 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1733 struct thread *t = &p->threads[p->thread_id];
1734 struct instruction *ip = t->ip;
1736 __instr_hdr_extract3_exec(p, t, ip);
1743 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1745 struct thread *t = &p->threads[p->thread_id];
1746 struct instruction *ip = t->ip;
1748 __instr_hdr_extract4_exec(p, t, ip);
1755 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1757 struct thread *t = &p->threads[p->thread_id];
1758 struct instruction *ip = t->ip;
1760 __instr_hdr_extract5_exec(p, t, ip);
1767 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1769 struct thread *t = &p->threads[p->thread_id];
1770 struct instruction *ip = t->ip;
1772 __instr_hdr_extract6_exec(p, t, ip);
1779 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1781 struct thread *t = &p->threads[p->thread_id];
1782 struct instruction *ip = t->ip;
1784 __instr_hdr_extract7_exec(p, t, ip);
1791 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1793 struct thread *t = &p->threads[p->thread_id];
1794 struct instruction *ip = t->ip;
1796 __instr_hdr_extract8_exec(p, t, ip);
1803 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1805 struct thread *t = &p->threads[p->thread_id];
1806 struct instruction *ip = t->ip;
1808 __instr_hdr_extract_m_exec(p, t, ip);
1815 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1817 struct thread *t = &p->threads[p->thread_id];
1818 struct instruction *ip = t->ip;
1820 __instr_hdr_lookahead_exec(p, t, ip);
1830 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1831 struct action *action __rte_unused,
1834 struct instruction *instr,
1835 struct instruction_data *data __rte_unused)
1839 CHECK(n_tokens == 2, EINVAL);
1841 h = header_parse(p, tokens[1]);
1844 instr->type = INSTR_HDR_EMIT;
1845 instr->io.hdr.header_id[0] = h->id;
1846 instr->io.hdr.struct_id[0] = h->struct_id;
1847 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1852 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1854 struct thread *t = &p->threads[p->thread_id];
1855 struct instruction *ip = t->ip;
1857 __instr_hdr_emit_exec(p, t, ip);
1864 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1866 struct thread *t = &p->threads[p->thread_id];
1867 struct instruction *ip = t->ip;
1869 __instr_hdr_emit_tx_exec(p, t, ip);
1872 thread_ip_reset(p, t);
1877 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1879 struct thread *t = &p->threads[p->thread_id];
1880 struct instruction *ip = t->ip;
1882 __instr_hdr_emit2_tx_exec(p, t, ip);
1885 thread_ip_reset(p, t);
1890 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1892 struct thread *t = &p->threads[p->thread_id];
1893 struct instruction *ip = t->ip;
1895 __instr_hdr_emit3_tx_exec(p, t, ip);
1898 thread_ip_reset(p, t);
1903 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1905 struct thread *t = &p->threads[p->thread_id];
1906 struct instruction *ip = t->ip;
1908 __instr_hdr_emit4_tx_exec(p, t, ip);
1911 thread_ip_reset(p, t);
1916 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1918 struct thread *t = &p->threads[p->thread_id];
1919 struct instruction *ip = t->ip;
1921 __instr_hdr_emit5_tx_exec(p, t, ip);
1924 thread_ip_reset(p, t);
1929 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1931 struct thread *t = &p->threads[p->thread_id];
1932 struct instruction *ip = t->ip;
1934 __instr_hdr_emit6_tx_exec(p, t, ip);
1937 thread_ip_reset(p, t);
1942 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1944 struct thread *t = &p->threads[p->thread_id];
1945 struct instruction *ip = t->ip;
1947 __instr_hdr_emit7_tx_exec(p, t, ip);
1950 thread_ip_reset(p, t);
1955 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1957 struct thread *t = &p->threads[p->thread_id];
1958 struct instruction *ip = t->ip;
1960 __instr_hdr_emit8_tx_exec(p, t, ip);
1963 thread_ip_reset(p, t);
1971 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1972 struct action *action __rte_unused,
1975 struct instruction *instr,
1976 struct instruction_data *data __rte_unused)
1980 CHECK(n_tokens == 2, EINVAL);
1982 h = header_parse(p, tokens[1]);
1985 instr->type = INSTR_HDR_VALIDATE;
1986 instr->valid.header_id = h->id;
1991 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1993 struct thread *t = &p->threads[p->thread_id];
1994 struct instruction *ip = t->ip;
1996 __instr_hdr_validate_exec(p, t, ip);
2006 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2007 struct action *action __rte_unused,
2010 struct instruction *instr,
2011 struct instruction_data *data __rte_unused)
2015 CHECK(n_tokens == 2, EINVAL);
2017 h = header_parse(p, tokens[1]);
2020 instr->type = INSTR_HDR_INVALIDATE;
2021 instr->valid.header_id = h->id;
2026 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2028 struct thread *t = &p->threads[p->thread_id];
2029 struct instruction *ip = t->ip;
2031 __instr_hdr_invalidate_exec(p, t, ip);
2040 static struct table *
2041 table_find(struct rte_swx_pipeline *p, const char *name);
2043 static struct selector *
2044 selector_find(struct rte_swx_pipeline *p, const char *name);
2046 static struct learner *
2047 learner_find(struct rte_swx_pipeline *p, const char *name);
2050 instr_table_translate(struct rte_swx_pipeline *p,
2051 struct action *action,
2054 struct instruction *instr,
2055 struct instruction_data *data __rte_unused)
2061 CHECK(!action, EINVAL);
2062 CHECK(n_tokens == 2, EINVAL);
2064 t = table_find(p, tokens[1]);
2066 instr->type = INSTR_TABLE;
2067 instr->table.table_id = t->id;
2071 s = selector_find(p, tokens[1]);
2073 instr->type = INSTR_SELECTOR;
2074 instr->table.table_id = s->id;
2078 l = learner_find(p, tokens[1]);
2080 instr->type = INSTR_LEARNER;
2081 instr->table.table_id = l->id;
2089 instr_table_exec(struct rte_swx_pipeline *p)
2091 struct thread *t = &p->threads[p->thread_id];
2092 struct instruction *ip = t->ip;
2093 uint32_t table_id = ip->table.table_id;
2094 struct rte_swx_table_state *ts = &t->table_state[table_id];
2095 struct table_runtime *table = &t->tables[table_id];
2096 struct table_statistics *stats = &p->table_stats[table_id];
2097 uint64_t action_id, n_pkts_hit, n_pkts_action;
2098 uint8_t *action_data;
2102 done = table->func(ts->obj,
2110 TRACE("[Thread %2u] table %u (not finalized)\n",
2118 action_id = hit ? action_id : ts->default_action_id;
2119 action_data = hit ? action_data : ts->default_action_data;
2120 n_pkts_hit = stats->n_pkts_hit[hit];
2121 n_pkts_action = stats->n_pkts_action[action_id];
2123 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2126 hit ? "hit" : "miss",
2127 (uint32_t)action_id);
2129 t->action_id = action_id;
2130 t->structs[0] = action_data;
2132 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2133 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2136 thread_ip_action_call(p, t, action_id);
2140 instr_table_af_exec(struct rte_swx_pipeline *p)
2142 struct thread *t = &p->threads[p->thread_id];
2143 struct instruction *ip = t->ip;
2144 uint32_t table_id = ip->table.table_id;
2145 struct rte_swx_table_state *ts = &t->table_state[table_id];
2146 struct table_runtime *table = &t->tables[table_id];
2147 struct table_statistics *stats = &p->table_stats[table_id];
2148 uint64_t action_id, n_pkts_hit, n_pkts_action;
2149 uint8_t *action_data;
2150 action_func_t action_func;
2154 done = table->func(ts->obj,
2162 TRACE("[Thread %2u] table %u (not finalized)\n",
2170 action_id = hit ? action_id : ts->default_action_id;
2171 action_data = hit ? action_data : ts->default_action_data;
2172 action_func = p->action_funcs[action_id];
2173 n_pkts_hit = stats->n_pkts_hit[hit];
2174 n_pkts_action = stats->n_pkts_action[action_id];
2176 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2179 hit ? "hit" : "miss",
2180 (uint32_t)action_id);
2182 t->action_id = action_id;
2183 t->structs[0] = action_data;
2185 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2186 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2196 instr_selector_exec(struct rte_swx_pipeline *p)
2198 struct thread *t = &p->threads[p->thread_id];
2199 struct instruction *ip = t->ip;
2200 uint32_t selector_id = ip->table.table_id;
2201 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2202 struct selector_runtime *selector = &t->selectors[selector_id];
2203 struct selector_statistics *stats = &p->selector_stats[selector_id];
2204 uint64_t n_pkts = stats->n_pkts;
2208 done = rte_swx_table_selector_select(ts->obj,
2210 selector->group_id_buffer,
2211 selector->selector_buffer,
2212 selector->member_id_buffer);
2215 TRACE("[Thread %2u] selector %u (not finalized)\n",
2224 TRACE("[Thread %2u] selector %u\n",
2228 stats->n_pkts = n_pkts + 1;
2235 instr_learner_exec(struct rte_swx_pipeline *p)
2237 struct thread *t = &p->threads[p->thread_id];
2238 struct instruction *ip = t->ip;
2239 uint32_t learner_id = ip->table.table_id;
2240 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2241 p->n_selectors + learner_id];
2242 struct learner_runtime *l = &t->learners[learner_id];
2243 struct learner_statistics *stats = &p->learner_stats[learner_id];
2244 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2245 uint8_t *action_data;
2249 time = rte_get_tsc_cycles();
2251 done = rte_swx_table_learner_lookup(ts->obj,
2260 TRACE("[Thread %2u] learner %u (not finalized)\n",
2268 action_id = hit ? action_id : ts->default_action_id;
2269 action_data = hit ? action_data : ts->default_action_data;
2270 n_pkts_hit = stats->n_pkts_hit[hit];
2271 n_pkts_action = stats->n_pkts_action[action_id];
2273 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2276 hit ? "hit" : "miss",
2277 (uint32_t)action_id);
2279 t->action_id = action_id;
2280 t->structs[0] = action_data;
2282 t->learner_id = learner_id;
2284 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2285 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2288 thread_ip_action_call(p, t, action_id);
2292 instr_learner_af_exec(struct rte_swx_pipeline *p)
2294 struct thread *t = &p->threads[p->thread_id];
2295 struct instruction *ip = t->ip;
2296 uint32_t learner_id = ip->table.table_id;
2297 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2298 p->n_selectors + learner_id];
2299 struct learner_runtime *l = &t->learners[learner_id];
2300 struct learner_statistics *stats = &p->learner_stats[learner_id];
2301 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2302 uint8_t *action_data;
2303 action_func_t action_func;
2307 time = rte_get_tsc_cycles();
2309 done = rte_swx_table_learner_lookup(ts->obj,
2318 TRACE("[Thread %2u] learner %u (not finalized)\n",
2326 action_id = hit ? action_id : ts->default_action_id;
2327 action_data = hit ? action_data : ts->default_action_data;
2328 action_func = p->action_funcs[action_id];
2329 n_pkts_hit = stats->n_pkts_hit[hit];
2330 n_pkts_action = stats->n_pkts_action[action_id];
2332 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2335 hit ? "hit" : "miss",
2336 (uint32_t)action_id);
2338 t->action_id = action_id;
2339 t->structs[0] = action_data;
2341 t->learner_id = learner_id;
2343 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2344 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2347 thread_ip_action_call(p, t, action_id);
2356 static struct action *
2357 action_find(struct rte_swx_pipeline *p, const char *name);
2360 action_has_nbo_args(struct action *a);
2363 instr_learn_translate(struct rte_swx_pipeline *p,
2364 struct action *action,
2367 struct instruction *instr,
2368 struct instruction_data *data __rte_unused)
2372 CHECK(action, EINVAL);
2373 CHECK(n_tokens == 2, EINVAL);
2375 a = action_find(p, tokens[1]);
2377 CHECK(!action_has_nbo_args(a), EINVAL);
2379 instr->type = INSTR_LEARNER_LEARN;
2380 instr->learn.action_id = a->id;
2386 instr_learn_exec(struct rte_swx_pipeline *p)
2388 struct thread *t = &p->threads[p->thread_id];
2389 struct instruction *ip = t->ip;
2391 __instr_learn_exec(p, t, ip);
2401 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2402 struct action *action,
2403 char **tokens __rte_unused,
2405 struct instruction *instr,
2406 struct instruction_data *data __rte_unused)
2408 CHECK(action, EINVAL);
2409 CHECK(n_tokens == 1, EINVAL);
2411 instr->type = INSTR_LEARNER_FORGET;
2417 instr_forget_exec(struct rte_swx_pipeline *p)
2419 struct thread *t = &p->threads[p->thread_id];
2420 struct instruction *ip = t->ip;
2422 __instr_forget_exec(p, t, ip);
2432 instr_extern_translate(struct rte_swx_pipeline *p,
2433 struct action *action __rte_unused,
2436 struct instruction *instr,
2437 struct instruction_data *data __rte_unused)
2439 char *token = tokens[1];
2441 CHECK(n_tokens == 2, EINVAL);
2443 if (token[0] == 'e') {
2444 struct extern_obj *obj;
2445 struct extern_type_member_func *func;
2447 func = extern_obj_member_func_parse(p, token, &obj);
2448 CHECK(func, EINVAL);
2450 instr->type = INSTR_EXTERN_OBJ;
2451 instr->ext_obj.ext_obj_id = obj->id;
2452 instr->ext_obj.func_id = func->id;
2457 if (token[0] == 'f') {
2458 struct extern_func *func;
2460 func = extern_func_parse(p, token);
2461 CHECK(func, EINVAL);
2463 instr->type = INSTR_EXTERN_FUNC;
2464 instr->ext_func.ext_func_id = func->id;
2473 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2475 struct thread *t = &p->threads[p->thread_id];
2476 struct instruction *ip = t->ip;
2479 /* Extern object member function execute. */
2480 done = __instr_extern_obj_exec(p, t, ip);
2483 thread_ip_inc_cond(t, done);
2484 thread_yield_cond(p, done ^ 1);
2488 instr_extern_func_exec(struct rte_swx_pipeline *p)
2490 struct thread *t = &p->threads[p->thread_id];
2491 struct instruction *ip = t->ip;
2494 /* Extern function execute. */
2495 done = __instr_extern_func_exec(p, t, ip);
2498 thread_ip_inc_cond(t, done);
2499 thread_yield_cond(p, done ^ 1);
2506 instr_mov_translate(struct rte_swx_pipeline *p,
2507 struct action *action,
2510 struct instruction *instr,
2511 struct instruction_data *data __rte_unused)
2513 char *dst = tokens[1], *src = tokens[2];
2514 struct field *fdst, *fsrc;
2516 uint32_t dst_struct_id = 0, src_struct_id = 0;
2518 CHECK(n_tokens == 3, EINVAL);
2520 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2521 CHECK(fdst, EINVAL);
2522 CHECK(!fdst->var_size, EINVAL);
2524 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2525 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2527 CHECK(!fsrc->var_size, EINVAL);
2529 instr->type = INSTR_MOV;
2530 if (dst[0] != 'h' && src[0] == 'h')
2531 instr->type = INSTR_MOV_MH;
2532 if (dst[0] == 'h' && src[0] != 'h')
2533 instr->type = INSTR_MOV_HM;
2534 if (dst[0] == 'h' && src[0] == 'h')
2535 instr->type = INSTR_MOV_HH;
2537 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2538 instr->mov.dst.n_bits = fdst->n_bits;
2539 instr->mov.dst.offset = fdst->offset / 8;
2540 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2541 instr->mov.src.n_bits = fsrc->n_bits;
2542 instr->mov.src.offset = fsrc->offset / 8;
2547 src_val = strtoull(src, &src, 0);
2548 CHECK(!src[0], EINVAL);
2551 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2553 instr->type = INSTR_MOV_I;
2554 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2555 instr->mov.dst.n_bits = fdst->n_bits;
2556 instr->mov.dst.offset = fdst->offset / 8;
2557 instr->mov.src_val = src_val;
2562 instr_mov_exec(struct rte_swx_pipeline *p)
2564 struct thread *t = &p->threads[p->thread_id];
2565 struct instruction *ip = t->ip;
2567 __instr_mov_exec(p, t, ip);
2574 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2576 struct thread *t = &p->threads[p->thread_id];
2577 struct instruction *ip = t->ip;
2579 __instr_mov_mh_exec(p, t, ip);
2586 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2588 struct thread *t = &p->threads[p->thread_id];
2589 struct instruction *ip = t->ip;
2591 __instr_mov_hm_exec(p, t, ip);
2598 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2600 struct thread *t = &p->threads[p->thread_id];
2601 struct instruction *ip = t->ip;
2603 __instr_mov_hh_exec(p, t, ip);
2610 instr_mov_i_exec(struct rte_swx_pipeline *p)
2612 struct thread *t = &p->threads[p->thread_id];
2613 struct instruction *ip = t->ip;
2615 __instr_mov_i_exec(p, t, ip);
2625 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2627 struct thread *t = &p->threads[p->thread_id];
2628 struct instruction *ip = t->ip;
2630 __instr_dma_ht_exec(p, t, ip);
2637 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2639 struct thread *t = &p->threads[p->thread_id];
2640 struct instruction *ip = t->ip;
2642 __instr_dma_ht2_exec(p, t, ip);
2649 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2651 struct thread *t = &p->threads[p->thread_id];
2652 struct instruction *ip = t->ip;
2654 __instr_dma_ht3_exec(p, t, ip);
2661 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2663 struct thread *t = &p->threads[p->thread_id];
2664 struct instruction *ip = t->ip;
2666 __instr_dma_ht4_exec(p, t, ip);
2673 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2675 struct thread *t = &p->threads[p->thread_id];
2676 struct instruction *ip = t->ip;
2678 __instr_dma_ht5_exec(p, t, ip);
2685 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2687 struct thread *t = &p->threads[p->thread_id];
2688 struct instruction *ip = t->ip;
2690 __instr_dma_ht6_exec(p, t, ip);
2697 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2699 struct thread *t = &p->threads[p->thread_id];
2700 struct instruction *ip = t->ip;
2702 __instr_dma_ht7_exec(p, t, ip);
2709 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2711 struct thread *t = &p->threads[p->thread_id];
2712 struct instruction *ip = t->ip;
2714 __instr_dma_ht8_exec(p, t, ip);
2724 instr_alu_add_translate(struct rte_swx_pipeline *p,
2725 struct action *action,
2728 struct instruction *instr,
2729 struct instruction_data *data __rte_unused)
2731 char *dst = tokens[1], *src = tokens[2];
2732 struct field *fdst, *fsrc;
2734 uint32_t dst_struct_id = 0, src_struct_id = 0;
2736 CHECK(n_tokens == 3, EINVAL);
2738 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2739 CHECK(fdst, EINVAL);
2740 CHECK(!fdst->var_size, EINVAL);
2742 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2743 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2745 CHECK(!fsrc->var_size, EINVAL);
2747 instr->type = INSTR_ALU_ADD;
2748 if (dst[0] == 'h' && src[0] != 'h')
2749 instr->type = INSTR_ALU_ADD_HM;
2750 if (dst[0] != 'h' && src[0] == 'h')
2751 instr->type = INSTR_ALU_ADD_MH;
2752 if (dst[0] == 'h' && src[0] == 'h')
2753 instr->type = INSTR_ALU_ADD_HH;
2755 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2756 instr->alu.dst.n_bits = fdst->n_bits;
2757 instr->alu.dst.offset = fdst->offset / 8;
2758 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2759 instr->alu.src.n_bits = fsrc->n_bits;
2760 instr->alu.src.offset = fsrc->offset / 8;
2764 /* ADD_MI, ADD_HI. */
2765 src_val = strtoull(src, &src, 0);
2766 CHECK(!src[0], EINVAL);
2768 instr->type = INSTR_ALU_ADD_MI;
2770 instr->type = INSTR_ALU_ADD_HI;
2772 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2773 instr->alu.dst.n_bits = fdst->n_bits;
2774 instr->alu.dst.offset = fdst->offset / 8;
2775 instr->alu.src_val = src_val;
2780 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2781 struct action *action,
2784 struct instruction *instr,
2785 struct instruction_data *data __rte_unused)
2787 char *dst = tokens[1], *src = tokens[2];
2788 struct field *fdst, *fsrc;
2790 uint32_t dst_struct_id = 0, src_struct_id = 0;
2792 CHECK(n_tokens == 3, EINVAL);
2794 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2795 CHECK(fdst, EINVAL);
2796 CHECK(!fdst->var_size, EINVAL);
2798 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2799 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2801 CHECK(!fsrc->var_size, EINVAL);
2803 instr->type = INSTR_ALU_SUB;
2804 if (dst[0] == 'h' && src[0] != 'h')
2805 instr->type = INSTR_ALU_SUB_HM;
2806 if (dst[0] != 'h' && src[0] == 'h')
2807 instr->type = INSTR_ALU_SUB_MH;
2808 if (dst[0] == 'h' && src[0] == 'h')
2809 instr->type = INSTR_ALU_SUB_HH;
2811 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2812 instr->alu.dst.n_bits = fdst->n_bits;
2813 instr->alu.dst.offset = fdst->offset / 8;
2814 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2815 instr->alu.src.n_bits = fsrc->n_bits;
2816 instr->alu.src.offset = fsrc->offset / 8;
2820 /* SUB_MI, SUB_HI. */
2821 src_val = strtoull(src, &src, 0);
2822 CHECK(!src[0], EINVAL);
2824 instr->type = INSTR_ALU_SUB_MI;
2826 instr->type = INSTR_ALU_SUB_HI;
2828 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2829 instr->alu.dst.n_bits = fdst->n_bits;
2830 instr->alu.dst.offset = fdst->offset / 8;
2831 instr->alu.src_val = src_val;
2836 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2837 struct action *action __rte_unused,
2840 struct instruction *instr,
2841 struct instruction_data *data __rte_unused)
2843 char *dst = tokens[1], *src = tokens[2];
2844 struct header *hdst, *hsrc;
2845 struct field *fdst, *fsrc;
2847 CHECK(n_tokens == 3, EINVAL);
2849 fdst = header_field_parse(p, dst, &hdst);
2850 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2851 CHECK(!fdst->var_size, EINVAL);
2854 fsrc = header_field_parse(p, src, &hsrc);
2856 CHECK(!fsrc->var_size, EINVAL);
2858 instr->type = INSTR_ALU_CKADD_FIELD;
2859 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2860 instr->alu.dst.n_bits = fdst->n_bits;
2861 instr->alu.dst.offset = fdst->offset / 8;
2862 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2863 instr->alu.src.n_bits = fsrc->n_bits;
2864 instr->alu.src.offset = fsrc->offset / 8;
2868 /* CKADD_STRUCT, CKADD_STRUCT20. */
2869 hsrc = header_parse(p, src);
2870 CHECK(hsrc, EINVAL);
2871 CHECK(!hsrc->st->var_size, EINVAL);
2873 instr->type = INSTR_ALU_CKADD_STRUCT;
2874 if ((hsrc->st->n_bits / 8) == 20)
2875 instr->type = INSTR_ALU_CKADD_STRUCT20;
2877 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2878 instr->alu.dst.n_bits = fdst->n_bits;
2879 instr->alu.dst.offset = fdst->offset / 8;
2880 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2881 instr->alu.src.n_bits = hsrc->st->n_bits;
2882 instr->alu.src.offset = 0; /* Unused. */
2887 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2888 struct action *action __rte_unused,
2891 struct instruction *instr,
2892 struct instruction_data *data __rte_unused)
2894 char *dst = tokens[1], *src = tokens[2];
2895 struct header *hdst, *hsrc;
2896 struct field *fdst, *fsrc;
2898 CHECK(n_tokens == 3, EINVAL);
2900 fdst = header_field_parse(p, dst, &hdst);
2901 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2902 CHECK(!fdst->var_size, EINVAL);
2904 fsrc = header_field_parse(p, src, &hsrc);
2905 CHECK(fsrc, EINVAL);
2906 CHECK(!fsrc->var_size, EINVAL);
2908 instr->type = INSTR_ALU_CKSUB_FIELD;
2909 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2910 instr->alu.dst.n_bits = fdst->n_bits;
2911 instr->alu.dst.offset = fdst->offset / 8;
2912 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2913 instr->alu.src.n_bits = fsrc->n_bits;
2914 instr->alu.src.offset = fsrc->offset / 8;
2919 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2920 struct action *action,
2923 struct instruction *instr,
2924 struct instruction_data *data __rte_unused)
2926 char *dst = tokens[1], *src = tokens[2];
2927 struct field *fdst, *fsrc;
2929 uint32_t dst_struct_id = 0, src_struct_id = 0;
2931 CHECK(n_tokens == 3, EINVAL);
2933 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2934 CHECK(fdst, EINVAL);
2935 CHECK(!fdst->var_size, EINVAL);
2937 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2938 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2940 CHECK(!fsrc->var_size, EINVAL);
2942 instr->type = INSTR_ALU_SHL;
2943 if (dst[0] == 'h' && src[0] != 'h')
2944 instr->type = INSTR_ALU_SHL_HM;
2945 if (dst[0] != 'h' && src[0] == 'h')
2946 instr->type = INSTR_ALU_SHL_MH;
2947 if (dst[0] == 'h' && src[0] == 'h')
2948 instr->type = INSTR_ALU_SHL_HH;
2950 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2951 instr->alu.dst.n_bits = fdst->n_bits;
2952 instr->alu.dst.offset = fdst->offset / 8;
2953 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2954 instr->alu.src.n_bits = fsrc->n_bits;
2955 instr->alu.src.offset = fsrc->offset / 8;
2959 /* SHL_MI, SHL_HI. */
2960 src_val = strtoull(src, &src, 0);
2961 CHECK(!src[0], EINVAL);
2963 instr->type = INSTR_ALU_SHL_MI;
2965 instr->type = INSTR_ALU_SHL_HI;
2967 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2968 instr->alu.dst.n_bits = fdst->n_bits;
2969 instr->alu.dst.offset = fdst->offset / 8;
2970 instr->alu.src_val = src_val;
2975 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2976 struct action *action,
2979 struct instruction *instr,
2980 struct instruction_data *data __rte_unused)
2982 char *dst = tokens[1], *src = tokens[2];
2983 struct field *fdst, *fsrc;
2985 uint32_t dst_struct_id = 0, src_struct_id = 0;
2987 CHECK(n_tokens == 3, EINVAL);
2989 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2990 CHECK(fdst, EINVAL);
2991 CHECK(!fdst->var_size, EINVAL);
2993 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2994 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2996 CHECK(!fsrc->var_size, EINVAL);
2998 instr->type = INSTR_ALU_SHR;
2999 if (dst[0] == 'h' && src[0] != 'h')
3000 instr->type = INSTR_ALU_SHR_HM;
3001 if (dst[0] != 'h' && src[0] == 'h')
3002 instr->type = INSTR_ALU_SHR_MH;
3003 if (dst[0] == 'h' && src[0] == 'h')
3004 instr->type = INSTR_ALU_SHR_HH;
3006 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3007 instr->alu.dst.n_bits = fdst->n_bits;
3008 instr->alu.dst.offset = fdst->offset / 8;
3009 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3010 instr->alu.src.n_bits = fsrc->n_bits;
3011 instr->alu.src.offset = fsrc->offset / 8;
3015 /* SHR_MI, SHR_HI. */
3016 src_val = strtoull(src, &src, 0);
3017 CHECK(!src[0], EINVAL);
3019 instr->type = INSTR_ALU_SHR_MI;
3021 instr->type = INSTR_ALU_SHR_HI;
3023 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3024 instr->alu.dst.n_bits = fdst->n_bits;
3025 instr->alu.dst.offset = fdst->offset / 8;
3026 instr->alu.src_val = src_val;
3031 instr_alu_and_translate(struct rte_swx_pipeline *p,
3032 struct action *action,
3035 struct instruction *instr,
3036 struct instruction_data *data __rte_unused)
3038 char *dst = tokens[1], *src = tokens[2];
3039 struct field *fdst, *fsrc;
3041 uint32_t dst_struct_id = 0, src_struct_id = 0;
3043 CHECK(n_tokens == 3, EINVAL);
3045 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3046 CHECK(fdst, EINVAL);
3047 CHECK(!fdst->var_size, EINVAL);
3049 /* AND, AND_MH, AND_HM, AND_HH. */
3050 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3052 CHECK(!fsrc->var_size, EINVAL);
3054 instr->type = INSTR_ALU_AND;
3055 if (dst[0] != 'h' && src[0] == 'h')
3056 instr->type = INSTR_ALU_AND_MH;
3057 if (dst[0] == 'h' && src[0] != 'h')
3058 instr->type = INSTR_ALU_AND_HM;
3059 if (dst[0] == 'h' && src[0] == 'h')
3060 instr->type = INSTR_ALU_AND_HH;
3062 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3063 instr->alu.dst.n_bits = fdst->n_bits;
3064 instr->alu.dst.offset = fdst->offset / 8;
3065 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3066 instr->alu.src.n_bits = fsrc->n_bits;
3067 instr->alu.src.offset = fsrc->offset / 8;
3072 src_val = strtoull(src, &src, 0);
3073 CHECK(!src[0], EINVAL);
3076 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3078 instr->type = INSTR_ALU_AND_I;
3079 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3080 instr->alu.dst.n_bits = fdst->n_bits;
3081 instr->alu.dst.offset = fdst->offset / 8;
3082 instr->alu.src_val = src_val;
3087 instr_alu_or_translate(struct rte_swx_pipeline *p,
3088 struct action *action,
3091 struct instruction *instr,
3092 struct instruction_data *data __rte_unused)
3094 char *dst = tokens[1], *src = tokens[2];
3095 struct field *fdst, *fsrc;
3097 uint32_t dst_struct_id = 0, src_struct_id = 0;
3099 CHECK(n_tokens == 3, EINVAL);
3101 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3102 CHECK(fdst, EINVAL);
3103 CHECK(!fdst->var_size, EINVAL);
3105 /* OR, OR_MH, OR_HM, OR_HH. */
3106 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3108 CHECK(!fsrc->var_size, EINVAL);
3110 instr->type = INSTR_ALU_OR;
3111 if (dst[0] != 'h' && src[0] == 'h')
3112 instr->type = INSTR_ALU_OR_MH;
3113 if (dst[0] == 'h' && src[0] != 'h')
3114 instr->type = INSTR_ALU_OR_HM;
3115 if (dst[0] == 'h' && src[0] == 'h')
3116 instr->type = INSTR_ALU_OR_HH;
3118 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3119 instr->alu.dst.n_bits = fdst->n_bits;
3120 instr->alu.dst.offset = fdst->offset / 8;
3121 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3122 instr->alu.src.n_bits = fsrc->n_bits;
3123 instr->alu.src.offset = fsrc->offset / 8;
3128 src_val = strtoull(src, &src, 0);
3129 CHECK(!src[0], EINVAL);
3132 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3134 instr->type = INSTR_ALU_OR_I;
3135 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3136 instr->alu.dst.n_bits = fdst->n_bits;
3137 instr->alu.dst.offset = fdst->offset / 8;
3138 instr->alu.src_val = src_val;
3143 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3144 struct action *action,
3147 struct instruction *instr,
3148 struct instruction_data *data __rte_unused)
3150 char *dst = tokens[1], *src = tokens[2];
3151 struct field *fdst, *fsrc;
3153 uint32_t dst_struct_id = 0, src_struct_id = 0;
3155 CHECK(n_tokens == 3, EINVAL);
3157 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3158 CHECK(fdst, EINVAL);
3159 CHECK(!fdst->var_size, EINVAL);
3161 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3162 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3164 CHECK(!fsrc->var_size, EINVAL);
3166 instr->type = INSTR_ALU_XOR;
3167 if (dst[0] != 'h' && src[0] == 'h')
3168 instr->type = INSTR_ALU_XOR_MH;
3169 if (dst[0] == 'h' && src[0] != 'h')
3170 instr->type = INSTR_ALU_XOR_HM;
3171 if (dst[0] == 'h' && src[0] == 'h')
3172 instr->type = INSTR_ALU_XOR_HH;
3174 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3175 instr->alu.dst.n_bits = fdst->n_bits;
3176 instr->alu.dst.offset = fdst->offset / 8;
3177 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3178 instr->alu.src.n_bits = fsrc->n_bits;
3179 instr->alu.src.offset = fsrc->offset / 8;
3184 src_val = strtoull(src, &src, 0);
3185 CHECK(!src[0], EINVAL);
3188 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3190 instr->type = INSTR_ALU_XOR_I;
3191 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3192 instr->alu.dst.n_bits = fdst->n_bits;
3193 instr->alu.dst.offset = fdst->offset / 8;
3194 instr->alu.src_val = src_val;
3199 instr_alu_add_exec(struct rte_swx_pipeline *p)
3201 struct thread *t = &p->threads[p->thread_id];
3202 struct instruction *ip = t->ip;
3205 __instr_alu_add_exec(p, t, ip);
3212 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3214 struct thread *t = &p->threads[p->thread_id];
3215 struct instruction *ip = t->ip;
3218 __instr_alu_add_mh_exec(p, t, ip);
3225 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3227 struct thread *t = &p->threads[p->thread_id];
3228 struct instruction *ip = t->ip;
3231 __instr_alu_add_hm_exec(p, t, ip);
3238 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3240 struct thread *t = &p->threads[p->thread_id];
3241 struct instruction *ip = t->ip;
3244 __instr_alu_add_hh_exec(p, t, ip);
3251 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3253 struct thread *t = &p->threads[p->thread_id];
3254 struct instruction *ip = t->ip;
3257 __instr_alu_add_mi_exec(p, t, ip);
3264 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3266 struct thread *t = &p->threads[p->thread_id];
3267 struct instruction *ip = t->ip;
3270 __instr_alu_add_hi_exec(p, t, ip);
3277 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3279 struct thread *t = &p->threads[p->thread_id];
3280 struct instruction *ip = t->ip;
3283 __instr_alu_sub_exec(p, t, ip);
3290 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3292 struct thread *t = &p->threads[p->thread_id];
3293 struct instruction *ip = t->ip;
3296 __instr_alu_sub_mh_exec(p, t, ip);
3303 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3305 struct thread *t = &p->threads[p->thread_id];
3306 struct instruction *ip = t->ip;
3309 __instr_alu_sub_hm_exec(p, t, ip);
3316 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3318 struct thread *t = &p->threads[p->thread_id];
3319 struct instruction *ip = t->ip;
3322 __instr_alu_sub_hh_exec(p, t, ip);
3329 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3331 struct thread *t = &p->threads[p->thread_id];
3332 struct instruction *ip = t->ip;
3335 __instr_alu_sub_mi_exec(p, t, ip);
3342 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3344 struct thread *t = &p->threads[p->thread_id];
3345 struct instruction *ip = t->ip;
3348 __instr_alu_sub_hi_exec(p, t, ip);
3355 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3357 struct thread *t = &p->threads[p->thread_id];
3358 struct instruction *ip = t->ip;
3361 __instr_alu_shl_exec(p, t, ip);
3368 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3370 struct thread *t = &p->threads[p->thread_id];
3371 struct instruction *ip = t->ip;
3374 __instr_alu_shl_mh_exec(p, t, ip);
3381 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3383 struct thread *t = &p->threads[p->thread_id];
3384 struct instruction *ip = t->ip;
3387 __instr_alu_shl_hm_exec(p, t, ip);
3394 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3396 struct thread *t = &p->threads[p->thread_id];
3397 struct instruction *ip = t->ip;
3400 __instr_alu_shl_hh_exec(p, t, ip);
3407 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3409 struct thread *t = &p->threads[p->thread_id];
3410 struct instruction *ip = t->ip;
3413 __instr_alu_shl_mi_exec(p, t, ip);
3420 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3422 struct thread *t = &p->threads[p->thread_id];
3423 struct instruction *ip = t->ip;
3426 __instr_alu_shl_hi_exec(p, t, ip);
3433 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3435 struct thread *t = &p->threads[p->thread_id];
3436 struct instruction *ip = t->ip;
3439 __instr_alu_shr_exec(p, t, ip);
3446 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3448 struct thread *t = &p->threads[p->thread_id];
3449 struct instruction *ip = t->ip;
3452 __instr_alu_shr_mh_exec(p, t, ip);
3459 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3461 struct thread *t = &p->threads[p->thread_id];
3462 struct instruction *ip = t->ip;
3465 __instr_alu_shr_hm_exec(p, t, ip);
3472 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3474 struct thread *t = &p->threads[p->thread_id];
3475 struct instruction *ip = t->ip;
3478 __instr_alu_shr_hh_exec(p, t, ip);
3485 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3487 struct thread *t = &p->threads[p->thread_id];
3488 struct instruction *ip = t->ip;
3491 __instr_alu_shr_mi_exec(p, t, ip);
3498 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3500 struct thread *t = &p->threads[p->thread_id];
3501 struct instruction *ip = t->ip;
3504 __instr_alu_shr_hi_exec(p, t, ip);
3511 instr_alu_and_exec(struct rte_swx_pipeline *p)
3513 struct thread *t = &p->threads[p->thread_id];
3514 struct instruction *ip = t->ip;
3517 __instr_alu_and_exec(p, t, ip);
3524 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3526 struct thread *t = &p->threads[p->thread_id];
3527 struct instruction *ip = t->ip;
3530 __instr_alu_and_mh_exec(p, t, ip);
3537 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3539 struct thread *t = &p->threads[p->thread_id];
3540 struct instruction *ip = t->ip;
3543 __instr_alu_and_hm_exec(p, t, ip);
3550 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3552 struct thread *t = &p->threads[p->thread_id];
3553 struct instruction *ip = t->ip;
3556 __instr_alu_and_hh_exec(p, t, ip);
3563 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3565 struct thread *t = &p->threads[p->thread_id];
3566 struct instruction *ip = t->ip;
3569 __instr_alu_and_i_exec(p, t, ip);
3576 instr_alu_or_exec(struct rte_swx_pipeline *p)
3578 struct thread *t = &p->threads[p->thread_id];
3579 struct instruction *ip = t->ip;
3582 __instr_alu_or_exec(p, t, ip);
3589 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3591 struct thread *t = &p->threads[p->thread_id];
3592 struct instruction *ip = t->ip;
3595 __instr_alu_or_mh_exec(p, t, ip);
3602 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3604 struct thread *t = &p->threads[p->thread_id];
3605 struct instruction *ip = t->ip;
3608 __instr_alu_or_hm_exec(p, t, ip);
3615 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3617 struct thread *t = &p->threads[p->thread_id];
3618 struct instruction *ip = t->ip;
3621 __instr_alu_or_hh_exec(p, t, ip);
3628 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3630 struct thread *t = &p->threads[p->thread_id];
3631 struct instruction *ip = t->ip;
3634 __instr_alu_or_i_exec(p, t, ip);
3641 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3643 struct thread *t = &p->threads[p->thread_id];
3644 struct instruction *ip = t->ip;
3647 __instr_alu_xor_exec(p, t, ip);
3654 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3656 struct thread *t = &p->threads[p->thread_id];
3657 struct instruction *ip = t->ip;
3660 __instr_alu_xor_mh_exec(p, t, ip);
3667 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3669 struct thread *t = &p->threads[p->thread_id];
3670 struct instruction *ip = t->ip;
3673 __instr_alu_xor_hm_exec(p, t, ip);
3680 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3682 struct thread *t = &p->threads[p->thread_id];
3683 struct instruction *ip = t->ip;
3686 __instr_alu_xor_hh_exec(p, t, ip);
3693 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3695 struct thread *t = &p->threads[p->thread_id];
3696 struct instruction *ip = t->ip;
3699 __instr_alu_xor_i_exec(p, t, ip);
3706 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3708 struct thread *t = &p->threads[p->thread_id];
3709 struct instruction *ip = t->ip;
3712 __instr_alu_ckadd_field_exec(p, t, ip);
3719 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3721 struct thread *t = &p->threads[p->thread_id];
3722 struct instruction *ip = t->ip;
3725 __instr_alu_cksub_field_exec(p, t, ip);
3732 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3734 struct thread *t = &p->threads[p->thread_id];
3735 struct instruction *ip = t->ip;
3738 __instr_alu_ckadd_struct20_exec(p, t, ip);
3745 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3747 struct thread *t = &p->threads[p->thread_id];
3748 struct instruction *ip = t->ip;
3751 __instr_alu_ckadd_struct_exec(p, t, ip);
3760 static struct regarray *
3761 regarray_find(struct rte_swx_pipeline *p, const char *name);
3764 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3765 struct action *action,
3768 struct instruction *instr,
3769 struct instruction_data *data __rte_unused)
3771 char *regarray = tokens[1], *idx = tokens[2];
3774 uint32_t idx_struct_id, idx_val;
3776 CHECK(n_tokens == 3, EINVAL);
3778 r = regarray_find(p, regarray);
3781 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3782 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3784 CHECK(!fidx->var_size, EINVAL);
3786 instr->type = INSTR_REGPREFETCH_RM;
3788 instr->type = INSTR_REGPREFETCH_RH;
3790 instr->regarray.regarray_id = r->id;
3791 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3792 instr->regarray.idx.n_bits = fidx->n_bits;
3793 instr->regarray.idx.offset = fidx->offset / 8;
3794 instr->regarray.dstsrc_val = 0; /* Unused. */
3798 /* REGPREFETCH_RI. */
3799 idx_val = strtoul(idx, &idx, 0);
3800 CHECK(!idx[0], EINVAL);
3802 instr->type = INSTR_REGPREFETCH_RI;
3803 instr->regarray.regarray_id = r->id;
3804 instr->regarray.idx_val = idx_val;
3805 instr->regarray.dstsrc_val = 0; /* Unused. */
3810 instr_regrd_translate(struct rte_swx_pipeline *p,
3811 struct action *action,
3814 struct instruction *instr,
3815 struct instruction_data *data __rte_unused)
3817 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3819 struct field *fdst, *fidx;
3820 uint32_t dst_struct_id, idx_struct_id, idx_val;
3822 CHECK(n_tokens == 4, EINVAL);
3824 r = regarray_find(p, regarray);
3827 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3828 CHECK(fdst, EINVAL);
3829 CHECK(!fdst->var_size, EINVAL);
3831 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3832 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3834 CHECK(!fidx->var_size, EINVAL);
3836 instr->type = INSTR_REGRD_MRM;
3837 if (dst[0] == 'h' && idx[0] != 'h')
3838 instr->type = INSTR_REGRD_HRM;
3839 if (dst[0] != 'h' && idx[0] == 'h')
3840 instr->type = INSTR_REGRD_MRH;
3841 if (dst[0] == 'h' && idx[0] == 'h')
3842 instr->type = INSTR_REGRD_HRH;
3844 instr->regarray.regarray_id = r->id;
3845 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3846 instr->regarray.idx.n_bits = fidx->n_bits;
3847 instr->regarray.idx.offset = fidx->offset / 8;
3848 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3849 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3850 instr->regarray.dstsrc.offset = fdst->offset / 8;
3854 /* REGRD_MRI, REGRD_HRI. */
3855 idx_val = strtoul(idx, &idx, 0);
3856 CHECK(!idx[0], EINVAL);
3858 instr->type = INSTR_REGRD_MRI;
3860 instr->type = INSTR_REGRD_HRI;
3862 instr->regarray.regarray_id = r->id;
3863 instr->regarray.idx_val = idx_val;
3864 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3865 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3866 instr->regarray.dstsrc.offset = fdst->offset / 8;
3871 instr_regwr_translate(struct rte_swx_pipeline *p,
3872 struct action *action,
3875 struct instruction *instr,
3876 struct instruction_data *data __rte_unused)
3878 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3880 struct field *fidx, *fsrc;
3882 uint32_t idx_struct_id, idx_val, src_struct_id;
3884 CHECK(n_tokens == 4, EINVAL);
3886 r = regarray_find(p, regarray);
3889 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3890 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3891 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3893 CHECK(!fidx->var_size, EINVAL);
3894 CHECK(!fsrc->var_size, EINVAL);
3896 instr->type = INSTR_REGWR_RMM;
3897 if (idx[0] == 'h' && src[0] != 'h')
3898 instr->type = INSTR_REGWR_RHM;
3899 if (idx[0] != 'h' && src[0] == 'h')
3900 instr->type = INSTR_REGWR_RMH;
3901 if (idx[0] == 'h' && src[0] == 'h')
3902 instr->type = INSTR_REGWR_RHH;
3904 instr->regarray.regarray_id = r->id;
3905 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3906 instr->regarray.idx.n_bits = fidx->n_bits;
3907 instr->regarray.idx.offset = fidx->offset / 8;
3908 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3909 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3910 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3914 /* REGWR_RHI, REGWR_RMI. */
3915 if (fidx && !fsrc) {
3916 CHECK(!fidx->var_size, EINVAL);
3918 src_val = strtoull(src, &src, 0);
3919 CHECK(!src[0], EINVAL);
3921 instr->type = INSTR_REGWR_RMI;
3923 instr->type = INSTR_REGWR_RHI;
3925 instr->regarray.regarray_id = r->id;
3926 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3927 instr->regarray.idx.n_bits = fidx->n_bits;
3928 instr->regarray.idx.offset = fidx->offset / 8;
3929 instr->regarray.dstsrc_val = src_val;
3933 /* REGWR_RIH, REGWR_RIM. */
3934 if (!fidx && fsrc) {
3935 idx_val = strtoul(idx, &idx, 0);
3936 CHECK(!idx[0], EINVAL);
3938 CHECK(!fsrc->var_size, EINVAL);
3940 instr->type = INSTR_REGWR_RIM;
3942 instr->type = INSTR_REGWR_RIH;
3944 instr->regarray.regarray_id = r->id;
3945 instr->regarray.idx_val = idx_val;
3946 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3947 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3948 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3953 src_val = strtoull(src, &src, 0);
3954 CHECK(!src[0], EINVAL);
3956 idx_val = strtoul(idx, &idx, 0);
3957 CHECK(!idx[0], EINVAL);
3959 instr->type = INSTR_REGWR_RII;
3960 instr->regarray.idx_val = idx_val;
3961 instr->regarray.dstsrc_val = src_val;
3967 instr_regadd_translate(struct rte_swx_pipeline *p,
3968 struct action *action,
3971 struct instruction *instr,
3972 struct instruction_data *data __rte_unused)
3974 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3976 struct field *fidx, *fsrc;
3978 uint32_t idx_struct_id, idx_val, src_struct_id;
3980 CHECK(n_tokens == 4, EINVAL);
3982 r = regarray_find(p, regarray);
3985 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3986 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3987 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3989 CHECK(!fidx->var_size, EINVAL);
3990 CHECK(!fsrc->var_size, EINVAL);
3992 instr->type = INSTR_REGADD_RMM;
3993 if (idx[0] == 'h' && src[0] != 'h')
3994 instr->type = INSTR_REGADD_RHM;
3995 if (idx[0] != 'h' && src[0] == 'h')
3996 instr->type = INSTR_REGADD_RMH;
3997 if (idx[0] == 'h' && src[0] == 'h')
3998 instr->type = INSTR_REGADD_RHH;
4000 instr->regarray.regarray_id = r->id;
4001 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4002 instr->regarray.idx.n_bits = fidx->n_bits;
4003 instr->regarray.idx.offset = fidx->offset / 8;
4004 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4005 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4006 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4010 /* REGADD_RHI, REGADD_RMI. */
4011 if (fidx && !fsrc) {
4012 CHECK(!fidx->var_size, EINVAL);
4014 src_val = strtoull(src, &src, 0);
4015 CHECK(!src[0], EINVAL);
4017 instr->type = INSTR_REGADD_RMI;
4019 instr->type = INSTR_REGADD_RHI;
4021 instr->regarray.regarray_id = r->id;
4022 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4023 instr->regarray.idx.n_bits = fidx->n_bits;
4024 instr->regarray.idx.offset = fidx->offset / 8;
4025 instr->regarray.dstsrc_val = src_val;
4029 /* REGADD_RIH, REGADD_RIM. */
4030 if (!fidx && fsrc) {
4031 idx_val = strtoul(idx, &idx, 0);
4032 CHECK(!idx[0], EINVAL);
4034 CHECK(!fsrc->var_size, EINVAL);
4036 instr->type = INSTR_REGADD_RIM;
4038 instr->type = INSTR_REGADD_RIH;
4040 instr->regarray.regarray_id = r->id;
4041 instr->regarray.idx_val = idx_val;
4042 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4043 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4044 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4049 src_val = strtoull(src, &src, 0);
4050 CHECK(!src[0], EINVAL);
4052 idx_val = strtoul(idx, &idx, 0);
4053 CHECK(!idx[0], EINVAL);
4055 instr->type = INSTR_REGADD_RII;
4056 instr->regarray.idx_val = idx_val;
4057 instr->regarray.dstsrc_val = src_val;
4062 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4064 struct thread *t = &p->threads[p->thread_id];
4065 struct instruction *ip = t->ip;
4068 __instr_regprefetch_rh_exec(p, t, ip);
4075 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4077 struct thread *t = &p->threads[p->thread_id];
4078 struct instruction *ip = t->ip;
4081 __instr_regprefetch_rm_exec(p, t, ip);
4088 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4090 struct thread *t = &p->threads[p->thread_id];
4091 struct instruction *ip = t->ip;
4094 __instr_regprefetch_ri_exec(p, t, ip);
4101 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4103 struct thread *t = &p->threads[p->thread_id];
4104 struct instruction *ip = t->ip;
4107 __instr_regrd_hrh_exec(p, t, ip);
4114 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4116 struct thread *t = &p->threads[p->thread_id];
4117 struct instruction *ip = t->ip;
4120 __instr_regrd_hrm_exec(p, t, ip);
4127 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4129 struct thread *t = &p->threads[p->thread_id];
4130 struct instruction *ip = t->ip;
4133 __instr_regrd_mrh_exec(p, t, ip);
4140 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4142 struct thread *t = &p->threads[p->thread_id];
4143 struct instruction *ip = t->ip;
4146 __instr_regrd_mrm_exec(p, t, ip);
4153 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4155 struct thread *t = &p->threads[p->thread_id];
4156 struct instruction *ip = t->ip;
4159 __instr_regrd_hri_exec(p, t, ip);
4166 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4168 struct thread *t = &p->threads[p->thread_id];
4169 struct instruction *ip = t->ip;
4172 __instr_regrd_mri_exec(p, t, ip);
4179 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4181 struct thread *t = &p->threads[p->thread_id];
4182 struct instruction *ip = t->ip;
4185 __instr_regwr_rhh_exec(p, t, ip);
4192 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4194 struct thread *t = &p->threads[p->thread_id];
4195 struct instruction *ip = t->ip;
4198 __instr_regwr_rhm_exec(p, t, ip);
4205 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4207 struct thread *t = &p->threads[p->thread_id];
4208 struct instruction *ip = t->ip;
4211 __instr_regwr_rmh_exec(p, t, ip);
4218 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4220 struct thread *t = &p->threads[p->thread_id];
4221 struct instruction *ip = t->ip;
4224 __instr_regwr_rmm_exec(p, t, ip);
4231 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4233 struct thread *t = &p->threads[p->thread_id];
4234 struct instruction *ip = t->ip;
4237 __instr_regwr_rhi_exec(p, t, ip);
4244 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4246 struct thread *t = &p->threads[p->thread_id];
4247 struct instruction *ip = t->ip;
4250 __instr_regwr_rmi_exec(p, t, ip);
4257 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4259 struct thread *t = &p->threads[p->thread_id];
4260 struct instruction *ip = t->ip;
4263 __instr_regwr_rih_exec(p, t, ip);
4270 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4272 struct thread *t = &p->threads[p->thread_id];
4273 struct instruction *ip = t->ip;
4276 __instr_regwr_rim_exec(p, t, ip);
4283 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4285 struct thread *t = &p->threads[p->thread_id];
4286 struct instruction *ip = t->ip;
4289 __instr_regwr_rii_exec(p, t, ip);
4296 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4298 struct thread *t = &p->threads[p->thread_id];
4299 struct instruction *ip = t->ip;
4302 __instr_regadd_rhh_exec(p, t, ip);
4309 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4311 struct thread *t = &p->threads[p->thread_id];
4312 struct instruction *ip = t->ip;
4315 __instr_regadd_rhm_exec(p, t, ip);
4322 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4324 struct thread *t = &p->threads[p->thread_id];
4325 struct instruction *ip = t->ip;
4328 __instr_regadd_rmh_exec(p, t, ip);
4335 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4337 struct thread *t = &p->threads[p->thread_id];
4338 struct instruction *ip = t->ip;
4341 __instr_regadd_rmm_exec(p, t, ip);
4348 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4350 struct thread *t = &p->threads[p->thread_id];
4351 struct instruction *ip = t->ip;
4354 __instr_regadd_rhi_exec(p, t, ip);
4361 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4363 struct thread *t = &p->threads[p->thread_id];
4364 struct instruction *ip = t->ip;
4367 __instr_regadd_rmi_exec(p, t, ip);
4374 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4376 struct thread *t = &p->threads[p->thread_id];
4377 struct instruction *ip = t->ip;
4380 __instr_regadd_rih_exec(p, t, ip);
4387 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4389 struct thread *t = &p->threads[p->thread_id];
4390 struct instruction *ip = t->ip;
4393 __instr_regadd_rim_exec(p, t, ip);
4400 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4402 struct thread *t = &p->threads[p->thread_id];
4403 struct instruction *ip = t->ip;
4406 __instr_regadd_rii_exec(p, t, ip);
4415 static struct metarray *
4416 metarray_find(struct rte_swx_pipeline *p, const char *name);
4419 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4420 struct action *action,
4423 struct instruction *instr,
4424 struct instruction_data *data __rte_unused)
4426 char *metarray = tokens[1], *idx = tokens[2];
4429 uint32_t idx_struct_id, idx_val;
4431 CHECK(n_tokens == 3, EINVAL);
4433 m = metarray_find(p, metarray);
4436 /* METPREFETCH_H, METPREFETCH_M. */
4437 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4439 CHECK(!fidx->var_size, EINVAL);
4441 instr->type = INSTR_METPREFETCH_M;
4443 instr->type = INSTR_METPREFETCH_H;
4445 instr->meter.metarray_id = m->id;
4446 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4447 instr->meter.idx.n_bits = fidx->n_bits;
4448 instr->meter.idx.offset = fidx->offset / 8;
4452 /* METPREFETCH_I. */
4453 idx_val = strtoul(idx, &idx, 0);
4454 CHECK(!idx[0], EINVAL);
4456 instr->type = INSTR_METPREFETCH_I;
4457 instr->meter.metarray_id = m->id;
4458 instr->meter.idx_val = idx_val;
4463 instr_meter_translate(struct rte_swx_pipeline *p,
4464 struct action *action,
4467 struct instruction *instr,
4468 struct instruction_data *data __rte_unused)
4470 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4471 char *color_in = tokens[4], *color_out = tokens[5];
4473 struct field *fidx, *flength, *fcin, *fcout;
4474 uint32_t idx_struct_id, length_struct_id;
4475 uint32_t color_in_struct_id, color_out_struct_id;
4477 CHECK(n_tokens == 6, EINVAL);
4479 m = metarray_find(p, metarray);
4482 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4484 flength = struct_field_parse(p, action, length, &length_struct_id);
4485 CHECK(flength, EINVAL);
4486 CHECK(!flength->var_size, EINVAL);
4488 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4490 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4491 CHECK(fcout, EINVAL);
4492 CHECK(!fcout->var_size, EINVAL);
4494 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4496 CHECK(!fidx->var_size, EINVAL);
4497 CHECK(!fcin->var_size, EINVAL);
4499 instr->type = INSTR_METER_MMM;
4500 if (idx[0] == 'h' && length[0] == 'h')
4501 instr->type = INSTR_METER_HHM;
4502 if (idx[0] == 'h' && length[0] != 'h')
4503 instr->type = INSTR_METER_HMM;
4504 if (idx[0] != 'h' && length[0] == 'h')
4505 instr->type = INSTR_METER_MHM;
4507 instr->meter.metarray_id = m->id;
4509 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4510 instr->meter.idx.n_bits = fidx->n_bits;
4511 instr->meter.idx.offset = fidx->offset / 8;
4513 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4514 instr->meter.length.n_bits = flength->n_bits;
4515 instr->meter.length.offset = flength->offset / 8;
4517 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4518 instr->meter.color_in.n_bits = fcin->n_bits;
4519 instr->meter.color_in.offset = fcin->offset / 8;
4521 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4522 instr->meter.color_out.n_bits = fcout->n_bits;
4523 instr->meter.color_out.offset = fcout->offset / 8;
4528 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4529 if (fidx && !fcin) {
4530 uint32_t color_in_val;
4532 CHECK(!fidx->var_size, EINVAL);
4534 color_in_val = strtoul(color_in, &color_in, 0);
4535 CHECK(!color_in[0], EINVAL);
4537 instr->type = INSTR_METER_MMI;
4538 if (idx[0] == 'h' && length[0] == 'h')
4539 instr->type = INSTR_METER_HHI;
4540 if (idx[0] == 'h' && length[0] != 'h')
4541 instr->type = INSTR_METER_HMI;
4542 if (idx[0] != 'h' && length[0] == 'h')
4543 instr->type = INSTR_METER_MHI;
4545 instr->meter.metarray_id = m->id;
4547 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4548 instr->meter.idx.n_bits = fidx->n_bits;
4549 instr->meter.idx.offset = fidx->offset / 8;
4551 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4552 instr->meter.length.n_bits = flength->n_bits;
4553 instr->meter.length.offset = flength->offset / 8;
4555 instr->meter.color_in_val = color_in_val;
4557 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4558 instr->meter.color_out.n_bits = fcout->n_bits;
4559 instr->meter.color_out.offset = fcout->offset / 8;
4564 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4565 if (!fidx && fcin) {
4568 idx_val = strtoul(idx, &idx, 0);
4569 CHECK(!idx[0], EINVAL);
4571 CHECK(!fcin->var_size, EINVAL);
4573 instr->type = INSTR_METER_IMM;
4574 if (length[0] == 'h')
4575 instr->type = INSTR_METER_IHM;
4577 instr->meter.metarray_id = m->id;
4579 instr->meter.idx_val = idx_val;
4581 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4582 instr->meter.length.n_bits = flength->n_bits;
4583 instr->meter.length.offset = flength->offset / 8;
4585 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4586 instr->meter.color_in.n_bits = fcin->n_bits;
4587 instr->meter.color_in.offset = fcin->offset / 8;
4589 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4590 instr->meter.color_out.n_bits = fcout->n_bits;
4591 instr->meter.color_out.offset = fcout->offset / 8;
4596 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4597 if (!fidx && !fcin) {
4598 uint32_t idx_val, color_in_val;
4600 idx_val = strtoul(idx, &idx, 0);
4601 CHECK(!idx[0], EINVAL);
4603 color_in_val = strtoul(color_in, &color_in, 0);
4604 CHECK(!color_in[0], EINVAL);
4606 instr->type = INSTR_METER_IMI;
4607 if (length[0] == 'h')
4608 instr->type = INSTR_METER_IHI;
4610 instr->meter.metarray_id = m->id;
4612 instr->meter.idx_val = idx_val;
4614 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4615 instr->meter.length.n_bits = flength->n_bits;
4616 instr->meter.length.offset = flength->offset / 8;
4618 instr->meter.color_in_val = color_in_val;
4620 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4621 instr->meter.color_out.n_bits = fcout->n_bits;
4622 instr->meter.color_out.offset = fcout->offset / 8;
4631 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4633 struct thread *t = &p->threads[p->thread_id];
4634 struct instruction *ip = t->ip;
4637 __instr_metprefetch_h_exec(p, t, ip);
4644 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4646 struct thread *t = &p->threads[p->thread_id];
4647 struct instruction *ip = t->ip;
4650 __instr_metprefetch_m_exec(p, t, ip);
4657 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4659 struct thread *t = &p->threads[p->thread_id];
4660 struct instruction *ip = t->ip;
4663 __instr_metprefetch_i_exec(p, t, ip);
4670 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4672 struct thread *t = &p->threads[p->thread_id];
4673 struct instruction *ip = t->ip;
4676 __instr_meter_hhm_exec(p, t, ip);
4683 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4685 struct thread *t = &p->threads[p->thread_id];
4686 struct instruction *ip = t->ip;
4689 __instr_meter_hhi_exec(p, t, ip);
4696 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4698 struct thread *t = &p->threads[p->thread_id];
4699 struct instruction *ip = t->ip;
4702 __instr_meter_hmm_exec(p, t, ip);
4709 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4711 struct thread *t = &p->threads[p->thread_id];
4712 struct instruction *ip = t->ip;
4715 __instr_meter_hmi_exec(p, t, ip);
4722 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4724 struct thread *t = &p->threads[p->thread_id];
4725 struct instruction *ip = t->ip;
4728 __instr_meter_mhm_exec(p, t, ip);
4735 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4737 struct thread *t = &p->threads[p->thread_id];
4738 struct instruction *ip = t->ip;
4741 __instr_meter_mhi_exec(p, t, ip);
4748 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4750 struct thread *t = &p->threads[p->thread_id];
4751 struct instruction *ip = t->ip;
4754 __instr_meter_mmm_exec(p, t, ip);
4761 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4763 struct thread *t = &p->threads[p->thread_id];
4764 struct instruction *ip = t->ip;
4767 __instr_meter_mmi_exec(p, t, ip);
4774 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4776 struct thread *t = &p->threads[p->thread_id];
4777 struct instruction *ip = t->ip;
4780 __instr_meter_ihm_exec(p, t, ip);
4787 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4789 struct thread *t = &p->threads[p->thread_id];
4790 struct instruction *ip = t->ip;
4793 __instr_meter_ihi_exec(p, t, ip);
4800 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4802 struct thread *t = &p->threads[p->thread_id];
4803 struct instruction *ip = t->ip;
4806 __instr_meter_imm_exec(p, t, ip);
4813 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4815 struct thread *t = &p->threads[p->thread_id];
4816 struct instruction *ip = t->ip;
4819 __instr_meter_imi_exec(p, t, ip);
4829 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4830 struct action *action __rte_unused,
4833 struct instruction *instr,
4834 struct instruction_data *data)
4836 CHECK(n_tokens == 2, EINVAL);
4838 strcpy(data->jmp_label, tokens[1]);
4840 instr->type = INSTR_JMP;
4841 instr->jmp.ip = NULL; /* Resolved later. */
4846 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4847 struct action *action __rte_unused,
4850 struct instruction *instr,
4851 struct instruction_data *data)
4855 CHECK(n_tokens == 3, EINVAL);
4857 strcpy(data->jmp_label, tokens[1]);
4859 h = header_parse(p, tokens[2]);
4862 instr->type = INSTR_JMP_VALID;
4863 instr->jmp.ip = NULL; /* Resolved later. */
4864 instr->jmp.header_id = h->id;
4869 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4870 struct action *action __rte_unused,
4873 struct instruction *instr,
4874 struct instruction_data *data)
4878 CHECK(n_tokens == 3, EINVAL);
4880 strcpy(data->jmp_label, tokens[1]);
4882 h = header_parse(p, tokens[2]);
4885 instr->type = INSTR_JMP_INVALID;
4886 instr->jmp.ip = NULL; /* Resolved later. */
4887 instr->jmp.header_id = h->id;
4892 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4893 struct action *action,
4896 struct instruction *instr,
4897 struct instruction_data *data)
4899 CHECK(!action, EINVAL);
4900 CHECK(n_tokens == 2, EINVAL);
4902 strcpy(data->jmp_label, tokens[1]);
4904 instr->type = INSTR_JMP_HIT;
4905 instr->jmp.ip = NULL; /* Resolved later. */
4910 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4911 struct action *action,
4914 struct instruction *instr,
4915 struct instruction_data *data)
4917 CHECK(!action, EINVAL);
4918 CHECK(n_tokens == 2, EINVAL);
4920 strcpy(data->jmp_label, tokens[1]);
4922 instr->type = INSTR_JMP_MISS;
4923 instr->jmp.ip = NULL; /* Resolved later. */
4928 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4929 struct action *action,
4932 struct instruction *instr,
4933 struct instruction_data *data)
4937 CHECK(!action, EINVAL);
4938 CHECK(n_tokens == 3, EINVAL);
4940 strcpy(data->jmp_label, tokens[1]);
4942 a = action_find(p, tokens[2]);
4945 instr->type = INSTR_JMP_ACTION_HIT;
4946 instr->jmp.ip = NULL; /* Resolved later. */
4947 instr->jmp.action_id = a->id;
4952 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4953 struct action *action,
4956 struct instruction *instr,
4957 struct instruction_data *data)
4961 CHECK(!action, EINVAL);
4962 CHECK(n_tokens == 3, EINVAL);
4964 strcpy(data->jmp_label, tokens[1]);
4966 a = action_find(p, tokens[2]);
4969 instr->type = INSTR_JMP_ACTION_MISS;
4970 instr->jmp.ip = NULL; /* Resolved later. */
4971 instr->jmp.action_id = a->id;
4976 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4977 struct action *action,
4980 struct instruction *instr,
4981 struct instruction_data *data)
4983 char *a = tokens[2], *b = tokens[3];
4984 struct field *fa, *fb;
4986 uint32_t a_struct_id, b_struct_id;
4988 CHECK(n_tokens == 4, EINVAL);
4990 strcpy(data->jmp_label, tokens[1]);
4992 fa = struct_field_parse(p, action, a, &a_struct_id);
4994 CHECK(!fa->var_size, EINVAL);
4996 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4997 fb = struct_field_parse(p, action, b, &b_struct_id);
4999 CHECK(!fb->var_size, EINVAL);
5001 instr->type = INSTR_JMP_EQ;
5002 if (a[0] != 'h' && b[0] == 'h')
5003 instr->type = INSTR_JMP_EQ_MH;
5004 if (a[0] == 'h' && b[0] != 'h')
5005 instr->type = INSTR_JMP_EQ_HM;
5006 if (a[0] == 'h' && b[0] == 'h')
5007 instr->type = INSTR_JMP_EQ_HH;
5008 instr->jmp.ip = NULL; /* Resolved later. */
5010 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5011 instr->jmp.a.n_bits = fa->n_bits;
5012 instr->jmp.a.offset = fa->offset / 8;
5013 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5014 instr->jmp.b.n_bits = fb->n_bits;
5015 instr->jmp.b.offset = fb->offset / 8;
5020 b_val = strtoull(b, &b, 0);
5021 CHECK(!b[0], EINVAL);
5024 b_val = hton64(b_val) >> (64 - fa->n_bits);
5026 instr->type = INSTR_JMP_EQ_I;
5027 instr->jmp.ip = NULL; /* Resolved later. */
5028 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5029 instr->jmp.a.n_bits = fa->n_bits;
5030 instr->jmp.a.offset = fa->offset / 8;
5031 instr->jmp.b_val = b_val;
5036 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5037 struct action *action,
5040 struct instruction *instr,
5041 struct instruction_data *data)
5043 char *a = tokens[2], *b = tokens[3];
5044 struct field *fa, *fb;
5046 uint32_t a_struct_id, b_struct_id;
5048 CHECK(n_tokens == 4, EINVAL);
5050 strcpy(data->jmp_label, tokens[1]);
5052 fa = struct_field_parse(p, action, a, &a_struct_id);
5054 CHECK(!fa->var_size, EINVAL);
5056 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5057 fb = struct_field_parse(p, action, b, &b_struct_id);
5059 CHECK(!fb->var_size, EINVAL);
5061 instr->type = INSTR_JMP_NEQ;
5062 if (a[0] != 'h' && b[0] == 'h')
5063 instr->type = INSTR_JMP_NEQ_MH;
5064 if (a[0] == 'h' && b[0] != 'h')
5065 instr->type = INSTR_JMP_NEQ_HM;
5066 if (a[0] == 'h' && b[0] == 'h')
5067 instr->type = INSTR_JMP_NEQ_HH;
5068 instr->jmp.ip = NULL; /* Resolved later. */
5070 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5071 instr->jmp.a.n_bits = fa->n_bits;
5072 instr->jmp.a.offset = fa->offset / 8;
5073 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5074 instr->jmp.b.n_bits = fb->n_bits;
5075 instr->jmp.b.offset = fb->offset / 8;
5080 b_val = strtoull(b, &b, 0);
5081 CHECK(!b[0], EINVAL);
5084 b_val = hton64(b_val) >> (64 - fa->n_bits);
5086 instr->type = INSTR_JMP_NEQ_I;
5087 instr->jmp.ip = NULL; /* Resolved later. */
5088 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5089 instr->jmp.a.n_bits = fa->n_bits;
5090 instr->jmp.a.offset = fa->offset / 8;
5091 instr->jmp.b_val = b_val;
5096 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5097 struct action *action,
5100 struct instruction *instr,
5101 struct instruction_data *data)
5103 char *a = tokens[2], *b = tokens[3];
5104 struct field *fa, *fb;
5106 uint32_t a_struct_id, b_struct_id;
5108 CHECK(n_tokens == 4, EINVAL);
5110 strcpy(data->jmp_label, tokens[1]);
5112 fa = struct_field_parse(p, action, a, &a_struct_id);
5114 CHECK(!fa->var_size, EINVAL);
5116 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5117 fb = struct_field_parse(p, action, b, &b_struct_id);
5119 CHECK(!fb->var_size, EINVAL);
5121 instr->type = INSTR_JMP_LT;
5122 if (a[0] == 'h' && b[0] != 'h')
5123 instr->type = INSTR_JMP_LT_HM;
5124 if (a[0] != 'h' && b[0] == 'h')
5125 instr->type = INSTR_JMP_LT_MH;
5126 if (a[0] == 'h' && b[0] == 'h')
5127 instr->type = INSTR_JMP_LT_HH;
5128 instr->jmp.ip = NULL; /* Resolved later. */
5130 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5131 instr->jmp.a.n_bits = fa->n_bits;
5132 instr->jmp.a.offset = fa->offset / 8;
5133 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5134 instr->jmp.b.n_bits = fb->n_bits;
5135 instr->jmp.b.offset = fb->offset / 8;
5139 /* JMP_LT_MI, JMP_LT_HI. */
5140 b_val = strtoull(b, &b, 0);
5141 CHECK(!b[0], EINVAL);
5143 instr->type = INSTR_JMP_LT_MI;
5145 instr->type = INSTR_JMP_LT_HI;
5146 instr->jmp.ip = NULL; /* Resolved later. */
5148 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5149 instr->jmp.a.n_bits = fa->n_bits;
5150 instr->jmp.a.offset = fa->offset / 8;
5151 instr->jmp.b_val = b_val;
5156 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5157 struct action *action,
5160 struct instruction *instr,
5161 struct instruction_data *data)
5163 char *a = tokens[2], *b = tokens[3];
5164 struct field *fa, *fb;
5166 uint32_t a_struct_id, b_struct_id;
5168 CHECK(n_tokens == 4, EINVAL);
5170 strcpy(data->jmp_label, tokens[1]);
5172 fa = struct_field_parse(p, action, a, &a_struct_id);
5174 CHECK(!fa->var_size, EINVAL);
5176 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5177 fb = struct_field_parse(p, action, b, &b_struct_id);
5179 CHECK(!fb->var_size, EINVAL);
5181 instr->type = INSTR_JMP_GT;
5182 if (a[0] == 'h' && b[0] != 'h')
5183 instr->type = INSTR_JMP_GT_HM;
5184 if (a[0] != 'h' && b[0] == 'h')
5185 instr->type = INSTR_JMP_GT_MH;
5186 if (a[0] == 'h' && b[0] == 'h')
5187 instr->type = INSTR_JMP_GT_HH;
5188 instr->jmp.ip = NULL; /* Resolved later. */
5190 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5191 instr->jmp.a.n_bits = fa->n_bits;
5192 instr->jmp.a.offset = fa->offset / 8;
5193 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5194 instr->jmp.b.n_bits = fb->n_bits;
5195 instr->jmp.b.offset = fb->offset / 8;
5199 /* JMP_GT_MI, JMP_GT_HI. */
5200 b_val = strtoull(b, &b, 0);
5201 CHECK(!b[0], EINVAL);
5203 instr->type = INSTR_JMP_GT_MI;
5205 instr->type = INSTR_JMP_GT_HI;
5206 instr->jmp.ip = NULL; /* Resolved later. */
5208 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5209 instr->jmp.a.n_bits = fa->n_bits;
5210 instr->jmp.a.offset = fa->offset / 8;
5211 instr->jmp.b_val = b_val;
5216 instr_jmp_exec(struct rte_swx_pipeline *p)
5218 struct thread *t = &p->threads[p->thread_id];
5219 struct instruction *ip = t->ip;
5221 TRACE("[Thread %2u] jmp\n", p->thread_id);
5223 thread_ip_set(t, ip->jmp.ip);
5227 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5229 struct thread *t = &p->threads[p->thread_id];
5230 struct instruction *ip = t->ip;
5231 uint32_t header_id = ip->jmp.header_id;
5233 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5235 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5239 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5241 struct thread *t = &p->threads[p->thread_id];
5242 struct instruction *ip = t->ip;
5243 uint32_t header_id = ip->jmp.header_id;
5245 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5247 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5251 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5253 struct thread *t = &p->threads[p->thread_id];
5254 struct instruction *ip = t->ip;
5255 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5257 TRACE("[Thread %2u] jmph\n", p->thread_id);
5259 t->ip = ip_next[t->hit];
5263 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5265 struct thread *t = &p->threads[p->thread_id];
5266 struct instruction *ip = t->ip;
5267 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5269 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5271 t->ip = ip_next[t->hit];
5275 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5277 struct thread *t = &p->threads[p->thread_id];
5278 struct instruction *ip = t->ip;
5280 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5282 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5286 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5288 struct thread *t = &p->threads[p->thread_id];
5289 struct instruction *ip = t->ip;
5291 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5293 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5297 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5299 struct thread *t = &p->threads[p->thread_id];
5300 struct instruction *ip = t->ip;
5302 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5308 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5310 struct thread *t = &p->threads[p->thread_id];
5311 struct instruction *ip = t->ip;
5313 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5315 JMP_CMP_MH(t, ip, ==);
5319 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5321 struct thread *t = &p->threads[p->thread_id];
5322 struct instruction *ip = t->ip;
5324 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5326 JMP_CMP_HM(t, ip, ==);
5330 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5332 struct thread *t = &p->threads[p->thread_id];
5333 struct instruction *ip = t->ip;
5335 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5337 JMP_CMP_HH_FAST(t, ip, ==);
5341 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5343 struct thread *t = &p->threads[p->thread_id];
5344 struct instruction *ip = t->ip;
5346 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5348 JMP_CMP_I(t, ip, ==);
5352 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5354 struct thread *t = &p->threads[p->thread_id];
5355 struct instruction *ip = t->ip;
5357 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5363 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5365 struct thread *t = &p->threads[p->thread_id];
5366 struct instruction *ip = t->ip;
5368 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5370 JMP_CMP_MH(t, ip, !=);
5374 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5376 struct thread *t = &p->threads[p->thread_id];
5377 struct instruction *ip = t->ip;
5379 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5381 JMP_CMP_HM(t, ip, !=);
5385 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5387 struct thread *t = &p->threads[p->thread_id];
5388 struct instruction *ip = t->ip;
5390 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5392 JMP_CMP_HH_FAST(t, ip, !=);
5396 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5398 struct thread *t = &p->threads[p->thread_id];
5399 struct instruction *ip = t->ip;
5401 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5403 JMP_CMP_I(t, ip, !=);
5407 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5409 struct thread *t = &p->threads[p->thread_id];
5410 struct instruction *ip = t->ip;
5412 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5418 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5420 struct thread *t = &p->threads[p->thread_id];
5421 struct instruction *ip = t->ip;
5423 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5425 JMP_CMP_MH(t, ip, <);
5429 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5431 struct thread *t = &p->threads[p->thread_id];
5432 struct instruction *ip = t->ip;
5434 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5436 JMP_CMP_HM(t, ip, <);
5440 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5442 struct thread *t = &p->threads[p->thread_id];
5443 struct instruction *ip = t->ip;
5445 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5447 JMP_CMP_HH(t, ip, <);
5451 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5453 struct thread *t = &p->threads[p->thread_id];
5454 struct instruction *ip = t->ip;
5456 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5458 JMP_CMP_MI(t, ip, <);
5462 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5464 struct thread *t = &p->threads[p->thread_id];
5465 struct instruction *ip = t->ip;
5467 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5469 JMP_CMP_HI(t, ip, <);
5473 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5475 struct thread *t = &p->threads[p->thread_id];
5476 struct instruction *ip = t->ip;
5478 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5484 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5486 struct thread *t = &p->threads[p->thread_id];
5487 struct instruction *ip = t->ip;
5489 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5491 JMP_CMP_MH(t, ip, >);
5495 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5497 struct thread *t = &p->threads[p->thread_id];
5498 struct instruction *ip = t->ip;
5500 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5502 JMP_CMP_HM(t, ip, >);
5506 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5508 struct thread *t = &p->threads[p->thread_id];
5509 struct instruction *ip = t->ip;
5511 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5513 JMP_CMP_HH(t, ip, >);
5517 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5519 struct thread *t = &p->threads[p->thread_id];
5520 struct instruction *ip = t->ip;
5522 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5524 JMP_CMP_MI(t, ip, >);
5528 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5530 struct thread *t = &p->threads[p->thread_id];
5531 struct instruction *ip = t->ip;
5533 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5535 JMP_CMP_HI(t, ip, >);
5542 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5543 struct action *action,
5544 char **tokens __rte_unused,
5546 struct instruction *instr,
5547 struct instruction_data *data __rte_unused)
5549 CHECK(action, EINVAL);
5550 CHECK(n_tokens == 1, EINVAL);
5552 instr->type = INSTR_RETURN;
5557 instr_return_exec(struct rte_swx_pipeline *p)
5559 struct thread *t = &p->threads[p->thread_id];
5561 TRACE("[Thread %2u] return\n", p->thread_id);
5567 instr_translate(struct rte_swx_pipeline *p,
5568 struct action *action,
5570 struct instruction *instr,
5571 struct instruction_data *data)
5573 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5574 int n_tokens = 0, tpos = 0;
5576 /* Parse the instruction string into tokens. */
5580 token = strtok_r(string, " \t\v", &string);
5584 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5585 CHECK_NAME(token, EINVAL);
5587 tokens[n_tokens] = token;
5591 CHECK(n_tokens, EINVAL);
5593 /* Handle the optional instruction label. */
5594 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5595 strcpy(data->label, tokens[0]);
5598 CHECK(n_tokens - tpos, EINVAL);
5601 /* Identify the instruction type. */
5602 if (!strcmp(tokens[tpos], "rx"))
5603 return instr_rx_translate(p,
5610 if (!strcmp(tokens[tpos], "tx"))
5611 return instr_tx_translate(p,
5618 if (!strcmp(tokens[tpos], "drop"))
5619 return instr_drop_translate(p,
5626 if (!strcmp(tokens[tpos], "extract"))
5627 return instr_hdr_extract_translate(p,
5634 if (!strcmp(tokens[tpos], "lookahead"))
5635 return instr_hdr_lookahead_translate(p,
5642 if (!strcmp(tokens[tpos], "emit"))
5643 return instr_hdr_emit_translate(p,
5650 if (!strcmp(tokens[tpos], "validate"))
5651 return instr_hdr_validate_translate(p,
5658 if (!strcmp(tokens[tpos], "invalidate"))
5659 return instr_hdr_invalidate_translate(p,
5666 if (!strcmp(tokens[tpos], "mov"))
5667 return instr_mov_translate(p,
5674 if (!strcmp(tokens[tpos], "add"))
5675 return instr_alu_add_translate(p,
5682 if (!strcmp(tokens[tpos], "sub"))
5683 return instr_alu_sub_translate(p,
5690 if (!strcmp(tokens[tpos], "ckadd"))
5691 return instr_alu_ckadd_translate(p,
5698 if (!strcmp(tokens[tpos], "cksub"))
5699 return instr_alu_cksub_translate(p,
5706 if (!strcmp(tokens[tpos], "and"))
5707 return instr_alu_and_translate(p,
5714 if (!strcmp(tokens[tpos], "or"))
5715 return instr_alu_or_translate(p,
5722 if (!strcmp(tokens[tpos], "xor"))
5723 return instr_alu_xor_translate(p,
5730 if (!strcmp(tokens[tpos], "shl"))
5731 return instr_alu_shl_translate(p,
5738 if (!strcmp(tokens[tpos], "shr"))
5739 return instr_alu_shr_translate(p,
5746 if (!strcmp(tokens[tpos], "regprefetch"))
5747 return instr_regprefetch_translate(p,
5754 if (!strcmp(tokens[tpos], "regrd"))
5755 return instr_regrd_translate(p,
5762 if (!strcmp(tokens[tpos], "regwr"))
5763 return instr_regwr_translate(p,
5770 if (!strcmp(tokens[tpos], "regadd"))
5771 return instr_regadd_translate(p,
5778 if (!strcmp(tokens[tpos], "metprefetch"))
5779 return instr_metprefetch_translate(p,
5786 if (!strcmp(tokens[tpos], "meter"))
5787 return instr_meter_translate(p,
5794 if (!strcmp(tokens[tpos], "table"))
5795 return instr_table_translate(p,
5802 if (!strcmp(tokens[tpos], "learn"))
5803 return instr_learn_translate(p,
5810 if (!strcmp(tokens[tpos], "forget"))
5811 return instr_forget_translate(p,
5818 if (!strcmp(tokens[tpos], "extern"))
5819 return instr_extern_translate(p,
5826 if (!strcmp(tokens[tpos], "jmp"))
5827 return instr_jmp_translate(p,
5834 if (!strcmp(tokens[tpos], "jmpv"))
5835 return instr_jmp_valid_translate(p,
5842 if (!strcmp(tokens[tpos], "jmpnv"))
5843 return instr_jmp_invalid_translate(p,
5850 if (!strcmp(tokens[tpos], "jmph"))
5851 return instr_jmp_hit_translate(p,
5858 if (!strcmp(tokens[tpos], "jmpnh"))
5859 return instr_jmp_miss_translate(p,
5866 if (!strcmp(tokens[tpos], "jmpa"))
5867 return instr_jmp_action_hit_translate(p,
5874 if (!strcmp(tokens[tpos], "jmpna"))
5875 return instr_jmp_action_miss_translate(p,
5882 if (!strcmp(tokens[tpos], "jmpeq"))
5883 return instr_jmp_eq_translate(p,
5890 if (!strcmp(tokens[tpos], "jmpneq"))
5891 return instr_jmp_neq_translate(p,
5898 if (!strcmp(tokens[tpos], "jmplt"))
5899 return instr_jmp_lt_translate(p,
5906 if (!strcmp(tokens[tpos], "jmpgt"))
5907 return instr_jmp_gt_translate(p,
5914 if (!strcmp(tokens[tpos], "return"))
5915 return instr_return_translate(p,
5925 static struct instruction_data *
5926 label_find(struct instruction_data *data, uint32_t n, const char *label)
5930 for (i = 0; i < n; i++)
5931 if (!strcmp(label, data[i].label))
5938 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5940 uint32_t count = 0, i;
5945 for (i = 0; i < n; i++)
5946 if (!strcmp(label, data[i].jmp_label))
5953 instr_label_check(struct instruction_data *instruction_data,
5954 uint32_t n_instructions)
5958 /* Check that all instruction labels are unique. */
5959 for (i = 0; i < n_instructions; i++) {
5960 struct instruction_data *data = &instruction_data[i];
5961 char *label = data->label;
5967 for (j = i + 1; j < n_instructions; j++)
5968 CHECK(strcmp(label, data[j].label), EINVAL);
5971 /* Get users for each instruction label. */
5972 for (i = 0; i < n_instructions; i++) {
5973 struct instruction_data *data = &instruction_data[i];
5974 char *label = data->label;
5976 data->n_users = label_is_used(instruction_data,
5985 instr_jmp_resolve(struct instruction *instructions,
5986 struct instruction_data *instruction_data,
5987 uint32_t n_instructions)
5991 for (i = 0; i < n_instructions; i++) {
5992 struct instruction *instr = &instructions[i];
5993 struct instruction_data *data = &instruction_data[i];
5994 struct instruction_data *found;
5996 if (!instruction_is_jmp(instr))
5999 found = label_find(instruction_data,
6002 CHECK(found, EINVAL);
6004 instr->jmp.ip = &instructions[found - instruction_data];
6011 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6013 struct instruction *instr,
6014 struct instruction_data *data __rte_unused,
6015 uint32_t n_instructions)
6018 enum instruction_type type;
6021 /* Check that the first instruction is rx. */
6022 CHECK(instr[0].type == INSTR_RX, EINVAL);
6024 /* Check that there is at least one tx instruction. */
6025 for (i = 0; i < n_instructions; i++) {
6026 type = instr[i].type;
6028 if (instruction_is_tx(type))
6031 CHECK(i < n_instructions, EINVAL);
6033 /* Check that the last instruction is either tx or unconditional
6036 type = instr[n_instructions - 1].type;
6037 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6041 enum instruction_type type;
6044 /* Check that there is at least one return or tx instruction. */
6045 for (i = 0; i < n_instructions; i++) {
6046 type = instr[i].type;
6048 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6051 CHECK(i < n_instructions, EINVAL);
6058 instr_compact(struct instruction *instructions,
6059 struct instruction_data *instruction_data,
6060 uint32_t n_instructions)
6062 uint32_t i, pos = 0;
6064 /* Eliminate the invalid instructions that have been optimized out. */
6065 for (i = 0; i < n_instructions; i++) {
6066 struct instruction *instr = &instructions[i];
6067 struct instruction_data *data = &instruction_data[i];
6073 memcpy(&instructions[pos], instr, sizeof(*instr));
6074 memcpy(&instruction_data[pos], data, sizeof(*data));
6084 instr_pattern_extract_many_search(struct instruction *instr,
6085 struct instruction_data *data,
6087 uint32_t *n_pattern_instr)
6091 for (i = 0; i < n_instr; i++) {
6092 if (data[i].invalid)
6095 if (instr[i].type != INSTR_HDR_EXTRACT)
6098 if (i == RTE_DIM(instr->io.hdr.header_id))
6101 if (i && data[i].n_users)
6108 *n_pattern_instr = i;
6113 instr_pattern_extract_many_replace(struct instruction *instr,
6114 struct instruction_data *data,
6119 for (i = 1; i < n_instr; i++) {
6121 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6122 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6123 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6125 data[i].invalid = 1;
6130 instr_pattern_extract_many_optimize(struct instruction *instructions,
6131 struct instruction_data *instruction_data,
6132 uint32_t n_instructions)
6136 for (i = 0; i < n_instructions; ) {
6137 struct instruction *instr = &instructions[i];
6138 struct instruction_data *data = &instruction_data[i];
6139 uint32_t n_instr = 0;
6143 detected = instr_pattern_extract_many_search(instr,
6148 instr_pattern_extract_many_replace(instr,
6155 /* No pattern starting at the current instruction. */
6159 /* Eliminate the invalid instructions that have been optimized out. */
6160 n_instructions = instr_compact(instructions,
6164 return n_instructions;
6168 instr_pattern_emit_many_tx_search(struct instruction *instr,
6169 struct instruction_data *data,
6171 uint32_t *n_pattern_instr)
6175 for (i = 0; i < n_instr; i++) {
6176 if (data[i].invalid)
6179 if (instr[i].type != INSTR_HDR_EMIT)
6182 if (i == RTE_DIM(instr->io.hdr.header_id))
6185 if (i && data[i].n_users)
6192 if (!instruction_is_tx(instr[i].type))
6195 if (data[i].n_users)
6200 *n_pattern_instr = i;
6205 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6206 struct instruction_data *data,
6211 /* Any emit instruction in addition to the first one. */
6212 for (i = 1; i < n_instr - 1; i++) {
6214 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6215 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6216 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6218 data[i].invalid = 1;
6221 /* The TX instruction is the last one in the pattern. */
6223 instr[0].io.io.offset = instr[i].io.io.offset;
6224 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6225 data[i].invalid = 1;
6229 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6230 struct instruction_data *instruction_data,
6231 uint32_t n_instructions)
6235 for (i = 0; i < n_instructions; ) {
6236 struct instruction *instr = &instructions[i];
6237 struct instruction_data *data = &instruction_data[i];
6238 uint32_t n_instr = 0;
6241 /* Emit many + TX. */
6242 detected = instr_pattern_emit_many_tx_search(instr,
6247 instr_pattern_emit_many_tx_replace(instr,
6254 /* No pattern starting at the current instruction. */
6258 /* Eliminate the invalid instructions that have been optimized out. */
6259 n_instructions = instr_compact(instructions,
6263 return n_instructions;
6267 action_arg_src_mov_count(struct action *a,
6269 struct instruction *instructions,
6270 struct instruction_data *instruction_data,
6271 uint32_t n_instructions);
6274 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6276 struct instruction *instr,
6277 struct instruction_data *data,
6279 struct instruction *instructions,
6280 struct instruction_data *instruction_data,
6281 uint32_t n_instructions,
6282 uint32_t *n_pattern_instr)
6285 uint32_t src_field_id, i, j;
6287 /* Prerequisites. */
6291 /* First instruction: MOV_HM. */
6292 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6295 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6296 if (!h || h->st->var_size)
6299 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6300 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6303 if (src_field_id == a->st->n_fields)
6306 if (instr[0].mov.dst.offset ||
6307 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6308 instr[0].mov.src.struct_id ||
6309 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6310 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6313 if ((n_instr < h->st->n_fields + 1) ||
6314 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6317 /* Subsequent instructions: MOV_HM. */
6318 for (i = 1; i < h->st->n_fields; i++)
6319 if (data[i].invalid ||
6321 (instr[i].type != INSTR_MOV_HM) ||
6322 (instr[i].mov.dst.struct_id != h->struct_id) ||
6323 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6324 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6325 instr[i].mov.src.struct_id ||
6326 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6327 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6328 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6331 /* Last instruction: HDR_VALIDATE. */
6332 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6333 (instr[i].valid.header_id != h->id))
6336 /* Check that none of the action args that are used as source for this
6337 * DMA transfer are not used as source in any other mov instruction.
6339 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6342 n_users = action_arg_src_mov_count(a,
6351 *n_pattern_instr = 1 + i;
6356 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6358 struct instruction *instr,
6359 struct instruction_data *data,
6363 uint32_t src_field_id, src_offset, i;
6365 /* Read from the instructions before they are modified. */
6366 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6370 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6371 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6374 if (src_field_id == a->st->n_fields)
6377 src_offset = instr[0].mov.src.offset;
6379 /* Modify the instructions. */
6380 instr[0].type = INSTR_DMA_HT;
6381 instr[0].dma.dst.header_id[0] = h->id;
6382 instr[0].dma.dst.struct_id[0] = h->struct_id;
6383 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6384 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6386 for (i = 1; i < n_instr; i++)
6387 data[i].invalid = 1;
6389 /* Update the endianness of the action arguments to header endianness. */
6390 for (i = 0; i < h->st->n_fields; i++)
6391 a->args_endianness[src_field_id + i] = 1;
6395 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6397 struct instruction *instructions,
6398 struct instruction_data *instruction_data,
6399 uint32_t n_instructions)
6404 return n_instructions;
6406 for (i = 0; i < n_instructions; ) {
6407 struct instruction *instr = &instructions[i];
6408 struct instruction_data *data = &instruction_data[i];
6409 uint32_t n_instr = 0;
6412 /* Mov all + validate. */
6413 detected = instr_pattern_mov_all_validate_search(p,
6423 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6428 /* No pattern starting at the current instruction. */
6432 /* Eliminate the invalid instructions that have been optimized out. */
6433 n_instructions = instr_compact(instructions,
6437 return n_instructions;
6441 instr_pattern_dma_many_search(struct instruction *instr,
6442 struct instruction_data *data,
6444 uint32_t *n_pattern_instr)
6448 for (i = 0; i < n_instr; i++) {
6449 if (data[i].invalid)
6452 if (instr[i].type != INSTR_DMA_HT)
6455 if (i == RTE_DIM(instr->dma.dst.header_id))
6458 if (i && data[i].n_users)
6465 *n_pattern_instr = i;
6470 instr_pattern_dma_many_replace(struct instruction *instr,
6471 struct instruction_data *data,
6476 for (i = 1; i < n_instr; i++) {
6478 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6479 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6480 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6481 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6483 data[i].invalid = 1;
6488 instr_pattern_dma_many_optimize(struct instruction *instructions,
6489 struct instruction_data *instruction_data,
6490 uint32_t n_instructions)
6494 for (i = 0; i < n_instructions; ) {
6495 struct instruction *instr = &instructions[i];
6496 struct instruction_data *data = &instruction_data[i];
6497 uint32_t n_instr = 0;
6501 detected = instr_pattern_dma_many_search(instr,
6506 instr_pattern_dma_many_replace(instr, data, n_instr);
6511 /* No pattern starting at the current instruction. */
6515 /* Eliminate the invalid instructions that have been optimized out. */
6516 n_instructions = instr_compact(instructions,
6520 return n_instructions;
6524 instr_optimize(struct rte_swx_pipeline *p,
6526 struct instruction *instructions,
6527 struct instruction_data *instruction_data,
6528 uint32_t n_instructions)
6531 n_instructions = instr_pattern_extract_many_optimize(instructions,
6535 /* Emit many + TX. */
6536 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6540 /* Mov all + validate. */
6541 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6548 n_instructions = instr_pattern_dma_many_optimize(instructions,
6552 return n_instructions;
6556 instruction_config(struct rte_swx_pipeline *p,
6558 const char **instructions,
6559 uint32_t n_instructions)
6561 struct instruction *instr = NULL;
6562 struct instruction_data *data = NULL;
6566 CHECK(n_instructions, EINVAL);
6567 CHECK(instructions, EINVAL);
6568 for (i = 0; i < n_instructions; i++)
6569 CHECK_INSTRUCTION(instructions[i], EINVAL);
6571 /* Memory allocation. */
6572 instr = calloc(n_instructions, sizeof(struct instruction));
6578 data = calloc(n_instructions, sizeof(struct instruction_data));
6584 for (i = 0; i < n_instructions; i++) {
6585 char *string = strdup(instructions[i]);
6591 err = instr_translate(p, a, string, &instr[i], &data[i]);
6600 err = instr_label_check(data, n_instructions);
6604 err = instr_verify(p, a, instr, data, n_instructions);
6608 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6610 err = instr_jmp_resolve(instr, data, n_instructions);
6615 a->instructions = instr;
6616 a->instruction_data = data;
6617 a->n_instructions = n_instructions;
6619 p->instructions = instr;
6620 p->instruction_data = data;
6621 p->n_instructions = n_instructions;
6632 static instr_exec_t instruction_table[] = {
6633 [INSTR_RX] = instr_rx_exec,
6634 [INSTR_TX] = instr_tx_exec,
6635 [INSTR_TX_I] = instr_tx_i_exec,
6637 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6638 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6639 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6640 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6641 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6642 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6643 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6644 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6645 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6646 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6648 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6649 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6650 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6651 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6652 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6653 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6654 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6655 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6656 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6658 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6659 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6661 [INSTR_MOV] = instr_mov_exec,
6662 [INSTR_MOV_MH] = instr_mov_mh_exec,
6663 [INSTR_MOV_HM] = instr_mov_hm_exec,
6664 [INSTR_MOV_HH] = instr_mov_hh_exec,
6665 [INSTR_MOV_I] = instr_mov_i_exec,
6667 [INSTR_DMA_HT] = instr_dma_ht_exec,
6668 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6669 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6670 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6671 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6672 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6673 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6674 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6676 [INSTR_ALU_ADD] = instr_alu_add_exec,
6677 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6678 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6679 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6680 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6681 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6683 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6684 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6685 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6686 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6687 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6688 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6690 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6691 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6692 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6693 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6695 [INSTR_ALU_AND] = instr_alu_and_exec,
6696 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6697 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6698 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6699 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6701 [INSTR_ALU_OR] = instr_alu_or_exec,
6702 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6703 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6704 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6705 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6707 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6708 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6709 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6710 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6711 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6713 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6714 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6715 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6716 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6717 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6718 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6720 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6721 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6722 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6723 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6724 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6725 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6727 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6728 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6729 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6731 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6732 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6733 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6734 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6735 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6736 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6738 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6739 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6740 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6741 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6742 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6743 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6744 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6745 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6746 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6748 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6749 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6750 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6751 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6752 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6753 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6754 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6755 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6756 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6758 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6759 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6760 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6762 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6763 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6764 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6765 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6766 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6767 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6768 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6769 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6770 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6771 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6772 [INSTR_METER_IMM] = instr_meter_imm_exec,
6773 [INSTR_METER_IMI] = instr_meter_imi_exec,
6775 [INSTR_TABLE] = instr_table_exec,
6776 [INSTR_TABLE_AF] = instr_table_af_exec,
6777 [INSTR_SELECTOR] = instr_selector_exec,
6778 [INSTR_LEARNER] = instr_learner_exec,
6779 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6780 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6781 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6782 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6783 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6785 [INSTR_JMP] = instr_jmp_exec,
6786 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6787 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6788 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6789 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6790 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6791 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6793 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6794 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6795 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6796 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6797 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6799 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6800 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6801 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6802 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6803 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6805 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6806 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6807 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6808 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6809 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6810 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6812 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6813 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6814 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6815 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6816 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6817 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6819 [INSTR_RETURN] = instr_return_exec,
6823 instruction_table_build(struct rte_swx_pipeline *p)
6825 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6826 sizeof(struct instr_exec_t *));
6827 if (!p->instruction_table)
6830 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6836 instruction_table_build_free(struct rte_swx_pipeline *p)
6838 if (!p->instruction_table)
6841 free(p->instruction_table);
6842 p->instruction_table = NULL;
6846 instruction_table_free(struct rte_swx_pipeline *p)
6848 instruction_table_build_free(p);
6852 instr_exec(struct rte_swx_pipeline *p)
6854 struct thread *t = &p->threads[p->thread_id];
6855 struct instruction *ip = t->ip;
6856 instr_exec_t instr = p->instruction_table[ip->type];
6864 static struct action *
6865 action_find(struct rte_swx_pipeline *p, const char *name)
6867 struct action *elem;
6872 TAILQ_FOREACH(elem, &p->actions, node)
6873 if (strcmp(elem->name, name) == 0)
6879 static struct action *
6880 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6882 struct action *action = NULL;
6884 TAILQ_FOREACH(action, &p->actions, node)
6885 if (action->id == id)
6891 static struct field *
6892 action_field_find(struct action *a, const char *name)
6894 return a->st ? struct_type_field_find(a->st, name) : NULL;
6897 static struct field *
6898 action_field_parse(struct action *action, const char *name)
6900 if (name[0] != 't' || name[1] != '.')
6903 return action_field_find(action, &name[2]);
6907 action_has_nbo_args(struct action *a)
6911 /* Return if the action does not have any args. */
6913 return 0; /* FALSE */
6915 for (i = 0; i < a->st->n_fields; i++)
6916 if (a->args_endianness[i])
6917 return 1; /* TRUE */
6919 return 0; /* FALSE */
6923 action_does_learning(struct action *a)
6927 for (i = 0; i < a->n_instructions; i++)
6928 switch (a->instructions[i].type) {
6929 case INSTR_LEARNER_LEARN:
6930 return 1; /* TRUE */
6932 case INSTR_LEARNER_FORGET:
6933 return 1; /* TRUE */
6939 return 0; /* FALSE */
6943 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6945 const char *args_struct_type_name,
6946 const char **instructions,
6947 uint32_t n_instructions)
6949 struct struct_type *args_struct_type = NULL;
6955 CHECK_NAME(name, EINVAL);
6956 CHECK(!action_find(p, name), EEXIST);
6958 if (args_struct_type_name) {
6959 CHECK_NAME(args_struct_type_name, EINVAL);
6960 args_struct_type = struct_type_find(p, args_struct_type_name);
6961 CHECK(args_struct_type, EINVAL);
6962 CHECK(!args_struct_type->var_size, EINVAL);
6965 /* Node allocation. */
6966 a = calloc(1, sizeof(struct action));
6968 if (args_struct_type) {
6969 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6970 if (!a->args_endianness) {
6976 /* Node initialization. */
6977 strcpy(a->name, name);
6978 a->st = args_struct_type;
6979 a->id = p->n_actions;
6981 /* Instruction translation. */
6982 err = instruction_config(p, a, instructions, n_instructions);
6984 free(a->args_endianness);
6989 /* Node add to tailq. */
6990 TAILQ_INSERT_TAIL(&p->actions, a, node);
6997 action_build(struct rte_swx_pipeline *p)
6999 struct action *action;
7001 /* p->action_instructions. */
7002 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7003 CHECK(p->action_instructions, ENOMEM);
7005 TAILQ_FOREACH(action, &p->actions, node)
7006 p->action_instructions[action->id] = action->instructions;
7008 /* p->action_funcs. */
7009 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7010 CHECK(p->action_funcs, ENOMEM);
7016 action_build_free(struct rte_swx_pipeline *p)
7018 free(p->action_funcs);
7019 p->action_funcs = NULL;
7021 free(p->action_instructions);
7022 p->action_instructions = NULL;
7026 action_free(struct rte_swx_pipeline *p)
7028 action_build_free(p);
7031 struct action *action;
7033 action = TAILQ_FIRST(&p->actions);
7037 TAILQ_REMOVE(&p->actions, action, node);
7038 free(action->instruction_data);
7039 free(action->instructions);
7045 action_arg_src_mov_count(struct action *a,
7047 struct instruction *instructions,
7048 struct instruction_data *instruction_data,
7049 uint32_t n_instructions)
7051 uint32_t offset, n_users = 0, i;
7054 (arg_id >= a->st->n_fields) ||
7056 !instruction_data ||
7060 offset = a->st->fields[arg_id].offset / 8;
7062 for (i = 0; i < n_instructions; i++) {
7063 struct instruction *instr = &instructions[i];
7064 struct instruction_data *data = &instruction_data[i];
7066 if (data->invalid ||
7067 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7068 instr->mov.src.struct_id ||
7069 (instr->mov.src.offset != offset))
7081 static struct table_type *
7082 table_type_find(struct rte_swx_pipeline *p, const char *name)
7084 struct table_type *elem;
7086 TAILQ_FOREACH(elem, &p->table_types, node)
7087 if (strcmp(elem->name, name) == 0)
7093 static struct table_type *
7094 table_type_resolve(struct rte_swx_pipeline *p,
7095 const char *recommended_type_name,
7096 enum rte_swx_table_match_type match_type)
7098 struct table_type *elem;
7100 /* Only consider the recommended type if the match type is correct. */
7101 if (recommended_type_name)
7102 TAILQ_FOREACH(elem, &p->table_types, node)
7103 if (!strcmp(elem->name, recommended_type_name) &&
7104 (elem->match_type == match_type))
7107 /* Ignore the recommended type and get the first element with this match
7110 TAILQ_FOREACH(elem, &p->table_types, node)
7111 if (elem->match_type == match_type)
7117 static struct table *
7118 table_find(struct rte_swx_pipeline *p, const char *name)
7122 TAILQ_FOREACH(elem, &p->tables, node)
7123 if (strcmp(elem->name, name) == 0)
7129 static struct table *
7130 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7132 struct table *table = NULL;
7134 TAILQ_FOREACH(table, &p->tables, node)
7135 if (table->id == id)
7142 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7144 enum rte_swx_table_match_type match_type,
7145 struct rte_swx_table_ops *ops)
7147 struct table_type *elem;
7151 CHECK_NAME(name, EINVAL);
7152 CHECK(!table_type_find(p, name), EEXIST);
7155 CHECK(ops->create, EINVAL);
7156 CHECK(ops->lkp, EINVAL);
7157 CHECK(ops->free, EINVAL);
7159 /* Node allocation. */
7160 elem = calloc(1, sizeof(struct table_type));
7161 CHECK(elem, ENOMEM);
7163 /* Node initialization. */
7164 strcpy(elem->name, name);
7165 elem->match_type = match_type;
7166 memcpy(&elem->ops, ops, sizeof(*ops));
7168 /* Node add to tailq. */
7169 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7175 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7177 enum rte_swx_table_match_type *match_type)
7179 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7181 for (i = 0; i < n_fields; i++) {
7182 struct rte_swx_match_field_params *f = &fields[i];
7184 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7187 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7191 if ((n_fields_lpm > 1) ||
7192 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7195 *match_type = (n_fields_em == n_fields) ?
7196 RTE_SWX_TABLE_MATCH_EXACT :
7197 RTE_SWX_TABLE_MATCH_WILDCARD;
7203 table_match_fields_check(struct rte_swx_pipeline *p,
7204 struct rte_swx_pipeline_table_params *params,
7205 struct header **header)
7207 struct header *h0 = NULL;
7208 struct field *hf, *mf;
7209 uint32_t *offset = NULL, i;
7212 /* Return if no match fields. */
7213 if (!params->n_fields) {
7214 if (params->fields) {
7225 /* Memory allocation. */
7226 offset = calloc(params->n_fields, sizeof(uint32_t));
7232 /* Check that all the match fields belong to either the same header or
7235 hf = header_field_parse(p, params->fields[0].name, &h0);
7236 mf = metadata_field_parse(p, params->fields[0].name);
7237 if ((!hf && !mf) || (hf && hf->var_size)) {
7242 offset[0] = h0 ? hf->offset : mf->offset;
7244 for (i = 1; i < params->n_fields; i++)
7248 hf = header_field_parse(p, params->fields[i].name, &h);
7249 if (!hf || (h->id != h0->id) || hf->var_size) {
7254 offset[i] = hf->offset;
7256 mf = metadata_field_parse(p, params->fields[i].name);
7262 offset[i] = mf->offset;
7265 /* Check that there are no duplicated match fields. */
7266 for (i = 0; i < params->n_fields; i++) {
7269 for (j = 0; j < i; j++)
7270 if (offset[j] == offset[i]) {
7286 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7288 struct rte_swx_pipeline_table_params *params,
7289 const char *recommended_table_type_name,
7293 struct table_type *type;
7295 struct action *default_action;
7296 struct header *header = NULL;
7297 uint32_t action_data_size_max = 0, i;
7302 CHECK_NAME(name, EINVAL);
7303 CHECK(!table_find(p, name), EEXIST);
7304 CHECK(!selector_find(p, name), EEXIST);
7305 CHECK(!learner_find(p, name), EEXIST);
7307 CHECK(params, EINVAL);
7310 status = table_match_fields_check(p, params, &header);
7314 /* Action checks. */
7315 CHECK(params->n_actions, EINVAL);
7316 CHECK(params->action_names, EINVAL);
7317 for (i = 0; i < params->n_actions; i++) {
7318 const char *action_name = params->action_names[i];
7320 uint32_t action_data_size;
7322 CHECK_NAME(action_name, EINVAL);
7324 a = action_find(p, action_name);
7326 CHECK(!action_does_learning(a), EINVAL);
7328 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7329 if (action_data_size > action_data_size_max)
7330 action_data_size_max = action_data_size;
7333 CHECK_NAME(params->default_action_name, EINVAL);
7334 for (i = 0; i < p->n_actions; i++)
7335 if (!strcmp(params->action_names[i],
7336 params->default_action_name))
7338 CHECK(i < params->n_actions, EINVAL);
7339 default_action = action_find(p, params->default_action_name);
7340 CHECK((default_action->st && params->default_action_data) ||
7341 !params->default_action_data, EINVAL);
7343 /* Table type checks. */
7344 if (recommended_table_type_name)
7345 CHECK_NAME(recommended_table_type_name, EINVAL);
7347 if (params->n_fields) {
7348 enum rte_swx_table_match_type match_type;
7350 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7354 type = table_type_resolve(p, recommended_table_type_name, match_type);
7355 CHECK(type, EINVAL);
7360 /* Memory allocation. */
7361 t = calloc(1, sizeof(struct table));
7364 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7370 t->actions = calloc(params->n_actions, sizeof(struct action *));
7377 if (action_data_size_max) {
7378 t->default_action_data = calloc(1, action_data_size_max);
7379 if (!t->default_action_data) {
7387 /* Node initialization. */
7388 strcpy(t->name, name);
7389 if (args && args[0])
7390 strcpy(t->args, args);
7393 for (i = 0; i < params->n_fields; i++) {
7394 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7395 struct match_field *f = &t->fields[i];
7397 f->match_type = field->match_type;
7399 header_field_parse(p, field->name, NULL) :
7400 metadata_field_parse(p, field->name);
7402 t->n_fields = params->n_fields;
7405 for (i = 0; i < params->n_actions; i++)
7406 t->actions[i] = action_find(p, params->action_names[i]);
7407 t->default_action = default_action;
7408 if (default_action->st)
7409 memcpy(t->default_action_data,
7410 params->default_action_data,
7411 default_action->st->n_bits / 8);
7412 t->n_actions = params->n_actions;
7413 t->default_action_is_const = params->default_action_is_const;
7414 t->action_data_size_max = action_data_size_max;
7417 t->id = p->n_tables;
7419 /* Node add to tailq. */
7420 TAILQ_INSERT_TAIL(&p->tables, t, node);
7426 static struct rte_swx_table_params *
7427 table_params_get(struct table *table)
7429 struct rte_swx_table_params *params;
7430 struct field *first, *last;
7432 uint32_t key_size, key_offset, action_data_size, i;
7434 /* Memory allocation. */
7435 params = calloc(1, sizeof(struct rte_swx_table_params));
7439 /* Find first (smallest offset) and last (biggest offset) match fields. */
7440 first = table->fields[0].field;
7441 last = table->fields[0].field;
7443 for (i = 0; i < table->n_fields; i++) {
7444 struct field *f = table->fields[i].field;
7446 if (f->offset < first->offset)
7449 if (f->offset > last->offset)
7453 /* Key offset and size. */
7454 key_offset = first->offset / 8;
7455 key_size = (last->offset + last->n_bits - first->offset) / 8;
7457 /* Memory allocation. */
7458 key_mask = calloc(1, key_size);
7465 for (i = 0; i < table->n_fields; i++) {
7466 struct field *f = table->fields[i].field;
7467 uint32_t start = (f->offset - first->offset) / 8;
7468 size_t size = f->n_bits / 8;
7470 memset(&key_mask[start], 0xFF, size);
7473 /* Action data size. */
7474 action_data_size = 0;
7475 for (i = 0; i < table->n_actions; i++) {
7476 struct action *action = table->actions[i];
7477 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7479 if (ads > action_data_size)
7480 action_data_size = ads;
7484 params->match_type = table->type->match_type;
7485 params->key_size = key_size;
7486 params->key_offset = key_offset;
7487 params->key_mask0 = key_mask;
7488 params->action_data_size = action_data_size;
7489 params->n_keys_max = table->size;
7495 table_params_free(struct rte_swx_table_params *params)
7500 free(params->key_mask0);
7505 table_stub_lkp(void *table __rte_unused,
7506 void *mailbox __rte_unused,
7507 uint8_t **key __rte_unused,
7508 uint64_t *action_id __rte_unused,
7509 uint8_t **action_data __rte_unused,
7513 return 1; /* DONE. */
7517 table_build(struct rte_swx_pipeline *p)
7521 /* Per pipeline: table statistics. */
7522 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7523 CHECK(p->table_stats, ENOMEM);
7525 for (i = 0; i < p->n_tables; i++) {
7526 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7527 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7530 /* Per thread: table runt-time. */
7531 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7532 struct thread *t = &p->threads[i];
7533 struct table *table;
7535 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7536 CHECK(t->tables, ENOMEM);
7538 TAILQ_FOREACH(table, &p->tables, node) {
7539 struct table_runtime *r = &t->tables[table->id];
7544 size = table->type->ops.mailbox_size_get();
7547 r->func = table->type->ops.lkp;
7551 r->mailbox = calloc(1, size);
7552 CHECK(r->mailbox, ENOMEM);
7556 r->key = table->header ?
7557 &t->structs[table->header->struct_id] :
7558 &t->structs[p->metadata_struct_id];
7560 r->func = table_stub_lkp;
7569 table_build_free(struct rte_swx_pipeline *p)
7573 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7574 struct thread *t = &p->threads[i];
7580 for (j = 0; j < p->n_tables; j++) {
7581 struct table_runtime *r = &t->tables[j];
7590 if (p->table_stats) {
7591 for (i = 0; i < p->n_tables; i++)
7592 free(p->table_stats[i].n_pkts_action);
7594 free(p->table_stats);
7599 table_free(struct rte_swx_pipeline *p)
7601 table_build_free(p);
7607 elem = TAILQ_FIRST(&p->tables);
7611 TAILQ_REMOVE(&p->tables, elem, node);
7613 free(elem->actions);
7614 free(elem->default_action_data);
7620 struct table_type *elem;
7622 elem = TAILQ_FIRST(&p->table_types);
7626 TAILQ_REMOVE(&p->table_types, elem, node);
7634 static struct selector *
7635 selector_find(struct rte_swx_pipeline *p, const char *name)
7639 TAILQ_FOREACH(s, &p->selectors, node)
7640 if (strcmp(s->name, name) == 0)
7646 static struct selector *
7647 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7649 struct selector *s = NULL;
7651 TAILQ_FOREACH(s, &p->selectors, node)
7659 selector_fields_check(struct rte_swx_pipeline *p,
7660 struct rte_swx_pipeline_selector_params *params,
7661 struct header **header)
7663 struct header *h0 = NULL;
7664 struct field *hf, *mf;
7667 /* Return if no selector fields. */
7668 if (!params->n_selector_fields || !params->selector_field_names)
7671 /* Check that all the selector fields either belong to the same header
7672 * or are all meta-data fields.
7674 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7675 mf = metadata_field_parse(p, params->selector_field_names[0]);
7679 for (i = 1; i < params->n_selector_fields; i++)
7683 hf = header_field_parse(p, params->selector_field_names[i], &h);
7684 if (!hf || (h->id != h0->id))
7687 mf = metadata_field_parse(p, params->selector_field_names[i]);
7692 /* Check that there are no duplicated match fields. */
7693 for (i = 0; i < params->n_selector_fields; i++) {
7694 const char *field_name = params->selector_field_names[i];
7697 for (j = i + 1; j < params->n_selector_fields; j++)
7698 if (!strcmp(params->selector_field_names[j], field_name))
7710 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7712 struct rte_swx_pipeline_selector_params *params)
7715 struct header *selector_header = NULL;
7716 struct field *group_id_field, *member_id_field;
7722 CHECK_NAME(name, EINVAL);
7723 CHECK(!table_find(p, name), EEXIST);
7724 CHECK(!selector_find(p, name), EEXIST);
7725 CHECK(!learner_find(p, name), EEXIST);
7727 CHECK(params, EINVAL);
7729 CHECK_NAME(params->group_id_field_name, EINVAL);
7730 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7731 CHECK(group_id_field, EINVAL);
7733 for (i = 0; i < params->n_selector_fields; i++) {
7734 const char *field_name = params->selector_field_names[i];
7736 CHECK_NAME(field_name, EINVAL);
7738 status = selector_fields_check(p, params, &selector_header);
7742 CHECK_NAME(params->member_id_field_name, EINVAL);
7743 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7744 CHECK(member_id_field, EINVAL);
7746 CHECK(params->n_groups_max, EINVAL);
7748 CHECK(params->n_members_per_group_max, EINVAL);
7750 /* Memory allocation. */
7751 s = calloc(1, sizeof(struct selector));
7757 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7758 if (!s->selector_fields) {
7763 /* Node initialization. */
7764 strcpy(s->name, name);
7766 s->group_id_field = group_id_field;
7768 for (i = 0; i < params->n_selector_fields; i++) {
7769 const char *field_name = params->selector_field_names[i];
7771 s->selector_fields[i] = selector_header ?
7772 header_field_parse(p, field_name, NULL) :
7773 metadata_field_parse(p, field_name);
7776 s->n_selector_fields = params->n_selector_fields;
7778 s->selector_header = selector_header;
7780 s->member_id_field = member_id_field;
7782 s->n_groups_max = params->n_groups_max;
7784 s->n_members_per_group_max = params->n_members_per_group_max;
7786 s->id = p->n_selectors;
7788 /* Node add to tailq. */
7789 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7798 free(s->selector_fields);
7806 selector_params_free(struct rte_swx_table_selector_params *params)
7811 free(params->selector_mask);
7816 static struct rte_swx_table_selector_params *
7817 selector_table_params_get(struct selector *s)
7819 struct rte_swx_table_selector_params *params = NULL;
7820 struct field *first, *last;
7823 /* Memory allocation. */
7824 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7829 params->group_id_offset = s->group_id_field->offset / 8;
7831 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7832 first = s->selector_fields[0];
7833 last = s->selector_fields[0];
7835 for (i = 0; i < s->n_selector_fields; i++) {
7836 struct field *f = s->selector_fields[i];
7838 if (f->offset < first->offset)
7841 if (f->offset > last->offset)
7845 /* Selector offset and size. */
7846 params->selector_offset = first->offset / 8;
7847 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7849 /* Memory allocation. */
7850 params->selector_mask = calloc(1, params->selector_size);
7851 if (!params->selector_mask)
7854 /* Selector mask. */
7855 for (i = 0; i < s->n_selector_fields; i++) {
7856 struct field *f = s->selector_fields[i];
7857 uint32_t start = (f->offset - first->offset) / 8;
7858 size_t size = f->n_bits / 8;
7860 memset(¶ms->selector_mask[start], 0xFF, size);
7864 params->member_id_offset = s->member_id_field->offset / 8;
7866 /* Maximum number of groups. */
7867 params->n_groups_max = s->n_groups_max;
7869 /* Maximum number of members per group. */
7870 params->n_members_per_group_max = s->n_members_per_group_max;
7875 selector_params_free(params);
7880 selector_build_free(struct rte_swx_pipeline *p)
7884 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7885 struct thread *t = &p->threads[i];
7891 for (j = 0; j < p->n_selectors; j++) {
7892 struct selector_runtime *r = &t->selectors[j];
7898 t->selectors = NULL;
7901 free(p->selector_stats);
7902 p->selector_stats = NULL;
7906 selector_build(struct rte_swx_pipeline *p)
7911 /* Per pipeline: selector statistics. */
7912 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7913 if (!p->selector_stats) {
7918 /* Per thread: selector run-time. */
7919 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7920 struct thread *t = &p->threads[i];
7923 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7924 if (!t->selectors) {
7929 TAILQ_FOREACH(s, &p->selectors, node) {
7930 struct selector_runtime *r = &t->selectors[s->id];
7934 size = rte_swx_table_selector_mailbox_size_get();
7936 r->mailbox = calloc(1, size);
7943 /* r->group_id_buffer. */
7944 r->group_id_buffer = &t->structs[p->metadata_struct_id];
7946 /* r->selector_buffer. */
7947 r->selector_buffer = s->selector_header ?
7948 &t->structs[s->selector_header->struct_id] :
7949 &t->structs[p->metadata_struct_id];
7951 /* r->member_id_buffer. */
7952 r->member_id_buffer = &t->structs[p->metadata_struct_id];
7959 selector_build_free(p);
7964 selector_free(struct rte_swx_pipeline *p)
7966 selector_build_free(p);
7968 /* Selector tables. */
7970 struct selector *elem;
7972 elem = TAILQ_FIRST(&p->selectors);
7976 TAILQ_REMOVE(&p->selectors, elem, node);
7977 free(elem->selector_fields);
7985 static struct learner *
7986 learner_find(struct rte_swx_pipeline *p, const char *name)
7990 TAILQ_FOREACH(l, &p->learners, node)
7991 if (!strcmp(l->name, name))
7997 static struct learner *
7998 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8000 struct learner *l = NULL;
8002 TAILQ_FOREACH(l, &p->learners, node)
8010 learner_match_fields_check(struct rte_swx_pipeline *p,
8011 struct rte_swx_pipeline_learner_params *params,
8012 struct header **header)
8014 struct header *h0 = NULL;
8015 struct field *hf, *mf;
8018 /* Return if no match fields. */
8019 if (!params->n_fields || !params->field_names)
8022 /* Check that all the match fields either belong to the same header
8023 * or are all meta-data fields.
8025 hf = header_field_parse(p, params->field_names[0], &h0);
8026 mf = metadata_field_parse(p, params->field_names[0]);
8030 for (i = 1; i < params->n_fields; i++)
8034 hf = header_field_parse(p, params->field_names[i], &h);
8035 if (!hf || (h->id != h0->id))
8038 mf = metadata_field_parse(p, params->field_names[i]);
8043 /* Check that there are no duplicated match fields. */
8044 for (i = 0; i < params->n_fields; i++) {
8045 const char *field_name = params->field_names[i];
8048 for (j = i + 1; j < params->n_fields; j++)
8049 if (!strcmp(params->field_names[j], field_name))
8061 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8063 struct struct_type *mst = p->metadata_st, *ast = a->st;
8064 struct field *mf, *af;
8074 /* Check that mf_name is the name of a valid meta-data field. */
8075 CHECK_NAME(mf_name, EINVAL);
8076 mf = metadata_field_parse(p, mf_name);
8079 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8080 * all the action arguments.
8082 mf_pos = mf - mst->fields;
8083 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8085 /* Check that the size of each of the identified meta-data fields matches exactly the size
8086 * of the corresponding action argument.
8088 for (i = 0; i < ast->n_fields; i++) {
8089 mf = &mst->fields[mf_pos + i];
8090 af = &ast->fields[i];
8092 CHECK(mf->n_bits == af->n_bits, EINVAL);
8099 learner_action_learning_check(struct rte_swx_pipeline *p,
8100 struct action *action,
8101 const char **action_names,
8106 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8107 * the action passed as argument to the "learn" instruction) is also enabled for the
8108 * current learner table.
8110 for (i = 0; i < action->n_instructions; i++) {
8111 struct instruction *instr = &action->instructions[i];
8112 uint32_t found = 0, j;
8114 if (instr->type != INSTR_LEARNER_LEARN)
8117 for (j = 0; j < n_actions; j++) {
8120 a = action_find(p, action_names[j]);
8124 if (a->id == instr->learn.action_id)
8136 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8138 struct rte_swx_pipeline_learner_params *params,
8142 struct learner *l = NULL;
8143 struct action *default_action;
8144 struct header *header = NULL;
8145 uint32_t action_data_size_max = 0, i;
8150 CHECK_NAME(name, EINVAL);
8151 CHECK(!table_find(p, name), EEXIST);
8152 CHECK(!selector_find(p, name), EEXIST);
8153 CHECK(!learner_find(p, name), EEXIST);
8155 CHECK(params, EINVAL);
8158 status = learner_match_fields_check(p, params, &header);
8162 /* Action checks. */
8163 CHECK(params->n_actions, EINVAL);
8165 CHECK(params->action_names, EINVAL);
8166 for (i = 0; i < params->n_actions; i++) {
8167 const char *action_name = params->action_names[i];
8168 const char *action_field_name = params->action_field_names[i];
8170 uint32_t action_data_size;
8172 CHECK_NAME(action_name, EINVAL);
8174 a = action_find(p, action_name);
8177 status = learner_action_args_check(p, a, action_field_name);
8181 status = learner_action_learning_check(p,
8183 params->action_names,
8188 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8189 if (action_data_size > action_data_size_max)
8190 action_data_size_max = action_data_size;
8193 CHECK_NAME(params->default_action_name, EINVAL);
8194 for (i = 0; i < p->n_actions; i++)
8195 if (!strcmp(params->action_names[i],
8196 params->default_action_name))
8198 CHECK(i < params->n_actions, EINVAL);
8200 default_action = action_find(p, params->default_action_name);
8201 CHECK((default_action->st && params->default_action_data) ||
8202 !params->default_action_data, EINVAL);
8204 /* Any other checks. */
8205 CHECK(size, EINVAL);
8206 CHECK(timeout, EINVAL);
8208 /* Memory allocation. */
8209 l = calloc(1, sizeof(struct learner));
8213 l->fields = calloc(params->n_fields, sizeof(struct field *));
8217 l->actions = calloc(params->n_actions, sizeof(struct action *));
8221 l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8225 if (action_data_size_max) {
8226 l->default_action_data = calloc(1, action_data_size_max);
8227 if (!l->default_action_data)
8231 /* Node initialization. */
8232 strcpy(l->name, name);
8234 for (i = 0; i < params->n_fields; i++) {
8235 const char *field_name = params->field_names[i];
8237 l->fields[i] = header ?
8238 header_field_parse(p, field_name, NULL) :
8239 metadata_field_parse(p, field_name);
8242 l->n_fields = params->n_fields;
8246 for (i = 0; i < params->n_actions; i++) {
8247 const char *mf_name = params->action_field_names[i];
8249 l->actions[i] = action_find(p, params->action_names[i]);
8251 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8254 l->default_action = default_action;
8256 if (default_action->st)
8257 memcpy(l->default_action_data,
8258 params->default_action_data,
8259 default_action->st->n_bits / 8);
8261 l->n_actions = params->n_actions;
8263 l->default_action_is_const = params->default_action_is_const;
8265 l->action_data_size_max = action_data_size_max;
8269 l->timeout = timeout;
8271 l->id = p->n_learners;
8273 /* Node add to tailq. */
8274 TAILQ_INSERT_TAIL(&p->learners, l, node);
8283 free(l->action_arg);
8292 learner_params_free(struct rte_swx_table_learner_params *params)
8297 free(params->key_mask0);
8302 static struct rte_swx_table_learner_params *
8303 learner_params_get(struct learner *l)
8305 struct rte_swx_table_learner_params *params = NULL;
8306 struct field *first, *last;
8309 /* Memory allocation. */
8310 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8314 /* Find first (smallest offset) and last (biggest offset) match fields. */
8315 first = l->fields[0];
8316 last = l->fields[0];
8318 for (i = 0; i < l->n_fields; i++) {
8319 struct field *f = l->fields[i];
8321 if (f->offset < first->offset)
8324 if (f->offset > last->offset)
8328 /* Key offset and size. */
8329 params->key_offset = first->offset / 8;
8330 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8332 /* Memory allocation. */
8333 params->key_mask0 = calloc(1, params->key_size);
8334 if (!params->key_mask0)
8338 for (i = 0; i < l->n_fields; i++) {
8339 struct field *f = l->fields[i];
8340 uint32_t start = (f->offset - first->offset) / 8;
8341 size_t size = f->n_bits / 8;
8343 memset(¶ms->key_mask0[start], 0xFF, size);
8346 /* Action data size. */
8347 params->action_data_size = l->action_data_size_max;
8349 /* Maximum number of keys. */
8350 params->n_keys_max = l->size;
8353 params->key_timeout = l->timeout;
8358 learner_params_free(params);
8363 learner_build_free(struct rte_swx_pipeline *p)
8367 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8368 struct thread *t = &p->threads[i];
8374 for (j = 0; j < p->n_learners; j++) {
8375 struct learner_runtime *r = &t->learners[j];
8378 free(r->action_data);
8385 if (p->learner_stats) {
8386 for (i = 0; i < p->n_learners; i++)
8387 free(p->learner_stats[i].n_pkts_action);
8389 free(p->learner_stats);
8394 learner_build(struct rte_swx_pipeline *p)
8399 /* Per pipeline: learner statistics. */
8400 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8401 CHECK(p->learner_stats, ENOMEM);
8403 for (i = 0; i < p->n_learners; i++) {
8404 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8405 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8408 /* Per thread: learner run-time. */
8409 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8410 struct thread *t = &p->threads[i];
8413 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8419 TAILQ_FOREACH(l, &p->learners, node) {
8420 struct learner_runtime *r = &t->learners[l->id];
8425 size = rte_swx_table_learner_mailbox_size_get();
8427 r->mailbox = calloc(1, size);
8435 r->key = l->header ?
8436 &t->structs[l->header->struct_id] :
8437 &t->structs[p->metadata_struct_id];
8439 /* r->action_data. */
8440 r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8441 if (!r->action_data) {
8446 for (j = 0; j < l->n_actions; j++) {
8447 struct action *a = l->actions[j];
8448 struct field *mf = l->action_arg[j];
8449 uint8_t *m = t->structs[p->metadata_struct_id];
8451 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8459 learner_build_free(p);
8464 learner_free(struct rte_swx_pipeline *p)
8466 learner_build_free(p);
8468 /* Learner tables. */
8472 l = TAILQ_FIRST(&p->learners);
8476 TAILQ_REMOVE(&p->learners, l, node);
8479 free(l->action_arg);
8480 free(l->default_action_data);
8489 table_state_build(struct rte_swx_pipeline *p)
8491 struct table *table;
8495 p->table_state = calloc(p->n_tables + p->n_selectors,
8496 sizeof(struct rte_swx_table_state));
8497 CHECK(p->table_state, ENOMEM);
8499 TAILQ_FOREACH(table, &p->tables, node) {
8500 struct rte_swx_table_state *ts = &p->table_state[table->id];
8503 struct rte_swx_table_params *params;
8506 params = table_params_get(table);
8507 CHECK(params, ENOMEM);
8509 ts->obj = table->type->ops.create(params,
8514 table_params_free(params);
8515 CHECK(ts->obj, ENODEV);
8518 /* ts->default_action_data. */
8519 if (table->action_data_size_max) {
8520 ts->default_action_data =
8521 malloc(table->action_data_size_max);
8522 CHECK(ts->default_action_data, ENOMEM);
8524 memcpy(ts->default_action_data,
8525 table->default_action_data,
8526 table->action_data_size_max);
8529 /* ts->default_action_id. */
8530 ts->default_action_id = table->default_action->id;
8533 TAILQ_FOREACH(s, &p->selectors, node) {
8534 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8535 struct rte_swx_table_selector_params *params;
8538 params = selector_table_params_get(s);
8539 CHECK(params, ENOMEM);
8541 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8543 selector_params_free(params);
8544 CHECK(ts->obj, ENODEV);
8547 TAILQ_FOREACH(l, &p->learners, node) {
8548 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8549 p->n_selectors + l->id];
8550 struct rte_swx_table_learner_params *params;
8553 params = learner_params_get(l);
8554 CHECK(params, ENOMEM);
8556 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8557 learner_params_free(params);
8558 CHECK(ts->obj, ENODEV);
8560 /* ts->default_action_data. */
8561 if (l->action_data_size_max) {
8562 ts->default_action_data = malloc(l->action_data_size_max);
8563 CHECK(ts->default_action_data, ENOMEM);
8565 memcpy(ts->default_action_data,
8566 l->default_action_data,
8567 l->action_data_size_max);
8570 /* ts->default_action_id. */
8571 ts->default_action_id = l->default_action->id;
8578 table_state_build_free(struct rte_swx_pipeline *p)
8582 if (!p->table_state)
8585 for (i = 0; i < p->n_tables; i++) {
8586 struct rte_swx_table_state *ts = &p->table_state[i];
8587 struct table *table = table_find_by_id(p, i);
8590 if (table->type && ts->obj)
8591 table->type->ops.free(ts->obj);
8593 /* ts->default_action_data. */
8594 free(ts->default_action_data);
8597 for (i = 0; i < p->n_selectors; i++) {
8598 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8602 rte_swx_table_selector_free(ts->obj);
8605 for (i = 0; i < p->n_learners; i++) {
8606 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8610 rte_swx_table_learner_free(ts->obj);
8612 /* ts->default_action_data. */
8613 free(ts->default_action_data);
8616 free(p->table_state);
8617 p->table_state = NULL;
8621 table_state_free(struct rte_swx_pipeline *p)
8623 table_state_build_free(p);
8629 static struct regarray *
8630 regarray_find(struct rte_swx_pipeline *p, const char *name)
8632 struct regarray *elem;
8634 TAILQ_FOREACH(elem, &p->regarrays, node)
8635 if (!strcmp(elem->name, name))
8641 static struct regarray *
8642 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8644 struct regarray *elem = NULL;
8646 TAILQ_FOREACH(elem, &p->regarrays, node)
8654 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8663 CHECK_NAME(name, EINVAL);
8664 CHECK(!regarray_find(p, name), EEXIST);
8666 CHECK(size, EINVAL);
8667 size = rte_align32pow2(size);
8669 /* Memory allocation. */
8670 r = calloc(1, sizeof(struct regarray));
8673 /* Node initialization. */
8674 strcpy(r->name, name);
8675 r->init_val = init_val;
8677 r->id = p->n_regarrays;
8679 /* Node add to tailq. */
8680 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8687 regarray_build(struct rte_swx_pipeline *p)
8689 struct regarray *regarray;
8691 if (!p->n_regarrays)
8694 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8695 CHECK(p->regarray_runtime, ENOMEM);
8697 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8698 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8701 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8702 RTE_CACHE_LINE_SIZE,
8704 CHECK(r->regarray, ENOMEM);
8706 if (regarray->init_val)
8707 for (i = 0; i < regarray->size; i++)
8708 r->regarray[i] = regarray->init_val;
8710 r->size_mask = regarray->size - 1;
8717 regarray_build_free(struct rte_swx_pipeline *p)
8721 if (!p->regarray_runtime)
8724 for (i = 0; i < p->n_regarrays; i++) {
8725 struct regarray *regarray = regarray_find_by_id(p, i);
8726 struct regarray_runtime *r = &p->regarray_runtime[i];
8728 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8731 free(p->regarray_runtime);
8732 p->regarray_runtime = NULL;
8736 regarray_free(struct rte_swx_pipeline *p)
8738 regarray_build_free(p);
8741 struct regarray *elem;
8743 elem = TAILQ_FIRST(&p->regarrays);
8747 TAILQ_REMOVE(&p->regarrays, elem, node);
8755 static struct meter_profile *
8756 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8758 struct meter_profile *elem;
8760 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8761 if (!strcmp(elem->name, name))
8767 static struct metarray *
8768 metarray_find(struct rte_swx_pipeline *p, const char *name)
8770 struct metarray *elem;
8772 TAILQ_FOREACH(elem, &p->metarrays, node)
8773 if (!strcmp(elem->name, name))
8779 static struct metarray *
8780 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8782 struct metarray *elem = NULL;
8784 TAILQ_FOREACH(elem, &p->metarrays, node)
8792 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8800 CHECK_NAME(name, EINVAL);
8801 CHECK(!metarray_find(p, name), EEXIST);
8803 CHECK(size, EINVAL);
8804 size = rte_align32pow2(size);
8806 /* Memory allocation. */
8807 m = calloc(1, sizeof(struct metarray));
8810 /* Node initialization. */
8811 strcpy(m->name, name);
8813 m->id = p->n_metarrays;
8815 /* Node add to tailq. */
8816 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8822 struct meter_profile meter_profile_default = {
8831 .cir_bytes_per_period = 1,
8833 .pir_bytes_per_period = 1,
8840 meter_init(struct meter *m)
8842 memset(m, 0, sizeof(struct meter));
8843 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8844 m->profile = &meter_profile_default;
8845 m->color_mask = RTE_COLOR_GREEN;
8847 meter_profile_default.n_users++;
8851 metarray_build(struct rte_swx_pipeline *p)
8855 if (!p->n_metarrays)
8858 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8859 CHECK(p->metarray_runtime, ENOMEM);
8861 TAILQ_FOREACH(m, &p->metarrays, node) {
8862 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8865 r->metarray = env_malloc(m->size * sizeof(struct meter),
8866 RTE_CACHE_LINE_SIZE,
8868 CHECK(r->metarray, ENOMEM);
8870 for (i = 0; i < m->size; i++)
8871 meter_init(&r->metarray[i]);
8873 r->size_mask = m->size - 1;
8880 metarray_build_free(struct rte_swx_pipeline *p)
8884 if (!p->metarray_runtime)
8887 for (i = 0; i < p->n_metarrays; i++) {
8888 struct metarray *m = metarray_find_by_id(p, i);
8889 struct metarray_runtime *r = &p->metarray_runtime[i];
8891 env_free(r->metarray, m->size * sizeof(struct meter));
8894 free(p->metarray_runtime);
8895 p->metarray_runtime = NULL;
8899 metarray_free(struct rte_swx_pipeline *p)
8901 metarray_build_free(p);
8905 struct metarray *elem;
8907 elem = TAILQ_FIRST(&p->metarrays);
8911 TAILQ_REMOVE(&p->metarrays, elem, node);
8915 /* Meter profiles. */
8917 struct meter_profile *elem;
8919 elem = TAILQ_FIRST(&p->meter_profiles);
8923 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8932 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8934 struct rte_swx_pipeline *pipeline;
8936 /* Check input parameters. */
8939 /* Memory allocation. */
8940 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8941 CHECK(pipeline, ENOMEM);
8943 /* Initialization. */
8944 TAILQ_INIT(&pipeline->struct_types);
8945 TAILQ_INIT(&pipeline->port_in_types);
8946 TAILQ_INIT(&pipeline->ports_in);
8947 TAILQ_INIT(&pipeline->port_out_types);
8948 TAILQ_INIT(&pipeline->ports_out);
8949 TAILQ_INIT(&pipeline->extern_types);
8950 TAILQ_INIT(&pipeline->extern_objs);
8951 TAILQ_INIT(&pipeline->extern_funcs);
8952 TAILQ_INIT(&pipeline->headers);
8953 TAILQ_INIT(&pipeline->actions);
8954 TAILQ_INIT(&pipeline->table_types);
8955 TAILQ_INIT(&pipeline->tables);
8956 TAILQ_INIT(&pipeline->selectors);
8957 TAILQ_INIT(&pipeline->learners);
8958 TAILQ_INIT(&pipeline->regarrays);
8959 TAILQ_INIT(&pipeline->meter_profiles);
8960 TAILQ_INIT(&pipeline->metarrays);
8962 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8963 pipeline->numa_node = numa_node;
8970 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8979 free(p->instruction_data);
8980 free(p->instructions);
8984 table_state_free(p);
8989 instruction_table_free(p);
8992 extern_func_free(p);
9005 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9006 const char **instructions,
9007 uint32_t n_instructions)
9012 err = instruction_config(p, NULL, instructions, n_instructions);
9016 /* Thread instruction pointer reset. */
9017 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9018 struct thread *t = &p->threads[i];
9020 thread_ip_reset(p, t);
9027 pipeline_compile(struct rte_swx_pipeline *p);
9030 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9035 CHECK(p->build_done == 0, EEXIST);
9037 status = port_in_build(p);
9041 status = port_out_build(p);
9045 status = struct_build(p);
9049 status = extern_obj_build(p);
9053 status = extern_func_build(p);
9057 status = header_build(p);
9061 status = metadata_build(p);
9065 status = instruction_table_build(p);
9069 status = action_build(p);
9073 status = table_build(p);
9077 status = selector_build(p);
9081 status = learner_build(p);
9085 status = table_state_build(p);
9089 status = regarray_build(p);
9093 status = metarray_build(p);
9099 pipeline_compile(p);
9104 metarray_build_free(p);
9105 regarray_build_free(p);
9106 table_state_build_free(p);
9107 learner_build_free(p);
9108 selector_build_free(p);
9109 table_build_free(p);
9110 action_build_free(p);
9111 instruction_table_build_free(p);
9112 metadata_build_free(p);
9113 header_build_free(p);
9114 extern_func_build_free(p);
9115 extern_obj_build_free(p);
9116 port_out_build_free(p);
9117 port_in_build_free(p);
9118 struct_build_free(p);
9124 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9128 for (i = 0; i < n_instructions; i++)
9133 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9137 for (i = 0; i < p->n_ports_out; i++) {
9138 struct port_out_runtime *port = &p->out[i];
9141 port->flush(port->obj);
9149 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9150 struct rte_swx_ctl_pipeline_info *pipeline)
9152 struct action *action;
9153 struct table *table;
9154 uint32_t n_actions = 0, n_tables = 0;
9156 if (!p || !pipeline)
9159 TAILQ_FOREACH(action, &p->actions, node)
9162 TAILQ_FOREACH(table, &p->tables, node)
9165 pipeline->n_ports_in = p->n_ports_in;
9166 pipeline->n_ports_out = p->n_ports_out;
9167 pipeline->n_actions = n_actions;
9168 pipeline->n_tables = n_tables;
9169 pipeline->n_selectors = p->n_selectors;
9170 pipeline->n_learners = p->n_learners;
9171 pipeline->n_regarrays = p->n_regarrays;
9172 pipeline->n_metarrays = p->n_metarrays;
9178 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9180 if (!p || !numa_node)
9183 *numa_node = p->numa_node;
9188 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9190 struct rte_swx_ctl_action_info *action)
9192 struct action *a = NULL;
9194 if (!p || (action_id >= p->n_actions) || !action)
9197 a = action_find_by_id(p, action_id);
9201 strcpy(action->name, a->name);
9202 action->n_args = a->st ? a->st->n_fields : 0;
9207 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9209 uint32_t action_arg_id,
9210 struct rte_swx_ctl_action_arg_info *action_arg)
9212 struct action *a = NULL;
9213 struct field *arg = NULL;
9215 if (!p || (action_id >= p->n_actions) || !action_arg)
9218 a = action_find_by_id(p, action_id);
9219 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9222 arg = &a->st->fields[action_arg_id];
9223 strcpy(action_arg->name, arg->name);
9224 action_arg->n_bits = arg->n_bits;
9225 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9231 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9233 struct rte_swx_ctl_table_info *table)
9235 struct table *t = NULL;
9240 t = table_find_by_id(p, table_id);
9244 strcpy(table->name, t->name);
9245 strcpy(table->args, t->args);
9246 table->n_match_fields = t->n_fields;
9247 table->n_actions = t->n_actions;
9248 table->default_action_is_const = t->default_action_is_const;
9249 table->size = t->size;
9254 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9256 uint32_t match_field_id,
9257 struct rte_swx_ctl_table_match_field_info *match_field)
9260 struct match_field *f;
9262 if (!p || (table_id >= p->n_tables) || !match_field)
9265 t = table_find_by_id(p, table_id);
9266 if (!t || (match_field_id >= t->n_fields))
9269 f = &t->fields[match_field_id];
9270 match_field->match_type = f->match_type;
9271 match_field->is_header = t->header ? 1 : 0;
9272 match_field->n_bits = f->field->n_bits;
9273 match_field->offset = f->field->offset;
9279 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9281 uint32_t table_action_id,
9282 struct rte_swx_ctl_table_action_info *table_action)
9286 if (!p || (table_id >= p->n_tables) || !table_action)
9289 t = table_find_by_id(p, table_id);
9290 if (!t || (table_action_id >= t->n_actions))
9293 table_action->action_id = t->actions[table_action_id]->id;
9299 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9301 struct rte_swx_table_ops *table_ops,
9306 if (!p || (table_id >= p->n_tables))
9309 t = table_find_by_id(p, table_id);
9315 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9325 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9326 uint32_t selector_id,
9327 struct rte_swx_ctl_selector_info *selector)
9329 struct selector *s = NULL;
9331 if (!p || !selector)
9334 s = selector_find_by_id(p, selector_id);
9338 strcpy(selector->name, s->name);
9340 selector->n_selector_fields = s->n_selector_fields;
9341 selector->n_groups_max = s->n_groups_max;
9342 selector->n_members_per_group_max = s->n_members_per_group_max;
9348 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9349 uint32_t selector_id,
9350 struct rte_swx_ctl_table_match_field_info *field)
9354 if (!p || (selector_id >= p->n_selectors) || !field)
9357 s = selector_find_by_id(p, selector_id);
9361 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9362 field->is_header = 0;
9363 field->n_bits = s->group_id_field->n_bits;
9364 field->offset = s->group_id_field->offset;
9370 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9371 uint32_t selector_id,
9372 uint32_t selector_field_id,
9373 struct rte_swx_ctl_table_match_field_info *field)
9378 if (!p || (selector_id >= p->n_selectors) || !field)
9381 s = selector_find_by_id(p, selector_id);
9382 if (!s || (selector_field_id >= s->n_selector_fields))
9385 f = s->selector_fields[selector_field_id];
9386 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9387 field->is_header = s->selector_header ? 1 : 0;
9388 field->n_bits = f->n_bits;
9389 field->offset = f->offset;
9395 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9396 uint32_t selector_id,
9397 struct rte_swx_ctl_table_match_field_info *field)
9401 if (!p || (selector_id >= p->n_selectors) || !field)
9404 s = selector_find_by_id(p, selector_id);
9408 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9409 field->is_header = 0;
9410 field->n_bits = s->member_id_field->n_bits;
9411 field->offset = s->member_id_field->offset;
9417 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9418 uint32_t learner_id,
9419 struct rte_swx_ctl_learner_info *learner)
9421 struct learner *l = NULL;
9426 l = learner_find_by_id(p, learner_id);
9430 strcpy(learner->name, l->name);
9432 learner->n_match_fields = l->n_fields;
9433 learner->n_actions = l->n_actions;
9434 learner->default_action_is_const = l->default_action_is_const;
9435 learner->size = l->size;
9441 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9442 uint32_t learner_id,
9443 uint32_t match_field_id,
9444 struct rte_swx_ctl_table_match_field_info *match_field)
9449 if (!p || (learner_id >= p->n_learners) || !match_field)
9452 l = learner_find_by_id(p, learner_id);
9453 if (!l || (match_field_id >= l->n_fields))
9456 f = l->fields[match_field_id];
9457 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9458 match_field->is_header = l->header ? 1 : 0;
9459 match_field->n_bits = f->n_bits;
9460 match_field->offset = f->offset;
9466 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9467 uint32_t learner_id,
9468 uint32_t learner_action_id,
9469 struct rte_swx_ctl_table_action_info *learner_action)
9473 if (!p || (learner_id >= p->n_learners) || !learner_action)
9476 l = learner_find_by_id(p, learner_id);
9477 if (!l || (learner_action_id >= l->n_actions))
9480 learner_action->action_id = l->actions[learner_action_id]->id;
9486 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9487 struct rte_swx_table_state **table_state)
9489 if (!p || !table_state || !p->build_done)
9492 *table_state = p->table_state;
9497 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9498 struct rte_swx_table_state *table_state)
9500 if (!p || !table_state || !p->build_done)
9503 p->table_state = table_state;
9508 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9510 struct rte_swx_port_in_stats *stats)
9512 struct port_in *port;
9517 port = port_in_find(p, port_id);
9521 port->type->ops.stats_read(port->obj, stats);
9526 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9528 struct rte_swx_port_out_stats *stats)
9530 struct port_out *port;
9535 port = port_out_find(p, port_id);
9539 port->type->ops.stats_read(port->obj, stats);
9544 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9545 const char *table_name,
9546 struct rte_swx_table_stats *stats)
9548 struct table *table;
9549 struct table_statistics *table_stats;
9551 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9554 table = table_find(p, table_name);
9558 table_stats = &p->table_stats[table->id];
9560 memcpy(stats->n_pkts_action,
9561 table_stats->n_pkts_action,
9562 p->n_actions * sizeof(uint64_t));
9564 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9565 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9571 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9572 const char *selector_name,
9573 struct rte_swx_pipeline_selector_stats *stats)
9577 if (!p || !selector_name || !selector_name[0] || !stats)
9580 s = selector_find(p, selector_name);
9584 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9590 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9591 const char *learner_name,
9592 struct rte_swx_learner_stats *stats)
9595 struct learner_statistics *learner_stats;
9597 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9600 l = learner_find(p, learner_name);
9604 learner_stats = &p->learner_stats[l->id];
9606 memcpy(stats->n_pkts_action,
9607 learner_stats->n_pkts_action,
9608 p->n_actions * sizeof(uint64_t));
9610 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9611 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9613 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9614 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9616 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9622 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9623 uint32_t regarray_id,
9624 struct rte_swx_ctl_regarray_info *regarray)
9628 if (!p || !regarray)
9631 r = regarray_find_by_id(p, regarray_id);
9635 strcpy(regarray->name, r->name);
9636 regarray->size = r->size;
9641 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9642 const char *regarray_name,
9643 uint32_t regarray_index,
9646 struct regarray *regarray;
9647 struct regarray_runtime *r;
9649 if (!p || !regarray_name || !value)
9652 regarray = regarray_find(p, regarray_name);
9653 if (!regarray || (regarray_index >= regarray->size))
9656 r = &p->regarray_runtime[regarray->id];
9657 *value = r->regarray[regarray_index];
9662 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9663 const char *regarray_name,
9664 uint32_t regarray_index,
9667 struct regarray *regarray;
9668 struct regarray_runtime *r;
9670 if (!p || !regarray_name)
9673 regarray = regarray_find(p, regarray_name);
9674 if (!regarray || (regarray_index >= regarray->size))
9677 r = &p->regarray_runtime[regarray->id];
9678 r->regarray[regarray_index] = value;
9683 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9684 uint32_t metarray_id,
9685 struct rte_swx_ctl_metarray_info *metarray)
9689 if (!p || !metarray)
9692 m = metarray_find_by_id(p, metarray_id);
9696 strcpy(metarray->name, m->name);
9697 metarray->size = m->size;
9702 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9704 struct rte_meter_trtcm_params *params)
9706 struct meter_profile *mp;
9710 CHECK_NAME(name, EINVAL);
9711 CHECK(params, EINVAL);
9712 CHECK(!meter_profile_find(p, name), EEXIST);
9714 /* Node allocation. */
9715 mp = calloc(1, sizeof(struct meter_profile));
9718 /* Node initialization. */
9719 strcpy(mp->name, name);
9720 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9721 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9727 /* Node add to tailq. */
9728 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9734 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9737 struct meter_profile *mp;
9740 CHECK_NAME(name, EINVAL);
9742 mp = meter_profile_find(p, name);
9744 CHECK(!mp->n_users, EBUSY);
9746 /* Remove node from tailq. */
9747 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9754 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9755 const char *metarray_name,
9756 uint32_t metarray_index)
9758 struct meter_profile *mp_old;
9759 struct metarray *metarray;
9760 struct metarray_runtime *metarray_runtime;
9764 CHECK_NAME(metarray_name, EINVAL);
9766 metarray = metarray_find(p, metarray_name);
9767 CHECK(metarray, EINVAL);
9768 CHECK(metarray_index < metarray->size, EINVAL);
9770 metarray_runtime = &p->metarray_runtime[metarray->id];
9771 m = &metarray_runtime->metarray[metarray_index];
9772 mp_old = m->profile;
9782 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9783 const char *metarray_name,
9784 uint32_t metarray_index,
9785 const char *profile_name)
9787 struct meter_profile *mp, *mp_old;
9788 struct metarray *metarray;
9789 struct metarray_runtime *metarray_runtime;
9793 CHECK_NAME(metarray_name, EINVAL);
9795 metarray = metarray_find(p, metarray_name);
9796 CHECK(metarray, EINVAL);
9797 CHECK(metarray_index < metarray->size, EINVAL);
9799 mp = meter_profile_find(p, profile_name);
9802 metarray_runtime = &p->metarray_runtime[metarray->id];
9803 m = &metarray_runtime->metarray[metarray_index];
9804 mp_old = m->profile;
9806 memset(m, 0, sizeof(struct meter));
9807 rte_meter_trtcm_config(&m->m, &mp->profile);
9809 m->color_mask = RTE_COLORS;
9818 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9819 const char *metarray_name,
9820 uint32_t metarray_index,
9821 struct rte_swx_ctl_meter_stats *stats)
9823 struct metarray *metarray;
9824 struct metarray_runtime *metarray_runtime;
9828 CHECK_NAME(metarray_name, EINVAL);
9830 metarray = metarray_find(p, metarray_name);
9831 CHECK(metarray, EINVAL);
9832 CHECK(metarray_index < metarray->size, EINVAL);
9834 CHECK(stats, EINVAL);
9836 metarray_runtime = &p->metarray_runtime[metarray->id];
9837 m = &metarray_runtime->metarray[metarray_index];
9839 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9840 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9846 * Pipeline compilation.
9849 instr_type_to_name(struct instruction *instr)
9851 switch (instr->type) {
9852 case INSTR_RX: return "INSTR_RX";
9854 case INSTR_TX: return "INSTR_TX";
9855 case INSTR_TX_I: return "INSTR_TX_I";
9857 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
9858 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
9859 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
9860 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
9861 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
9862 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
9863 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
9864 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
9866 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
9868 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
9870 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
9871 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
9872 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
9873 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
9874 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
9875 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
9876 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
9877 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
9878 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
9880 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
9881 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
9883 case INSTR_MOV: return "INSTR_MOV";
9884 case INSTR_MOV_MH: return "INSTR_MOV_MH";
9885 case INSTR_MOV_HM: return "INSTR_MOV_HM";
9886 case INSTR_MOV_HH: return "INSTR_MOV_HH";
9887 case INSTR_MOV_I: return "INSTR_MOV_I";
9889 case INSTR_DMA_HT: return "INSTR_DMA_HT";
9890 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
9891 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
9892 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
9893 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
9894 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
9895 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
9896 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
9898 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
9899 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
9900 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
9901 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
9902 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
9903 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
9905 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
9906 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
9907 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
9908 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
9909 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
9910 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
9912 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
9913 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
9914 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
9915 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
9917 case INSTR_ALU_AND: return "INSTR_ALU_AND";
9918 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
9919 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
9920 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
9921 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
9923 case INSTR_ALU_OR: return "INSTR_ALU_OR";
9924 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
9925 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
9926 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
9927 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
9929 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
9930 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
9931 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
9932 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
9933 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
9935 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
9936 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
9937 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
9938 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
9939 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
9940 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
9942 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
9943 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
9944 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
9945 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
9946 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
9947 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
9949 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
9950 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
9951 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
9953 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
9954 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
9955 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
9956 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
9957 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
9958 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
9960 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
9961 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
9962 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
9963 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
9964 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
9965 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
9966 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
9967 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
9968 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
9970 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
9971 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
9972 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
9973 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
9974 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
9975 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
9976 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
9977 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
9978 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
9980 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
9981 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
9982 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
9984 case INSTR_METER_HHM: return "INSTR_METER_HHM";
9985 case INSTR_METER_HHI: return "INSTR_METER_HHI";
9986 case INSTR_METER_HMM: return "INSTR_METER_HMM";
9987 case INSTR_METER_HMI: return "INSTR_METER_HMI";
9988 case INSTR_METER_MHM: return "INSTR_METER_MHM";
9989 case INSTR_METER_MHI: return "INSTR_METER_MHI";
9990 case INSTR_METER_MMM: return "INSTR_METER_MMM";
9991 case INSTR_METER_MMI: return "INSTR_METER_MMI";
9992 case INSTR_METER_IHM: return "INSTR_METER_IHM";
9993 case INSTR_METER_IHI: return "INSTR_METER_IHI";
9994 case INSTR_METER_IMM: return "INSTR_METER_IMM";
9995 case INSTR_METER_IMI: return "INSTR_METER_IMI";
9997 case INSTR_TABLE: return "INSTR_TABLE";
9998 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
9999 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10000 case INSTR_LEARNER: return "INSTR_LEARNER";
10001 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10003 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10004 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10006 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10007 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10009 case INSTR_JMP: return "INSTR_JMP";
10010 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10011 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10012 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10013 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10014 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10015 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10016 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10017 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10018 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10019 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10020 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10021 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10022 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10023 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10024 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10025 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10026 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10027 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10028 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10029 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10030 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10031 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10032 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10033 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10034 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10035 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10036 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10037 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10039 case INSTR_RETURN: return "INSTR_RETURN";
10041 default: return "INSTR_UNKNOWN";
10046 (*instruction_export_t)(struct instruction *, FILE *);
10049 instr_io_export(struct instruction *instr, FILE *f)
10051 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10053 /* n_io, n_io_imm, n_hdrs. */
10054 if (instr->type == INSTR_RX ||
10055 instr->type == INSTR_TX ||
10056 instr->type == INSTR_HDR_EXTRACT_M ||
10057 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10060 if (instr->type == INSTR_TX_I)
10063 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10064 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10066 if (instr->type == INSTR_HDR_EXTRACT_M ||
10067 instr->type == INSTR_HDR_LOOKAHEAD ||
10068 instr->type == INSTR_HDR_EMIT)
10071 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10072 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10077 "\t\t.type = %s,\n",
10078 instr_type_to_name(instr));
10088 "\t\t\t\t.offset = %u,\n"
10089 "\t\t\t\t.n_bits = %u,\n"
10091 instr->io.io.offset,
10092 instr->io.io.n_bits);
10097 "\t\t\t\t.val = %u,\n"
10101 /* instr.io.hdr. */
10106 /* instr.io.hdr.header_id. */
10108 "\t\t\t.header_id = {");
10110 for (i = 0; i < n_hdrs; i++)
10113 instr->io.hdr.header_id[i]);
10118 /* instr.io.hdr.struct_id. */
10120 "\t\t\t.struct_id = {");
10122 for (i = 0; i < n_hdrs; i++)
10125 instr->io.hdr.struct_id[i]);
10130 /* instr.io.hdr.n_bytes. */
10132 "\t\t\t.n_bytes = {");
10134 for (i = 0; i < n_hdrs; i++)
10137 instr->io.hdr.n_bytes[i]);
10142 /* instr.io.hdr - closing curly brace. */
10147 /* instr.io - closing curly brace. */
10151 /* instr - closing curly brace. */
10157 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10161 "\t\t.type = %s,\n"
10163 "\t\t\t.header_id = %u,\n"
10166 instr_type_to_name(instr),
10167 instr->valid.header_id);
10171 instr_mov_export(struct instruction *instr, FILE *f)
10173 if (instr->type != INSTR_MOV_I)
10176 "\t\t.type = %s,\n"
10179 "\t\t\t\t.struct_id = %u,\n"
10180 "\t\t\t\t.n_bits = %u,\n"
10181 "\t\t\t\t.offset = %u,\n"
10184 "\t\t\t\t.struct_id = %u,\n"
10185 "\t\t\t\t.n_bits = %u,\n"
10186 "\t\t\t\t.offset = %u,\n"
10190 instr_type_to_name(instr),
10191 instr->mov.dst.struct_id,
10192 instr->mov.dst.n_bits,
10193 instr->mov.dst.offset,
10194 instr->mov.src.struct_id,
10195 instr->mov.src.n_bits,
10196 instr->mov.src.offset);
10200 "\t\t.type = %s,\n"
10203 "\t\t\t\t.struct_id = %u,\n"
10204 "\t\t\t\t.n_bits = %u,\n"
10205 "\t\t\t\t.offset = %u,\n"
10207 "\t\t\t.src_val = %" PRIu64 ",\n"
10210 instr_type_to_name(instr),
10211 instr->mov.dst.struct_id,
10212 instr->mov.dst.n_bits,
10213 instr->mov.dst.offset,
10214 instr->mov.src_val);
10218 instr_dma_ht_export(struct instruction *instr, FILE *f)
10220 uint32_t n_dma = 0, i;
10223 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10228 "\t\t.type = %s,\n",
10229 instr_type_to_name(instr));
10235 /* instr.dma.dst. */
10237 "\t\t\t.dst = {\n");
10239 /* instr.dma.dst.header_id. */
10241 "\t\t\t\t.header_id = {");
10243 for (i = 0; i < n_dma; i++)
10246 instr->dma.dst.header_id[i]);
10251 /* instr.dma.dst.struct_id. */
10253 "\t\t\t\t.struct_id = {");
10255 for (i = 0; i < n_dma; i++)
10258 instr->dma.dst.struct_id[i]);
10263 /* instr.dma.dst - closing curly brace. */
10267 /* instr.dma.src. */
10269 "\t\t\t.src = {\n");
10271 /* instr.dma.src.offset. */
10273 "\t\t\t\t.offset = {");
10275 for (i = 0; i < n_dma; i++)
10278 instr->dma.src.offset[i]);
10283 /* instr.dma.src - closing curly brace. */
10287 /* instr.dma.n_bytes. */
10289 "\t\t\t.n_bytes = {");
10291 for (i = 0; i < n_dma; i++)
10294 instr->dma.n_bytes[i]);
10299 /* instr.dma - closing curly brace. */
10303 /* instr - closing curly brace. */
10309 instr_alu_export(struct instruction *instr, FILE *f)
10313 if (instr->type == INSTR_ALU_ADD_MI ||
10314 instr->type == INSTR_ALU_ADD_HI ||
10315 instr->type == INSTR_ALU_SUB_MI ||
10316 instr->type == INSTR_ALU_SUB_HI ||
10317 instr->type == INSTR_ALU_SHL_MI ||
10318 instr->type == INSTR_ALU_SHL_HI ||
10319 instr->type == INSTR_ALU_SHR_MI ||
10320 instr->type == INSTR_ALU_SHR_HI ||
10321 instr->type == INSTR_ALU_AND_I ||
10322 instr->type == INSTR_ALU_OR_I ||
10323 instr->type == INSTR_ALU_XOR_I)
10329 "\t\t.type = %s,\n"
10332 "\t\t\t\t.struct_id = %u,\n"
10333 "\t\t\t\t.n_bits = %u,\n"
10334 "\t\t\t\t.offset = %u,\n"
10337 "\t\t\t\t.struct_id = %u,\n"
10338 "\t\t\t\t.n_bits = %u,\n"
10339 "\t\t\t\t.offset = %u,\n"
10343 instr_type_to_name(instr),
10344 instr->alu.dst.struct_id,
10345 instr->alu.dst.n_bits,
10346 instr->alu.dst.offset,
10347 instr->alu.src.struct_id,
10348 instr->alu.src.n_bits,
10349 instr->alu.src.offset);
10353 "\t\t.type = %s,\n"
10356 "\t\t\t\t.struct_id = %u,\n"
10357 "\t\t\t\t.n_bits = %u,\n"
10358 "\t\t\t\t.offset = %u,\n"
10360 "\t\t\t.src_val = %" PRIu64 ",\n"
10363 instr_type_to_name(instr),
10364 instr->alu.dst.struct_id,
10365 instr->alu.dst.n_bits,
10366 instr->alu.dst.offset,
10367 instr->alu.src_val);
10371 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10373 int prefetch = 0, idx_imm = 0, src_imm = 0;
10375 if (instr->type == INSTR_REGPREFETCH_RH ||
10376 instr->type == INSTR_REGPREFETCH_RM ||
10377 instr->type == INSTR_REGPREFETCH_RI)
10380 /* index is the 3rd operand for the regrd instruction and the 2nd
10381 * operand for the regwr and regadd instructions.
10383 if (instr->type == INSTR_REGPREFETCH_RI ||
10384 instr->type == INSTR_REGRD_HRI ||
10385 instr->type == INSTR_REGRD_MRI ||
10386 instr->type == INSTR_REGWR_RIH ||
10387 instr->type == INSTR_REGWR_RIM ||
10388 instr->type == INSTR_REGWR_RII ||
10389 instr->type == INSTR_REGADD_RIH ||
10390 instr->type == INSTR_REGADD_RIM ||
10391 instr->type == INSTR_REGADD_RII)
10394 /* src is the 3rd operand for the regwr and regadd instructions. */
10395 if (instr->type == INSTR_REGWR_RHI ||
10396 instr->type == INSTR_REGWR_RMI ||
10397 instr->type == INSTR_REGWR_RII ||
10398 instr->type == INSTR_REGADD_RHI ||
10399 instr->type == INSTR_REGADD_RMI ||
10400 instr->type == INSTR_REGADD_RII)
10403 /* instr.regarray.regarray_id. */
10406 "\t\t.type = %s,\n"
10407 "\t\t.regarray = {\n"
10408 "\t\t\t.regarray_id = %u,\n",
10409 instr_type_to_name(instr),
10410 instr->regarray.regarray_id);
10412 /* instr.regarray.idx / instr.regarray.idx_val. */
10415 "\t\t\t\t.idx = {\n"
10416 "\t\t\t\t\t.struct_id = %u,\n"
10417 "\t\t\t\t\t.n_bits = %u,\n"
10418 "\t\t\t\t\t.offset = %u,\n"
10420 instr->regarray.idx.struct_id,
10421 instr->regarray.idx.n_bits,
10422 instr->regarray.idx.offset);
10425 "\t\t\t\t.idx_val = %u,\n",
10426 instr->regarray.idx_val);
10428 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10432 "\t\t\t\t.dstsrc = {\n"
10433 "\t\t\t\t\t.struct_id = %u,\n"
10434 "\t\t\t\t\t.n_bits = %u,\n"
10435 "\t\t\t\t\t.offset = %u,\n"
10437 instr->regarray.dstsrc.struct_id,
10438 instr->regarray.dstsrc.n_bits,
10439 instr->regarray.dstsrc.offset);
10442 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10443 instr->regarray.dstsrc_val);
10446 /* instr.regarray and instr - closing curly braces. */
10453 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10455 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10457 if (instr->type == INSTR_METPREFETCH_H ||
10458 instr->type == INSTR_METPREFETCH_M ||
10459 instr->type == INSTR_METPREFETCH_I)
10463 if (instr->type == INSTR_METPREFETCH_I ||
10464 instr->type == INSTR_METER_IHM ||
10465 instr->type == INSTR_METER_IHI ||
10466 instr->type == INSTR_METER_IMM ||
10467 instr->type == INSTR_METER_IMI)
10470 /* color_in_imm. */
10471 if (instr->type == INSTR_METER_HHI ||
10472 instr->type == INSTR_METER_HMI ||
10473 instr->type == INSTR_METER_MHI ||
10474 instr->type == INSTR_METER_MMI ||
10475 instr->type == INSTR_METER_IHI ||
10476 instr->type == INSTR_METER_IMI)
10479 /* instr.meter.metarray_id. */
10482 "\t\t.type = %s,\n"
10484 "\t\t\t.metarray_id = %u,\n",
10485 instr_type_to_name(instr),
10486 instr->meter.metarray_id);
10488 /* instr.meter.idx / instr.meter.idx_val. */
10492 "\t\t\t\t.struct_id = %u,\n"
10493 "\t\t\t\t.n_bits = %u,\n"
10494 "\t\t\t\t.offset = %u,\n"
10496 instr->meter.idx.struct_id,
10497 instr->meter.idx.n_bits,
10498 instr->meter.idx.offset);
10501 "\t\t\t.idx_val = %u,\n",
10502 instr->meter.idx_val);
10505 /* instr.meter.length. */
10507 "\t\t\t.length = {\n"
10508 "\t\t\t\t.struct_id = %u,\n"
10509 "\t\t\t\t.n_bits = %u,\n"
10510 "\t\t\t\t.offset = %u,\n"
10512 instr->meter.length.struct_id,
10513 instr->meter.length.n_bits,
10514 instr->meter.length.offset);
10516 /* instr.meter.color_in / instr.meter.color_in_val. */
10519 "\t\t\t.color_in = {\n"
10520 "\t\t\t\t.struct_id = %u,\n"
10521 "\t\t\t\t.n_bits = %u,\n"
10522 "\t\t\t\t.offset = %u,\n"
10524 instr->meter.color_in.struct_id,
10525 instr->meter.color_in.n_bits,
10526 instr->meter.color_in.offset);
10529 "\t\t\t.color_in_val = %u,\n",
10530 (uint32_t)instr->meter.color_in_val);
10532 /* instr.meter.color_out. */
10534 "\t\t\t.color_out = {\n"
10535 "\t\t\t\t.struct_id = %u,\n"
10536 "\t\t\t\t.n_bits = %u,\n"
10537 "\t\t\t\t.offset = %u,\n"
10539 instr->meter.color_out.struct_id,
10540 instr->meter.color_out.n_bits,
10541 instr->meter.color_out.offset);
10544 /* instr.meter and instr - closing curly braces. */
10551 instr_table_export(struct instruction *instr,
10556 "\t\t.type = %s,\n"
10558 "\t\t\t.table_id = %u,\n"
10561 instr_type_to_name(instr),
10562 instr->table.table_id);
10566 instr_learn_export(struct instruction *instr, FILE *f)
10570 "\t\t.type = %s,\n"
10572 "\t\t\t\t.action_id = %u,\n"
10575 instr_type_to_name(instr),
10576 instr->learn.action_id);
10580 instr_forget_export(struct instruction *instr, FILE *f)
10584 "\t\t.type = %s,\n"
10586 instr_type_to_name(instr));
10590 instr_extern_export(struct instruction *instr, FILE *f)
10592 if (instr->type == INSTR_EXTERN_OBJ)
10595 "\t\t.type = %s,\n"
10596 "\t\t.ext_obj = {\n"
10597 "\t\t\t.ext_obj_id = %u,\n"
10598 "\t\t\t.func_id = %u,\n"
10601 instr_type_to_name(instr),
10602 instr->ext_obj.ext_obj_id,
10603 instr->ext_obj.func_id);
10607 "\t\t.type = %s,\n"
10608 "\t\t.ext_func = {\n"
10609 "\t\t\t.ext_func_id = %u,\n"
10612 instr_type_to_name(instr),
10613 instr->ext_func.ext_func_id);
10617 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10621 "\t\t.type = %s,\n"
10623 "\t\t\t.ip = NULL,\n",
10624 instr_type_to_name(instr));
10626 switch (instr->type) {
10627 case INSTR_JMP_VALID:
10628 case INSTR_JMP_INVALID:
10630 "\t\t\t.header_id = %u,\n",
10631 instr->jmp.header_id);
10634 case INSTR_JMP_ACTION_HIT:
10635 case INSTR_JMP_ACTION_MISS:
10637 "\t\t\t.action_id = %u,\n",
10638 instr->jmp.action_id);
10642 case INSTR_JMP_EQ_MH:
10643 case INSTR_JMP_EQ_HM:
10644 case INSTR_JMP_EQ_HH:
10645 case INSTR_JMP_NEQ:
10646 case INSTR_JMP_NEQ_MH:
10647 case INSTR_JMP_NEQ_HM:
10648 case INSTR_JMP_NEQ_HH:
10650 case INSTR_JMP_LT_MH:
10651 case INSTR_JMP_LT_HM:
10652 case INSTR_JMP_LT_HH:
10654 case INSTR_JMP_GT_MH:
10655 case INSTR_JMP_GT_HM:
10656 case INSTR_JMP_GT_HH:
10659 "\t\t\t\t.struct_id = %u,\n"
10660 "\t\t\t\t.n_bits = %u,\n"
10661 "\t\t\t\t.offset = %u,\n"
10664 "\t\t\t\t.struct_id = %u,\n"
10665 "\t\t\t\t.n_bits = %u,\n"
10666 "\t\t\t\t.offset = %u,\n"
10668 instr->jmp.a.struct_id,
10669 instr->jmp.a.n_bits,
10670 instr->jmp.a.offset,
10671 instr->jmp.b.struct_id,
10672 instr->jmp.b.n_bits,
10673 instr->jmp.b.offset);
10676 case INSTR_JMP_EQ_I:
10677 case INSTR_JMP_NEQ_I:
10678 case INSTR_JMP_LT_MI:
10679 case INSTR_JMP_LT_HI:
10680 case INSTR_JMP_GT_MI:
10681 case INSTR_JMP_GT_HI:
10684 "\t\t\t\t.struct_id = %u,\n"
10685 "\t\t\t\t.n_bits = %u,\n"
10686 "\t\t\t\t.offset = %u,\n"
10688 "\t\t\t.b_val = %" PRIu64 ",\n",
10689 instr->jmp.a.struct_id,
10690 instr->jmp.a.n_bits,
10691 instr->jmp.a.offset,
10705 instr_return_export(struct instruction *instr,
10710 "\t\t.type = %s,\n",
10711 instr_type_to_name(instr));
10717 static instruction_export_t export_table[] = {
10718 [INSTR_RX] = instr_io_export,
10720 [INSTR_TX] = instr_io_export,
10721 [INSTR_TX_I] = instr_io_export,
10723 [INSTR_HDR_EXTRACT] = instr_io_export,
10724 [INSTR_HDR_EXTRACT2] = instr_io_export,
10725 [INSTR_HDR_EXTRACT3] = instr_io_export,
10726 [INSTR_HDR_EXTRACT4] = instr_io_export,
10727 [INSTR_HDR_EXTRACT5] = instr_io_export,
10728 [INSTR_HDR_EXTRACT6] = instr_io_export,
10729 [INSTR_HDR_EXTRACT7] = instr_io_export,
10730 [INSTR_HDR_EXTRACT8] = instr_io_export,
10732 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10734 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10736 [INSTR_HDR_EMIT] = instr_io_export,
10737 [INSTR_HDR_EMIT_TX] = instr_io_export,
10738 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10739 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10740 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10741 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10742 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10743 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10744 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10746 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10747 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10749 [INSTR_MOV] = instr_mov_export,
10750 [INSTR_MOV_MH] = instr_mov_export,
10751 [INSTR_MOV_HM] = instr_mov_export,
10752 [INSTR_MOV_HH] = instr_mov_export,
10753 [INSTR_MOV_I] = instr_mov_export,
10755 [INSTR_DMA_HT] = instr_dma_ht_export,
10756 [INSTR_DMA_HT2] = instr_dma_ht_export,
10757 [INSTR_DMA_HT3] = instr_dma_ht_export,
10758 [INSTR_DMA_HT4] = instr_dma_ht_export,
10759 [INSTR_DMA_HT5] = instr_dma_ht_export,
10760 [INSTR_DMA_HT6] = instr_dma_ht_export,
10761 [INSTR_DMA_HT7] = instr_dma_ht_export,
10762 [INSTR_DMA_HT8] = instr_dma_ht_export,
10764 [INSTR_ALU_ADD] = instr_alu_export,
10765 [INSTR_ALU_ADD_MH] = instr_alu_export,
10766 [INSTR_ALU_ADD_HM] = instr_alu_export,
10767 [INSTR_ALU_ADD_HH] = instr_alu_export,
10768 [INSTR_ALU_ADD_MI] = instr_alu_export,
10769 [INSTR_ALU_ADD_HI] = instr_alu_export,
10771 [INSTR_ALU_SUB] = instr_alu_export,
10772 [INSTR_ALU_SUB_MH] = instr_alu_export,
10773 [INSTR_ALU_SUB_HM] = instr_alu_export,
10774 [INSTR_ALU_SUB_HH] = instr_alu_export,
10775 [INSTR_ALU_SUB_MI] = instr_alu_export,
10776 [INSTR_ALU_SUB_HI] = instr_alu_export,
10778 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10779 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10780 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10781 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10783 [INSTR_ALU_AND] = instr_alu_export,
10784 [INSTR_ALU_AND_MH] = instr_alu_export,
10785 [INSTR_ALU_AND_HM] = instr_alu_export,
10786 [INSTR_ALU_AND_HH] = instr_alu_export,
10787 [INSTR_ALU_AND_I] = instr_alu_export,
10789 [INSTR_ALU_OR] = instr_alu_export,
10790 [INSTR_ALU_OR_MH] = instr_alu_export,
10791 [INSTR_ALU_OR_HM] = instr_alu_export,
10792 [INSTR_ALU_OR_HH] = instr_alu_export,
10793 [INSTR_ALU_OR_I] = instr_alu_export,
10795 [INSTR_ALU_XOR] = instr_alu_export,
10796 [INSTR_ALU_XOR_MH] = instr_alu_export,
10797 [INSTR_ALU_XOR_HM] = instr_alu_export,
10798 [INSTR_ALU_XOR_HH] = instr_alu_export,
10799 [INSTR_ALU_XOR_I] = instr_alu_export,
10801 [INSTR_ALU_SHL] = instr_alu_export,
10802 [INSTR_ALU_SHL_MH] = instr_alu_export,
10803 [INSTR_ALU_SHL_HM] = instr_alu_export,
10804 [INSTR_ALU_SHL_HH] = instr_alu_export,
10805 [INSTR_ALU_SHL_MI] = instr_alu_export,
10806 [INSTR_ALU_SHL_HI] = instr_alu_export,
10808 [INSTR_ALU_SHR] = instr_alu_export,
10809 [INSTR_ALU_SHR_MH] = instr_alu_export,
10810 [INSTR_ALU_SHR_HM] = instr_alu_export,
10811 [INSTR_ALU_SHR_HH] = instr_alu_export,
10812 [INSTR_ALU_SHR_MI] = instr_alu_export,
10813 [INSTR_ALU_SHR_HI] = instr_alu_export,
10815 [INSTR_REGPREFETCH_RH] = instr_reg_export,
10816 [INSTR_REGPREFETCH_RM] = instr_reg_export,
10817 [INSTR_REGPREFETCH_RI] = instr_reg_export,
10819 [INSTR_REGRD_HRH] = instr_reg_export,
10820 [INSTR_REGRD_HRM] = instr_reg_export,
10821 [INSTR_REGRD_MRH] = instr_reg_export,
10822 [INSTR_REGRD_MRM] = instr_reg_export,
10823 [INSTR_REGRD_HRI] = instr_reg_export,
10824 [INSTR_REGRD_MRI] = instr_reg_export,
10826 [INSTR_REGWR_RHH] = instr_reg_export,
10827 [INSTR_REGWR_RHM] = instr_reg_export,
10828 [INSTR_REGWR_RMH] = instr_reg_export,
10829 [INSTR_REGWR_RMM] = instr_reg_export,
10830 [INSTR_REGWR_RHI] = instr_reg_export,
10831 [INSTR_REGWR_RMI] = instr_reg_export,
10832 [INSTR_REGWR_RIH] = instr_reg_export,
10833 [INSTR_REGWR_RIM] = instr_reg_export,
10834 [INSTR_REGWR_RII] = instr_reg_export,
10836 [INSTR_REGADD_RHH] = instr_reg_export,
10837 [INSTR_REGADD_RHM] = instr_reg_export,
10838 [INSTR_REGADD_RMH] = instr_reg_export,
10839 [INSTR_REGADD_RMM] = instr_reg_export,
10840 [INSTR_REGADD_RHI] = instr_reg_export,
10841 [INSTR_REGADD_RMI] = instr_reg_export,
10842 [INSTR_REGADD_RIH] = instr_reg_export,
10843 [INSTR_REGADD_RIM] = instr_reg_export,
10844 [INSTR_REGADD_RII] = instr_reg_export,
10846 [INSTR_METPREFETCH_H] = instr_meter_export,
10847 [INSTR_METPREFETCH_M] = instr_meter_export,
10848 [INSTR_METPREFETCH_I] = instr_meter_export,
10850 [INSTR_METER_HHM] = instr_meter_export,
10851 [INSTR_METER_HHI] = instr_meter_export,
10852 [INSTR_METER_HMM] = instr_meter_export,
10853 [INSTR_METER_HMI] = instr_meter_export,
10854 [INSTR_METER_MHM] = instr_meter_export,
10855 [INSTR_METER_MHI] = instr_meter_export,
10856 [INSTR_METER_MMM] = instr_meter_export,
10857 [INSTR_METER_MMI] = instr_meter_export,
10858 [INSTR_METER_IHM] = instr_meter_export,
10859 [INSTR_METER_IHI] = instr_meter_export,
10860 [INSTR_METER_IMM] = instr_meter_export,
10861 [INSTR_METER_IMI] = instr_meter_export,
10863 [INSTR_TABLE] = instr_table_export,
10864 [INSTR_TABLE_AF] = instr_table_export,
10865 [INSTR_SELECTOR] = instr_table_export,
10866 [INSTR_LEARNER] = instr_table_export,
10867 [INSTR_LEARNER_AF] = instr_table_export,
10869 [INSTR_LEARNER_LEARN] = instr_learn_export,
10870 [INSTR_LEARNER_FORGET] = instr_forget_export,
10872 [INSTR_EXTERN_OBJ] = instr_extern_export,
10873 [INSTR_EXTERN_FUNC] = instr_extern_export,
10875 [INSTR_JMP] = instr_jmp_export,
10876 [INSTR_JMP_VALID] = instr_jmp_export,
10877 [INSTR_JMP_INVALID] = instr_jmp_export,
10878 [INSTR_JMP_HIT] = instr_jmp_export,
10879 [INSTR_JMP_MISS] = instr_jmp_export,
10880 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
10881 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
10883 [INSTR_JMP_EQ] = instr_jmp_export,
10884 [INSTR_JMP_EQ_MH] = instr_jmp_export,
10885 [INSTR_JMP_EQ_HM] = instr_jmp_export,
10886 [INSTR_JMP_EQ_HH] = instr_jmp_export,
10887 [INSTR_JMP_EQ_I] = instr_jmp_export,
10889 [INSTR_JMP_NEQ] = instr_jmp_export,
10890 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
10891 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
10892 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
10893 [INSTR_JMP_NEQ_I] = instr_jmp_export,
10895 [INSTR_JMP_LT] = instr_jmp_export,
10896 [INSTR_JMP_LT_MH] = instr_jmp_export,
10897 [INSTR_JMP_LT_HM] = instr_jmp_export,
10898 [INSTR_JMP_LT_HH] = instr_jmp_export,
10899 [INSTR_JMP_LT_MI] = instr_jmp_export,
10900 [INSTR_JMP_LT_HI] = instr_jmp_export,
10902 [INSTR_JMP_GT] = instr_jmp_export,
10903 [INSTR_JMP_GT_MH] = instr_jmp_export,
10904 [INSTR_JMP_GT_HM] = instr_jmp_export,
10905 [INSTR_JMP_GT_HH] = instr_jmp_export,
10906 [INSTR_JMP_GT_MI] = instr_jmp_export,
10907 [INSTR_JMP_GT_HI] = instr_jmp_export,
10909 [INSTR_RETURN] = instr_return_export,
10913 action_data_codegen(struct action *a, FILE *f)
10918 "static const struct instruction action_%s_instructions[] = {\n",
10921 for (i = 0; i < a->n_instructions; i++) {
10922 struct instruction *instr = &a->instructions[i];
10923 instruction_export_t func = export_table[instr->type];
10928 fprintf(f, "};\n");
10931 static const char *
10932 instr_type_to_func(struct instruction *instr)
10934 switch (instr->type) {
10935 case INSTR_RX: return NULL;
10937 case INSTR_TX: return "__instr_tx_exec";
10938 case INSTR_TX_I: return "__instr_tx_i_exec";
10940 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
10941 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
10942 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
10943 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
10944 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
10945 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
10946 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
10947 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
10949 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
10951 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
10953 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
10954 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
10955 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
10956 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
10957 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
10958 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
10959 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
10960 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
10961 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
10963 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
10964 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
10966 case INSTR_MOV: return "__instr_mov_exec";
10967 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
10968 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
10969 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
10970 case INSTR_MOV_I: return "__instr_mov_i_exec";
10972 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
10973 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
10974 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
10975 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
10976 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
10977 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
10978 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
10979 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
10981 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
10982 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
10983 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
10984 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
10985 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
10986 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
10988 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
10989 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
10990 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
10991 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
10992 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
10993 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
10995 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
10996 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
10997 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
10998 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11000 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11001 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11002 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11003 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11004 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11006 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11007 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11008 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11009 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11010 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11012 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11013 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11014 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11015 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11016 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11018 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11019 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11020 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11021 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11022 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11023 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11025 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11026 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11027 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11028 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11029 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11030 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11032 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11033 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11034 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11036 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11037 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11038 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11039 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11040 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11041 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11043 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11044 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11045 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11046 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11047 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11048 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11049 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11050 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11051 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11053 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11054 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11055 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11056 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11057 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11058 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11059 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11060 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11061 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11063 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11064 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11065 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11067 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11068 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11069 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11070 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11071 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11072 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11073 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11074 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11075 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11076 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11077 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11078 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11080 case INSTR_TABLE: return NULL;
11081 case INSTR_TABLE_AF: return NULL;
11082 case INSTR_SELECTOR: return NULL;
11083 case INSTR_LEARNER: return NULL;
11084 case INSTR_LEARNER_AF: return NULL;
11086 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11087 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11089 case INSTR_EXTERN_OBJ: return NULL;
11090 case INSTR_EXTERN_FUNC: return NULL;
11092 case INSTR_JMP: return NULL;
11093 case INSTR_JMP_VALID: return NULL;
11094 case INSTR_JMP_INVALID: return NULL;
11095 case INSTR_JMP_HIT: return NULL;
11096 case INSTR_JMP_MISS: return NULL;
11097 case INSTR_JMP_ACTION_HIT: return NULL;
11098 case INSTR_JMP_ACTION_MISS: return NULL;
11099 case INSTR_JMP_EQ: return NULL;
11100 case INSTR_JMP_EQ_MH: return NULL;
11101 case INSTR_JMP_EQ_HM: return NULL;
11102 case INSTR_JMP_EQ_HH: return NULL;
11103 case INSTR_JMP_EQ_I: return NULL;
11104 case INSTR_JMP_NEQ: return NULL;
11105 case INSTR_JMP_NEQ_MH: return NULL;
11106 case INSTR_JMP_NEQ_HM: return NULL;
11107 case INSTR_JMP_NEQ_HH: return NULL;
11108 case INSTR_JMP_NEQ_I: return NULL;
11109 case INSTR_JMP_LT: return NULL;
11110 case INSTR_JMP_LT_MH: return NULL;
11111 case INSTR_JMP_LT_HM: return NULL;
11112 case INSTR_JMP_LT_HH: return NULL;
11113 case INSTR_JMP_LT_MI: return NULL;
11114 case INSTR_JMP_LT_HI: return NULL;
11115 case INSTR_JMP_GT: return NULL;
11116 case INSTR_JMP_GT_MH: return NULL;
11117 case INSTR_JMP_GT_HM: return NULL;
11118 case INSTR_JMP_GT_HH: return NULL;
11119 case INSTR_JMP_GT_MI: return NULL;
11120 case INSTR_JMP_GT_HI: return NULL;
11122 case INSTR_RETURN: return NULL;
11124 default: return NULL;
11129 action_instr_does_tx_codegen(struct action *a,
11130 uint32_t instr_pos,
11131 struct instruction *instr,
11135 "%s(p, t, &action_%s_instructions[%u]);\n"
11136 "\tthread_ip_reset(p, t);\n"
11137 "\tinstr_rx_exec(p);\n"
11139 instr_type_to_func(instr),
11145 action_instr_extern_obj_codegen(struct action *a,
11146 uint32_t instr_pos,
11150 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11156 action_instr_extern_func_codegen(struct action *a,
11157 uint32_t instr_pos,
11161 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11167 action_instr_jmp_codegen(struct action *a,
11168 uint32_t instr_pos,
11169 struct instruction *instr,
11170 struct instruction_data *data,
11173 switch (instr->type) {
11180 case INSTR_JMP_VALID:
11182 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11189 case INSTR_JMP_INVALID:
11191 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11198 case INSTR_JMP_HIT:
11205 case INSTR_JMP_MISS:
11212 case INSTR_JMP_ACTION_HIT:
11214 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11221 case INSTR_JMP_ACTION_MISS:
11223 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11232 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11233 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11242 case INSTR_JMP_EQ_MH:
11244 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11245 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11254 case INSTR_JMP_EQ_HM:
11256 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11257 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11266 case INSTR_JMP_EQ_HH:
11268 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11269 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11278 case INSTR_JMP_EQ_I:
11280 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11281 "action_%s_instructions[%u].jmp.b_val)\n"
11290 case INSTR_JMP_NEQ:
11292 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11293 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11302 case INSTR_JMP_NEQ_MH:
11304 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11305 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11314 case INSTR_JMP_NEQ_HM:
11316 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11317 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11326 case INSTR_JMP_NEQ_HH:
11328 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11329 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11338 case INSTR_JMP_NEQ_I:
11340 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11341 "action_%s_instructions[%u].jmp.b_val)\n"
11352 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11353 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11362 case INSTR_JMP_LT_MH:
11364 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11365 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11374 case INSTR_JMP_LT_HM:
11376 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11377 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11386 case INSTR_JMP_LT_HH:
11388 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11389 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11398 case INSTR_JMP_LT_MI:
11400 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11401 "action_%s_instructions[%u].jmp.b_val)\n"
11410 case INSTR_JMP_LT_HI:
11412 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11413 "action_%s_instructions[%u].jmp.b_val)\n"
11424 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11425 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11434 case INSTR_JMP_GT_MH:
11436 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11437 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11446 case INSTR_JMP_GT_HM:
11448 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11449 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11458 case INSTR_JMP_GT_HH:
11460 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11461 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11470 case INSTR_JMP_GT_MI:
11472 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11473 "action_%s_instructions[%u].jmp.b_val)\n"
11482 case INSTR_JMP_GT_HI:
11484 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11485 "action_%s_instructions[%u].jmp.b_val)\n"
11500 action_instr_return_codegen(FILE *f)
11507 action_instr_codegen(struct action *a, FILE *f)
11513 "action_%s_run(struct rte_swx_pipeline *p)\n"
11515 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11519 for (i = 0; i < a->n_instructions; i++) {
11520 struct instruction *instr = &a->instructions[i];
11521 struct instruction_data *data = &a->instruction_data[i];
11523 /* Label, if present. */
11524 if (data->label[0])
11525 fprintf(f, "\n%s : ", data->label);
11527 fprintf(f, "\n\t");
11529 /* TX instruction type. */
11530 if (instruction_does_tx(instr)) {
11531 action_instr_does_tx_codegen(a, i, instr, f);
11535 /* Extern object/function instruction type. */
11536 if (instr->type == INSTR_EXTERN_OBJ) {
11537 action_instr_extern_obj_codegen(a, i, f);
11541 if (instr->type == INSTR_EXTERN_FUNC) {
11542 action_instr_extern_func_codegen(a, i, f);
11546 /* Jump instruction type. */
11547 if (instruction_is_jmp(instr)) {
11548 action_instr_jmp_codegen(a, i, instr, data, f);
11552 /* Return instruction type. */
11553 if (instr->type == INSTR_RETURN) {
11554 action_instr_return_codegen(f);
11558 /* Any other instruction type. */
11560 "%s(p, t, &action_%s_instructions[%u]);\n",
11561 instr_type_to_func(instr),
11566 fprintf(f, "}\n\n");
11569 struct instruction_group {
11570 TAILQ_ENTRY(instruction_group) node;
11574 uint32_t first_instr_id;
11576 uint32_t last_instr_id;
11581 TAILQ_HEAD(instruction_group_list, instruction_group);
11583 static struct instruction_group *
11584 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11586 struct instruction_group *g;
11588 TAILQ_FOREACH(g, igl, node)
11589 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11596 instruction_group_list_free(struct instruction_group_list *igl)
11602 struct instruction_group *g;
11604 g = TAILQ_FIRST(igl);
11608 TAILQ_REMOVE(igl, g, node);
11615 static struct instruction_group_list *
11616 instruction_group_list_create(struct rte_swx_pipeline *p)
11618 struct instruction_group_list *igl = NULL;
11619 struct instruction_group *g = NULL;
11620 uint32_t n_groups = 0, i;
11622 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11626 igl = calloc(1, sizeof(struct instruction_group_list));
11632 /* Allocate the first group. */
11633 g = calloc(1, sizeof(struct instruction_group));
11637 /* Iteration 1: Separate the instructions into groups based on the thread yield
11638 * instructions. Do not worry about the jump instructions at this point.
11640 for (i = 0; i < p->n_instructions; i++) {
11641 struct instruction *instr = &p->instructions[i];
11643 /* Check for thread yield instructions. */
11644 if (!instruction_does_thread_yield(instr))
11647 /* If the current group contains at least one instruction, then finalize it (with
11648 * the previous instruction), add it to the list and allocate a new group (that
11649 * starts with the current instruction).
11651 if (i - g->first_instr_id) {
11652 /* Finalize the group. */
11653 g->last_instr_id = i - 1;
11655 /* Add the group to the list. Advance the number of groups. */
11656 TAILQ_INSERT_TAIL(igl, g, node);
11659 /* Allocate a new group. */
11660 g = calloc(1, sizeof(struct instruction_group));
11664 /* Initialize the new group. */
11665 g->group_id = n_groups;
11666 g->first_instr_id = i;
11669 /* Finalize the current group (with the current instruction, therefore this group
11670 * contains just the current thread yield instruction), add it to the list and
11671 * allocate a new group (that starts with the next instruction).
11674 /* Finalize the group. */
11675 g->last_instr_id = i;
11677 /* Add the group to the list. Advance the number of groups. */
11678 TAILQ_INSERT_TAIL(igl, g, node);
11681 /* Allocate a new group. */
11682 g = calloc(1, sizeof(struct instruction_group));
11686 /* Initialize the new group. */
11687 g->group_id = n_groups;
11688 g->first_instr_id = i + 1;
11691 /* Handle the last group. */
11692 if (i - g->first_instr_id) {
11693 /* Finalize the group. */
11694 g->last_instr_id = i - 1;
11696 /* Add the group to the list. Advance the number of groups. */
11697 TAILQ_INSERT_TAIL(igl, g, node);
11704 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11705 * the destination of a jump instruction located in a different group ("far jump"), then the
11706 * current group has to be split, so that the instruction representing the far jump
11707 * destination is at the start of its group.
11710 int is_modified = 0;
11712 for (i = 0; i < p->n_instructions; i++) {
11713 struct instruction_data *data = &p->instruction_data[i];
11714 struct instruction_group *g;
11717 /* Continue when the current instruction is not a jump destination. */
11718 if (!data->n_users)
11721 g = instruction_group_list_group_find(igl, i);
11725 /* Find out all the jump instructions with this destination. */
11726 for (j = 0; j < p->n_instructions; j++) {
11727 struct instruction *jmp_instr = &p->instructions[j];
11728 struct instruction_data *jmp_data = &p->instruction_data[j];
11729 struct instruction_group *jmp_g, *new_g;
11731 /* Continue when not a jump instruction. Even when jump instruction,
11732 * continue when the jump destination is not this instruction.
11734 if (!instruction_is_jmp(jmp_instr) ||
11735 strcmp(jmp_data->jmp_label, data->label))
11738 jmp_g = instruction_group_list_group_find(igl, j);
11742 /* Continue when both the jump instruction and the jump destination
11743 * instruction are in the same group. Even when in different groups,
11744 * still continue if the jump destination instruction is already the
11745 * first instruction of its group.
11747 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11750 /* Split the group of the current jump destination instruction to
11751 * make this instruction the first instruction of a new group.
11753 new_g = calloc(1, sizeof(struct instruction_group));
11757 new_g->group_id = n_groups;
11758 new_g->first_instr_id = i;
11759 new_g->last_instr_id = g->last_instr_id;
11761 g->last_instr_id = i - 1;
11763 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11767 /* The decision to split this group (to make the current instruction
11768 * the first instruction of a new group) is already taken and fully
11769 * implemented, so no need to search for more reasons to do it.
11775 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11776 * previously considered local (i.e. the jump destination is in the same group as
11777 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11778 * different group than the jump instruction). Wost case scenario: each instruction
11779 * that is a jump destination ends up as the first instruction of its group.
11785 /* Re-assign the group IDs to be in incremental order. */
11787 TAILQ_FOREACH(g, igl, node) {
11796 instruction_group_list_free(igl);
11804 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11805 uint32_t instr_pos,
11806 struct instruction *instr,
11810 "%s(p, t, &pipeline_instructions[%u]);\n"
11811 "\tthread_ip_reset(p, t);\n"
11812 "\tinstr_rx_exec(p);\n"
11814 instr_type_to_func(instr),
11819 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11820 struct instruction_group_list *igl,
11821 uint32_t jmp_instr_id,
11822 struct instruction *jmp_instr,
11823 struct instruction_data *jmp_data,
11826 struct instruction_group *jmp_g, *g;
11827 struct instruction_data *data;
11830 switch (jmp_instr->type) {
11834 case INSTR_JMP_VALID:
11836 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11840 case INSTR_JMP_INVALID:
11842 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11846 case INSTR_JMP_HIT:
11851 case INSTR_JMP_MISS:
11856 case INSTR_JMP_ACTION_HIT:
11858 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
11862 case INSTR_JMP_ACTION_MISS:
11864 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
11870 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11871 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11876 case INSTR_JMP_EQ_MH:
11878 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11879 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11884 case INSTR_JMP_EQ_HM:
11886 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11887 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11892 case INSTR_JMP_EQ_HH:
11894 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
11895 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11900 case INSTR_JMP_EQ_I:
11902 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
11903 "pipeline_instructions[%u].jmp.b_val)",
11908 case INSTR_JMP_NEQ:
11910 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11911 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11916 case INSTR_JMP_NEQ_MH:
11918 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11919 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11924 case INSTR_JMP_NEQ_HM:
11926 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11927 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11932 case INSTR_JMP_NEQ_HH:
11934 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
11935 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11940 case INSTR_JMP_NEQ_I:
11942 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
11943 "pipeline_instructions[%u].jmp.b_val)",
11950 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11951 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11956 case INSTR_JMP_LT_MH:
11958 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11959 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11964 case INSTR_JMP_LT_HM:
11966 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11967 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
11972 case INSTR_JMP_LT_HH:
11974 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11975 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
11980 case INSTR_JMP_LT_MI:
11982 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
11983 "pipeline_instructions[%u].jmp.b_val)",
11988 case INSTR_JMP_LT_HI:
11990 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
11991 "pipeline_instructions[%u].jmp.b_val)",
11998 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
11999 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12004 case INSTR_JMP_GT_MH:
12006 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12007 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12012 case INSTR_JMP_GT_HM:
12014 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12015 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12020 case INSTR_JMP_GT_HH:
12022 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12023 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12028 case INSTR_JMP_GT_MI:
12030 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12031 "pipeline_instructions[%u].jmp.b_val)",
12036 case INSTR_JMP_GT_HI:
12038 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12039 "pipeline_instructions[%u].jmp.b_val)",
12048 /* Find the instruction group of the jump instruction. */
12049 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12053 /* Find the instruction group of the jump destination instruction. */
12054 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12058 instr_id = data - p->instruction_data;
12060 g = instruction_group_list_group_find(igl, instr_id);
12064 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12065 * instruction group).
12067 if (g->group_id == jmp_g->group_id)
12069 "\n\t\tgoto %s;\n",
12070 jmp_data->jmp_label);
12074 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12083 instruction_group_list_codegen(struct instruction_group_list *igl,
12084 struct rte_swx_pipeline *p,
12087 struct instruction_group *g;
12089 int is_required = 0;
12091 /* Check if code generation is required. */
12092 TAILQ_FOREACH(g, igl, node)
12093 if (g->first_instr_id < g->last_instr_id)
12099 /* Generate the code for the pipeline instruction array. */
12101 "static const struct instruction pipeline_instructions[] = {\n");
12103 for (i = 0; i < p->n_instructions; i++) {
12104 struct instruction *instr = &p->instructions[i];
12105 instruction_export_t func = export_table[instr->type];
12110 fprintf(f, "};\n\n");
12112 /* Generate the code for the pipeline functions: one function for each instruction group
12113 * that contains more than one instruction.
12115 TAILQ_FOREACH(g, igl, node) {
12116 struct instruction *last_instr;
12119 /* Skip if group contains a single instruction. */
12120 if (g->last_instr_id == g->first_instr_id)
12123 /* Generate new pipeline function. */
12126 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12128 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12132 /* Generate the code for each pipeline instruction. */
12133 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12134 struct instruction *instr = &p->instructions[j];
12135 struct instruction_data *data = &p->instruction_data[j];
12137 /* Label, if present. */
12138 if (data->label[0])
12139 fprintf(f, "\n%s : ", data->label);
12141 fprintf(f, "\n\t");
12143 /* TX instruction type. */
12144 if (instruction_does_tx(instr)) {
12145 pipeline_instr_does_tx_codegen(p, j, instr, f);
12149 /* Jump instruction type. */
12150 if (instruction_is_jmp(instr)) {
12151 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12155 /* Any other instruction type. */
12157 "%s(p, t, &pipeline_instructions[%u]);\n",
12158 instr_type_to_func(instr),
12162 /* Finalize the generated pipeline function. For some instructions such as TX,
12163 * emit-many-and-TX and unconditional jump, the next instruction has been already
12164 * decided unconditionally and the instruction pointer of the current thread set
12165 * accordingly; for all the other instructions, the instruction pointer must be
12168 last_instr = &p->instructions[g->last_instr_id];
12170 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12172 "thread_ip_inc(p);\n");
12181 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12186 /* Create the .c file. */
12187 f = fopen("/tmp/pipeline.c", "w");
12191 /* Include the .h file. */
12192 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12194 /* Add the code for each action. */
12195 TAILQ_FOREACH(a, &p->actions, node) {
12196 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12198 action_data_codegen(a, f);
12202 action_instr_codegen(a, f);
12207 /* Add the pipeline code. */
12208 instruction_group_list_codegen(igl, p, f);
12210 /* Close the .c file. */
12216 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12217 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12221 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12224 struct instruction_group *g;
12225 char *dir_in, *buffer = NULL;
12226 const char *dir_out;
12229 /* Get the environment variables. */
12230 dir_in = getenv("RTE_INSTALL_DIR");
12238 /* Memory allocation for the command buffer. */
12239 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12246 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12247 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12248 "-I %s/lib/pipeline "
12249 "-I %s/lib/eal/include "
12250 "-I %s/lib/eal/x86/include "
12251 "-I %s/lib/eal/include/generic "
12255 "-I %s/lib/pipeline "
12258 "-I %s/lib/eal/linux/include "
12259 ">%s/pipeline.log 2>&1 "
12261 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12262 ">>%s/pipeline.log 2>&1",
12281 /* Build the shared object library. */
12282 status = system(buffer);
12286 /* Open library. */
12288 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12289 "%s/libpipeline.so",
12292 p->lib = dlopen(buffer, RTLD_LAZY);
12298 /* Get the action function symbols. */
12299 TAILQ_FOREACH(a, &p->actions, node) {
12300 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12302 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12303 if (!p->action_funcs[a->id]) {
12309 /* Get the pipeline function symbols. */
12310 TAILQ_FOREACH(g, igl, node) {
12311 if (g->first_instr_id == g->last_instr_id)
12314 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12316 g->func = dlsym(p->lib, buffer);
12324 if (status && p->lib) {
12335 pipeline_compile(struct rte_swx_pipeline *p)
12337 struct instruction_group_list *igl = NULL;
12340 igl = instruction_group_list_create(p);
12346 /* Code generation. */
12347 status = pipeline_codegen(p, igl);
12351 /* Build and load the shared object library. */
12352 status = pipeline_libload(p, igl);
12357 instruction_group_list_free(igl);