1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <rte_hash_crc.h>
12 #include <rte_swx_port_ethdev.h>
13 #include <rte_swx_port_fd.h>
14 #include <rte_swx_port_ring.h>
15 #include "rte_swx_port_source_sink.h"
17 #include <rte_swx_table_em.h>
18 #include <rte_swx_table_wm.h>
20 #include "rte_swx_pipeline_internal.h"
22 #define CHECK(condition, err_code) \
28 #define CHECK_NAME(name, err_code) \
31 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
34 #define CHECK_INSTRUCTION(instr, err_code) \
37 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
38 RTE_SWX_INSTRUCTION_SIZE), \
44 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
46 #include <rte_malloc.h>
49 env_malloc(size_t size, size_t alignment, int numa_node)
51 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
55 env_free(void *start, size_t size __rte_unused)
65 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
69 if (numa_available() == -1)
72 start = numa_alloc_onnode(size, numa_node);
76 memset(start, 0, size);
81 env_free(void *start, size_t size)
83 if (numa_available() == -1)
86 numa_free(start, size);
94 static struct struct_type *
95 struct_type_find(struct rte_swx_pipeline *p, const char *name)
97 struct struct_type *elem;
99 TAILQ_FOREACH(elem, &p->struct_types, node)
100 if (strcmp(elem->name, name) == 0)
106 static struct field *
107 struct_type_field_find(struct struct_type *st, const char *name)
111 for (i = 0; i < st->n_fields; i++) {
112 struct field *f = &st->fields[i];
114 if (strcmp(f->name, name) == 0)
122 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
124 struct rte_swx_field_params *fields,
126 int last_field_has_variable_size)
128 struct struct_type *st;
132 CHECK_NAME(name, EINVAL);
133 CHECK(fields, EINVAL);
134 CHECK(n_fields, EINVAL);
136 for (i = 0; i < n_fields; i++) {
137 struct rte_swx_field_params *f = &fields[i];
138 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
141 CHECK_NAME(f->name, EINVAL);
142 CHECK(f->n_bits, EINVAL);
143 CHECK((f->n_bits <= 64) || var_size, EINVAL);
144 CHECK((f->n_bits & 7) == 0, EINVAL);
146 for (j = 0; j < i; j++) {
147 struct rte_swx_field_params *f_prev = &fields[j];
149 CHECK(strcmp(f->name, f_prev->name), EINVAL);
153 CHECK(!struct_type_find(p, name), EEXIST);
155 /* Node allocation. */
156 st = calloc(1, sizeof(struct struct_type));
159 st->fields = calloc(n_fields, sizeof(struct field));
165 /* Node initialization. */
166 strcpy(st->name, name);
167 for (i = 0; i < n_fields; i++) {
168 struct field *dst = &st->fields[i];
169 struct rte_swx_field_params *src = &fields[i];
170 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
172 strcpy(dst->name, src->name);
173 dst->n_bits = src->n_bits;
174 dst->offset = st->n_bits;
175 dst->var_size = var_size;
177 st->n_bits += src->n_bits;
178 st->n_bits_min += var_size ? 0 : src->n_bits;
180 st->n_fields = n_fields;
181 st->var_size = last_field_has_variable_size;
183 /* Node add to tailq. */
184 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
190 struct_build(struct rte_swx_pipeline *p)
194 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
195 struct thread *t = &p->threads[i];
197 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
198 CHECK(t->structs, ENOMEM);
205 struct_build_free(struct rte_swx_pipeline *p)
209 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
210 struct thread *t = &p->threads[i];
218 struct_free(struct rte_swx_pipeline *p)
220 struct_build_free(p);
224 struct struct_type *elem;
226 elem = TAILQ_FIRST(&p->struct_types);
230 TAILQ_REMOVE(&p->struct_types, elem, node);
239 static struct port_in_type *
240 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
242 struct port_in_type *elem;
247 TAILQ_FOREACH(elem, &p->port_in_types, node)
248 if (strcmp(elem->name, name) == 0)
255 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
257 struct rte_swx_port_in_ops *ops)
259 struct port_in_type *elem;
262 CHECK_NAME(name, EINVAL);
264 CHECK(ops->create, EINVAL);
265 CHECK(ops->free, EINVAL);
266 CHECK(ops->pkt_rx, EINVAL);
267 CHECK(ops->stats_read, EINVAL);
269 CHECK(!port_in_type_find(p, name), EEXIST);
271 /* Node allocation. */
272 elem = calloc(1, sizeof(struct port_in_type));
275 /* Node initialization. */
276 strcpy(elem->name, name);
277 memcpy(&elem->ops, ops, sizeof(*ops));
279 /* Node add to tailq. */
280 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
285 static struct port_in *
286 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
288 struct port_in *port;
290 TAILQ_FOREACH(port, &p->ports_in, node)
291 if (port->id == port_id)
298 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
300 const char *port_type_name,
303 struct port_in_type *type = NULL;
304 struct port_in *port = NULL;
309 CHECK(!port_in_find(p, port_id), EINVAL);
311 CHECK_NAME(port_type_name, EINVAL);
312 type = port_in_type_find(p, port_type_name);
315 obj = type->ops.create(args);
318 /* Node allocation. */
319 port = calloc(1, sizeof(struct port_in));
322 /* Node initialization. */
327 /* Node add to tailq. */
328 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
329 if (p->n_ports_in < port_id + 1)
330 p->n_ports_in = port_id + 1;
336 port_in_build(struct rte_swx_pipeline *p)
338 struct port_in *port;
341 CHECK(p->n_ports_in, EINVAL);
342 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
344 for (i = 0; i < p->n_ports_in; i++)
345 CHECK(port_in_find(p, i), EINVAL);
347 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
348 CHECK(p->in, ENOMEM);
350 TAILQ_FOREACH(port, &p->ports_in, node) {
351 struct port_in_runtime *in = &p->in[port->id];
353 in->pkt_rx = port->type->ops.pkt_rx;
361 port_in_build_free(struct rte_swx_pipeline *p)
368 port_in_free(struct rte_swx_pipeline *p)
370 port_in_build_free(p);
374 struct port_in *port;
376 port = TAILQ_FIRST(&p->ports_in);
380 TAILQ_REMOVE(&p->ports_in, port, node);
381 port->type->ops.free(port->obj);
385 /* Input port types. */
387 struct port_in_type *elem;
389 elem = TAILQ_FIRST(&p->port_in_types);
393 TAILQ_REMOVE(&p->port_in_types, elem, node);
401 static struct port_out_type *
402 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
404 struct port_out_type *elem;
409 TAILQ_FOREACH(elem, &p->port_out_types, node)
410 if (!strcmp(elem->name, name))
417 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
419 struct rte_swx_port_out_ops *ops)
421 struct port_out_type *elem;
424 CHECK_NAME(name, EINVAL);
426 CHECK(ops->create, EINVAL);
427 CHECK(ops->free, EINVAL);
428 CHECK(ops->pkt_tx, EINVAL);
429 CHECK(ops->pkt_fast_clone_tx, EINVAL);
430 CHECK(ops->pkt_clone_tx, EINVAL);
431 CHECK(ops->stats_read, EINVAL);
433 CHECK(!port_out_type_find(p, name), EEXIST);
435 /* Node allocation. */
436 elem = calloc(1, sizeof(struct port_out_type));
439 /* Node initialization. */
440 strcpy(elem->name, name);
441 memcpy(&elem->ops, ops, sizeof(*ops));
443 /* Node add to tailq. */
444 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
449 static struct port_out *
450 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
452 struct port_out *port;
454 TAILQ_FOREACH(port, &p->ports_out, node)
455 if (port->id == port_id)
462 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
464 const char *port_type_name,
467 struct port_out_type *type = NULL;
468 struct port_out *port = NULL;
473 CHECK(!port_out_find(p, port_id), EINVAL);
475 CHECK_NAME(port_type_name, EINVAL);
476 type = port_out_type_find(p, port_type_name);
479 obj = type->ops.create(args);
482 /* Node allocation. */
483 port = calloc(1, sizeof(struct port_out));
486 /* Node initialization. */
491 /* Node add to tailq. */
492 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
493 if (p->n_ports_out < port_id + 1)
494 p->n_ports_out = port_id + 1;
500 port_out_build(struct rte_swx_pipeline *p)
502 struct port_out *port;
505 CHECK(p->n_ports_out, EINVAL);
507 for (i = 0; i < p->n_ports_out; i++)
508 CHECK(port_out_find(p, i), EINVAL);
510 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
511 CHECK(p->out, ENOMEM);
513 TAILQ_FOREACH(port, &p->ports_out, node) {
514 struct port_out_runtime *out = &p->out[port->id];
516 out->pkt_tx = port->type->ops.pkt_tx;
517 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
518 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
519 out->flush = port->type->ops.flush;
520 out->obj = port->obj;
527 port_out_build_free(struct rte_swx_pipeline *p)
534 port_out_free(struct rte_swx_pipeline *p)
536 port_out_build_free(p);
540 struct port_out *port;
542 port = TAILQ_FIRST(&p->ports_out);
546 TAILQ_REMOVE(&p->ports_out, port, node);
547 port->type->ops.free(port->obj);
551 /* Output port types. */
553 struct port_out_type *elem;
555 elem = TAILQ_FIRST(&p->port_out_types);
559 TAILQ_REMOVE(&p->port_out_types, elem, node);
568 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
569 struct rte_swx_pipeline_mirroring_params *params)
572 CHECK(params, EINVAL);
573 CHECK(params->n_slots, EINVAL);
574 CHECK(params->n_sessions, EINVAL);
575 CHECK(!p->build_done, EEXIST);
577 p->n_mirroring_slots = rte_align32pow2(params->n_slots);
578 if (p->n_mirroring_slots > 64)
579 p->n_mirroring_slots = 64;
581 p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
587 mirroring_build_free(struct rte_swx_pipeline *p)
591 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
592 struct thread *t = &p->threads[i];
594 /* mirroring_slots. */
595 free(t->mirroring_slots);
596 t->mirroring_slots = NULL;
599 /* mirroring_sessions. */
600 free(p->mirroring_sessions);
601 p->mirroring_sessions = NULL;
605 mirroring_free(struct rte_swx_pipeline *p)
607 mirroring_build_free(p);
611 mirroring_build(struct rte_swx_pipeline *p)
615 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
616 struct thread *t = &p->threads[i];
618 /* mirroring_slots. */
619 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
620 if (!t->mirroring_slots)
624 /* mirroring_sessions. */
625 p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
626 if (!p->mirroring_sessions)
632 mirroring_build_free(p);
639 static struct extern_type *
640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
642 struct extern_type *elem;
644 TAILQ_FOREACH(elem, &p->extern_types, node)
645 if (strcmp(elem->name, name) == 0)
651 static struct extern_type_member_func *
652 extern_type_member_func_find(struct extern_type *type, const char *name)
654 struct extern_type_member_func *elem;
656 TAILQ_FOREACH(elem, &type->funcs, node)
657 if (strcmp(elem->name, name) == 0)
663 static struct extern_obj *
664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
666 struct extern_obj *elem;
668 TAILQ_FOREACH(elem, &p->extern_objs, node)
669 if (strcmp(elem->name, name) == 0)
675 static struct extern_type_member_func *
676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
678 struct extern_obj **obj)
680 struct extern_obj *object;
681 struct extern_type_member_func *func;
682 char *object_name, *func_name;
684 if (name[0] != 'e' || name[1] != '.')
687 object_name = strdup(&name[2]);
691 func_name = strchr(object_name, '.');
700 object = extern_obj_find(p, object_name);
706 func = extern_type_member_func_find(object->type, func_name);
719 static struct field *
720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
722 struct extern_obj **object)
724 struct extern_obj *obj;
726 char *obj_name, *field_name;
728 if ((name[0] != 'e') || (name[1] != '.'))
731 obj_name = strdup(&name[2]);
735 field_name = strchr(obj_name, '.');
744 obj = extern_obj_find(p, obj_name);
750 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
766 const char *mailbox_struct_type_name,
767 rte_swx_extern_type_constructor_t constructor,
768 rte_swx_extern_type_destructor_t destructor)
770 struct extern_type *elem;
771 struct struct_type *mailbox_struct_type;
775 CHECK_NAME(name, EINVAL);
776 CHECK(!extern_type_find(p, name), EEXIST);
778 CHECK_NAME(mailbox_struct_type_name, EINVAL);
779 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
780 CHECK(mailbox_struct_type, EINVAL);
781 CHECK(!mailbox_struct_type->var_size, EINVAL);
783 CHECK(constructor, EINVAL);
784 CHECK(destructor, EINVAL);
786 /* Node allocation. */
787 elem = calloc(1, sizeof(struct extern_type));
790 /* Node initialization. */
791 strcpy(elem->name, name);
792 elem->mailbox_struct_type = mailbox_struct_type;
793 elem->constructor = constructor;
794 elem->destructor = destructor;
795 TAILQ_INIT(&elem->funcs);
797 /* Node add to tailq. */
798 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
805 const char *extern_type_name,
807 rte_swx_extern_type_member_func_t member_func)
809 struct extern_type *type;
810 struct extern_type_member_func *type_member;
814 CHECK_NAME(extern_type_name, EINVAL);
815 type = extern_type_find(p, extern_type_name);
817 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
819 CHECK_NAME(name, EINVAL);
820 CHECK(!extern_type_member_func_find(type, name), EEXIST);
822 CHECK(member_func, EINVAL);
824 /* Node allocation. */
825 type_member = calloc(1, sizeof(struct extern_type_member_func));
826 CHECK(type_member, ENOMEM);
828 /* Node initialization. */
829 strcpy(type_member->name, name);
830 type_member->func = member_func;
831 type_member->id = type->n_funcs;
833 /* Node add to tailq. */
834 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
842 const char *extern_type_name,
846 struct extern_type *type;
847 struct extern_obj *obj;
852 CHECK_NAME(extern_type_name, EINVAL);
853 type = extern_type_find(p, extern_type_name);
856 CHECK_NAME(name, EINVAL);
857 CHECK(!extern_obj_find(p, name), EEXIST);
859 /* Node allocation. */
860 obj = calloc(1, sizeof(struct extern_obj));
863 /* Object construction. */
864 obj_handle = type->constructor(args);
870 /* Node initialization. */
871 strcpy(obj->name, name);
873 obj->obj = obj_handle;
874 obj->struct_id = p->n_structs;
875 obj->id = p->n_extern_objs;
877 /* Node add to tailq. */
878 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
886 extern_obj_build(struct rte_swx_pipeline *p)
890 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
891 struct thread *t = &p->threads[i];
892 struct extern_obj *obj;
894 t->extern_objs = calloc(p->n_extern_objs,
895 sizeof(struct extern_obj_runtime));
896 CHECK(t->extern_objs, ENOMEM);
898 TAILQ_FOREACH(obj, &p->extern_objs, node) {
899 struct extern_obj_runtime *r =
900 &t->extern_objs[obj->id];
901 struct extern_type_member_func *func;
902 uint32_t mailbox_size =
903 obj->type->mailbox_struct_type->n_bits / 8;
907 r->mailbox = calloc(1, mailbox_size);
908 CHECK(r->mailbox, ENOMEM);
910 TAILQ_FOREACH(func, &obj->type->funcs, node)
911 r->funcs[func->id] = func->func;
913 t->structs[obj->struct_id] = r->mailbox;
921 extern_obj_build_free(struct rte_swx_pipeline *p)
925 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
926 struct thread *t = &p->threads[i];
932 for (j = 0; j < p->n_extern_objs; j++) {
933 struct extern_obj_runtime *r = &t->extern_objs[j];
938 free(t->extern_objs);
939 t->extern_objs = NULL;
944 extern_obj_free(struct rte_swx_pipeline *p)
946 extern_obj_build_free(p);
948 /* Extern objects. */
950 struct extern_obj *elem;
952 elem = TAILQ_FIRST(&p->extern_objs);
956 TAILQ_REMOVE(&p->extern_objs, elem, node);
958 elem->type->destructor(elem->obj);
964 struct extern_type *elem;
966 elem = TAILQ_FIRST(&p->extern_types);
970 TAILQ_REMOVE(&p->extern_types, elem, node);
973 struct extern_type_member_func *func;
975 func = TAILQ_FIRST(&elem->funcs);
979 TAILQ_REMOVE(&elem->funcs, func, node);
990 static struct extern_func *
991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
993 struct extern_func *elem;
995 TAILQ_FOREACH(elem, &p->extern_funcs, node)
996 if (strcmp(elem->name, name) == 0)
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1006 if (name[0] != 'f' || name[1] != '.')
1009 return extern_func_find(p, &name[2]);
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1015 struct extern_func **function)
1017 struct extern_func *func;
1019 char *func_name, *field_name;
1021 if ((name[0] != 'f') || (name[1] != '.'))
1024 func_name = strdup(&name[2]);
1028 field_name = strchr(func_name, '.');
1037 func = extern_func_find(p, func_name);
1043 f = struct_type_field_find(func->mailbox_struct_type, field_name);
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1059 const char *mailbox_struct_type_name,
1060 rte_swx_extern_func_t func)
1062 struct extern_func *f;
1063 struct struct_type *mailbox_struct_type;
1067 CHECK_NAME(name, EINVAL);
1068 CHECK(!extern_func_find(p, name), EEXIST);
1070 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072 CHECK(mailbox_struct_type, EINVAL);
1073 CHECK(!mailbox_struct_type->var_size, EINVAL);
1075 CHECK(func, EINVAL);
1077 /* Node allocation. */
1078 f = calloc(1, sizeof(struct extern_func));
1079 CHECK(func, ENOMEM);
1081 /* Node initialization. */
1082 strcpy(f->name, name);
1083 f->mailbox_struct_type = mailbox_struct_type;
1085 f->struct_id = p->n_structs;
1086 f->id = p->n_extern_funcs;
1088 /* Node add to tailq. */
1089 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090 p->n_extern_funcs++;
1097 extern_func_build(struct rte_swx_pipeline *p)
1101 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102 struct thread *t = &p->threads[i];
1103 struct extern_func *func;
1105 /* Memory allocation. */
1106 t->extern_funcs = calloc(p->n_extern_funcs,
1107 sizeof(struct extern_func_runtime));
1108 CHECK(t->extern_funcs, ENOMEM);
1110 /* Extern function. */
1111 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112 struct extern_func_runtime *r =
1113 &t->extern_funcs[func->id];
1114 uint32_t mailbox_size =
1115 func->mailbox_struct_type->n_bits / 8;
1117 r->func = func->func;
1119 r->mailbox = calloc(1, mailbox_size);
1120 CHECK(r->mailbox, ENOMEM);
1122 t->structs[func->struct_id] = r->mailbox;
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1134 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135 struct thread *t = &p->threads[i];
1138 if (!t->extern_funcs)
1141 for (j = 0; j < p->n_extern_funcs; j++) {
1142 struct extern_func_runtime *r = &t->extern_funcs[j];
1147 free(t->extern_funcs);
1148 t->extern_funcs = NULL;
1153 extern_func_free(struct rte_swx_pipeline *p)
1155 extern_func_build_free(p);
1158 struct extern_func *elem;
1160 elem = TAILQ_FIRST(&p->extern_funcs);
1164 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1172 static struct hash_func *
1173 hash_func_find(struct rte_swx_pipeline *p, const char *name)
1175 struct hash_func *elem;
1177 TAILQ_FOREACH(elem, &p->hash_funcs, node)
1178 if (strcmp(elem->name, name) == 0)
1185 rte_swx_pipeline_hash_func_register(struct rte_swx_pipeline *p,
1187 rte_swx_hash_func_t func)
1189 struct hash_func *f;
1193 CHECK_NAME(name, EINVAL);
1194 CHECK(!hash_func_find(p, name), EEXIST);
1196 CHECK(func, EINVAL);
1198 /* Node allocation. */
1199 f = calloc(1, sizeof(struct hash_func));
1200 CHECK(func, ENOMEM);
1202 /* Node initialization. */
1203 strcpy(f->name, name);
1205 f->id = p->n_hash_funcs;
1207 /* Node add to tailq. */
1208 TAILQ_INSERT_TAIL(&p->hash_funcs, f, node);
1215 hash_func_build(struct rte_swx_pipeline *p)
1217 struct hash_func *func;
1219 /* Memory allocation. */
1220 p->hash_func_runtime = calloc(p->n_hash_funcs, sizeof(struct hash_func_runtime));
1221 CHECK(p->hash_func_runtime, ENOMEM);
1223 /* Hash function. */
1224 TAILQ_FOREACH(func, &p->hash_funcs, node) {
1225 struct hash_func_runtime *r = &p->hash_func_runtime[func->id];
1227 r->func = func->func;
1234 hash_func_build_free(struct rte_swx_pipeline *p)
1236 free(p->hash_func_runtime);
1237 p->hash_func_runtime = NULL;
1241 hash_func_free(struct rte_swx_pipeline *p)
1243 hash_func_build_free(p);
1246 struct hash_func *elem;
1248 elem = TAILQ_FIRST(&p->hash_funcs);
1252 TAILQ_REMOVE(&p->hash_funcs, elem, node);
1260 static struct header *
1261 header_find(struct rte_swx_pipeline *p, const char *name)
1263 struct header *elem;
1265 TAILQ_FOREACH(elem, &p->headers, node)
1266 if (strcmp(elem->name, name) == 0)
1272 static struct header *
1273 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1275 struct header *elem;
1277 TAILQ_FOREACH(elem, &p->headers, node)
1278 if (elem->struct_id == struct_id)
1284 static struct header *
1285 header_parse(struct rte_swx_pipeline *p,
1288 if (name[0] != 'h' || name[1] != '.')
1291 return header_find(p, &name[2]);
1294 static struct field *
1295 header_field_parse(struct rte_swx_pipeline *p,
1297 struct header **header)
1301 char *header_name, *field_name;
1303 if ((name[0] != 'h') || (name[1] != '.'))
1306 header_name = strdup(&name[2]);
1310 field_name = strchr(header_name, '.');
1319 h = header_find(p, header_name);
1325 f = struct_type_field_find(h->st, field_name);
1339 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1341 const char *struct_type_name)
1343 struct struct_type *st;
1345 size_t n_headers_max;
1348 CHECK_NAME(name, EINVAL);
1349 CHECK_NAME(struct_type_name, EINVAL);
1351 CHECK(!header_find(p, name), EEXIST);
1353 st = struct_type_find(p, struct_type_name);
1356 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1357 CHECK(p->n_headers < n_headers_max, ENOSPC);
1359 /* Node allocation. */
1360 h = calloc(1, sizeof(struct header));
1363 /* Node initialization. */
1364 strcpy(h->name, name);
1366 h->struct_id = p->n_structs;
1367 h->id = p->n_headers;
1369 /* Node add to tailq. */
1370 TAILQ_INSERT_TAIL(&p->headers, h, node);
1378 header_build(struct rte_swx_pipeline *p)
1381 uint32_t n_bytes = 0, i;
1383 TAILQ_FOREACH(h, &p->headers, node) {
1384 n_bytes += h->st->n_bits / 8;
1387 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1388 struct thread *t = &p->threads[i];
1389 uint32_t offset = 0;
1391 t->headers = calloc(p->n_headers,
1392 sizeof(struct header_runtime));
1393 CHECK(t->headers, ENOMEM);
1395 t->headers_out = calloc(p->n_headers,
1396 sizeof(struct header_out_runtime));
1397 CHECK(t->headers_out, ENOMEM);
1399 t->header_storage = calloc(1, n_bytes);
1400 CHECK(t->header_storage, ENOMEM);
1402 t->header_out_storage = calloc(1, n_bytes);
1403 CHECK(t->header_out_storage, ENOMEM);
1405 TAILQ_FOREACH(h, &p->headers, node) {
1406 uint8_t *header_storage;
1407 uint32_t n_bytes = h->st->n_bits / 8;
1409 header_storage = &t->header_storage[offset];
1412 t->headers[h->id].ptr0 = header_storage;
1413 t->headers[h->id].n_bytes = n_bytes;
1415 t->structs[h->struct_id] = header_storage;
1423 header_build_free(struct rte_swx_pipeline *p)
1427 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1428 struct thread *t = &p->threads[i];
1430 free(t->headers_out);
1431 t->headers_out = NULL;
1436 free(t->header_out_storage);
1437 t->header_out_storage = NULL;
1439 free(t->header_storage);
1440 t->header_storage = NULL;
1445 header_free(struct rte_swx_pipeline *p)
1447 header_build_free(p);
1450 struct header *elem;
1452 elem = TAILQ_FIRST(&p->headers);
1456 TAILQ_REMOVE(&p->headers, elem, node);
1464 static struct field *
1465 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1467 if (!p->metadata_st)
1470 if (name[0] != 'm' || name[1] != '.')
1473 return struct_type_field_find(p->metadata_st, &name[2]);
1477 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1478 const char *struct_type_name)
1480 struct struct_type *st = NULL;
1484 CHECK_NAME(struct_type_name, EINVAL);
1485 st = struct_type_find(p, struct_type_name);
1487 CHECK(!st->var_size, EINVAL);
1488 CHECK(!p->metadata_st, EINVAL);
1490 p->metadata_st = st;
1491 p->metadata_struct_id = p->n_structs;
1499 metadata_build(struct rte_swx_pipeline *p)
1501 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1504 /* Thread-level initialization. */
1505 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1506 struct thread *t = &p->threads[i];
1509 metadata = calloc(1, n_bytes);
1510 CHECK(metadata, ENOMEM);
1512 t->metadata = metadata;
1513 t->structs[p->metadata_struct_id] = metadata;
1520 metadata_build_free(struct rte_swx_pipeline *p)
1524 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1525 struct thread *t = &p->threads[i];
1533 metadata_free(struct rte_swx_pipeline *p)
1535 metadata_build_free(p);
1542 instruction_is_tx(enum instruction_type type)
1556 instruction_does_tx(struct instruction *instr)
1558 switch (instr->type) {
1562 case INSTR_HDR_EMIT_TX:
1563 case INSTR_HDR_EMIT2_TX:
1564 case INSTR_HDR_EMIT3_TX:
1565 case INSTR_HDR_EMIT4_TX:
1566 case INSTR_HDR_EMIT5_TX:
1567 case INSTR_HDR_EMIT6_TX:
1568 case INSTR_HDR_EMIT7_TX:
1569 case INSTR_HDR_EMIT8_TX:
1577 instruction_is_jmp(struct instruction *instr)
1579 switch (instr->type) {
1581 case INSTR_JMP_VALID:
1582 case INSTR_JMP_INVALID:
1584 case INSTR_JMP_MISS:
1585 case INSTR_JMP_ACTION_HIT:
1586 case INSTR_JMP_ACTION_MISS:
1588 case INSTR_JMP_EQ_MH:
1589 case INSTR_JMP_EQ_HM:
1590 case INSTR_JMP_EQ_HH:
1591 case INSTR_JMP_EQ_I:
1593 case INSTR_JMP_NEQ_MH:
1594 case INSTR_JMP_NEQ_HM:
1595 case INSTR_JMP_NEQ_HH:
1596 case INSTR_JMP_NEQ_I:
1598 case INSTR_JMP_LT_MH:
1599 case INSTR_JMP_LT_HM:
1600 case INSTR_JMP_LT_HH:
1601 case INSTR_JMP_LT_MI:
1602 case INSTR_JMP_LT_HI:
1604 case INSTR_JMP_GT_MH:
1605 case INSTR_JMP_GT_HM:
1606 case INSTR_JMP_GT_HH:
1607 case INSTR_JMP_GT_MI:
1608 case INSTR_JMP_GT_HI:
1617 instruction_does_thread_yield(struct instruction *instr)
1619 switch (instr->type) {
1622 case INSTR_TABLE_AF:
1623 case INSTR_SELECTOR:
1625 case INSTR_LEARNER_AF:
1626 case INSTR_EXTERN_OBJ:
1627 case INSTR_EXTERN_FUNC:
1634 static struct field *
1635 action_field_parse(struct action *action, const char *name);
1637 static struct field *
1638 struct_field_parse(struct rte_swx_pipeline *p,
1639 struct action *action,
1641 uint32_t *struct_id)
1648 struct header *header;
1650 f = header_field_parse(p, name, &header);
1654 *struct_id = header->struct_id;
1660 f = metadata_field_parse(p, name);
1664 *struct_id = p->metadata_struct_id;
1673 f = action_field_parse(action, name);
1683 struct extern_obj *obj;
1685 f = extern_obj_mailbox_field_parse(p, name, &obj);
1689 *struct_id = obj->struct_id;
1695 struct extern_func *func;
1697 f = extern_func_mailbox_field_parse(p, name, &func);
1701 *struct_id = func->struct_id;
1714 instr_rx_translate(struct rte_swx_pipeline *p,
1715 struct action *action,
1718 struct instruction *instr,
1719 struct instruction_data *data __rte_unused)
1723 CHECK(!action, EINVAL);
1724 CHECK(n_tokens == 2, EINVAL);
1726 f = metadata_field_parse(p, tokens[1]);
1729 instr->type = INSTR_RX;
1730 instr->io.io.offset = f->offset / 8;
1731 instr->io.io.n_bits = f->n_bits;
1739 instr_tx_translate(struct rte_swx_pipeline *p,
1740 struct action *action __rte_unused,
1743 struct instruction *instr,
1744 struct instruction_data *data __rte_unused)
1746 char *port = tokens[1];
1750 CHECK(n_tokens == 2, EINVAL);
1752 f = metadata_field_parse(p, port);
1754 instr->type = INSTR_TX;
1755 instr->io.io.offset = f->offset / 8;
1756 instr->io.io.n_bits = f->n_bits;
1761 port_val = strtoul(port, &port, 0);
1762 CHECK(!port[0], EINVAL);
1764 instr->type = INSTR_TX_I;
1765 instr->io.io.val = port_val;
1770 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1771 struct action *action __rte_unused,
1772 char **tokens __rte_unused,
1774 struct instruction *instr,
1775 struct instruction_data *data __rte_unused)
1777 CHECK(n_tokens == 1, EINVAL);
1780 instr->type = INSTR_DROP;
1785 instr_tx_exec(struct rte_swx_pipeline *p)
1787 struct thread *t = &p->threads[p->thread_id];
1788 struct instruction *ip = t->ip;
1790 __instr_tx_exec(p, t, ip);
1793 thread_ip_reset(p, t);
1798 instr_tx_i_exec(struct rte_swx_pipeline *p)
1800 struct thread *t = &p->threads[p->thread_id];
1801 struct instruction *ip = t->ip;
1803 __instr_tx_i_exec(p, t, ip);
1806 thread_ip_reset(p, t);
1811 instr_drop_exec(struct rte_swx_pipeline *p)
1813 struct thread *t = &p->threads[p->thread_id];
1814 struct instruction *ip = t->ip;
1816 __instr_drop_exec(p, t, ip);
1819 thread_ip_reset(p, t);
1827 instr_mirror_translate(struct rte_swx_pipeline *p,
1828 struct action *action,
1831 struct instruction *instr,
1832 struct instruction_data *data __rte_unused)
1834 char *dst = tokens[1], *src = tokens[2];
1835 struct field *fdst, *fsrc;
1836 uint32_t dst_struct_id = 0, src_struct_id = 0;
1838 CHECK(n_tokens == 3, EINVAL);
1840 fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1841 CHECK(fdst, EINVAL);
1842 CHECK(dst[0] != 'h', EINVAL);
1843 CHECK(!fdst->var_size, EINVAL);
1845 fsrc = struct_field_parse(p, action, src, &src_struct_id);
1846 CHECK(fsrc, EINVAL);
1847 CHECK(src[0] != 'h', EINVAL);
1848 CHECK(!fsrc->var_size, EINVAL);
1850 instr->type = INSTR_MIRROR;
1851 instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1852 instr->mirror.dst.n_bits = fdst->n_bits;
1853 instr->mirror.dst.offset = fdst->offset / 8;
1854 instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1855 instr->mirror.src.n_bits = fsrc->n_bits;
1856 instr->mirror.src.offset = fsrc->offset / 8;
1862 instr_mirror_exec(struct rte_swx_pipeline *p)
1864 struct thread *t = &p->threads[p->thread_id];
1865 struct instruction *ip = t->ip;
1867 __instr_mirror_exec(p, t, ip);
1877 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1878 struct action *action __rte_unused,
1879 char **tokens __rte_unused,
1881 struct instruction *instr,
1882 struct instruction_data *data __rte_unused)
1884 CHECK(n_tokens == 1, EINVAL);
1886 instr->type = INSTR_RECIRCULATE;
1891 instr_recircid_translate(struct rte_swx_pipeline *p,
1892 struct action *action __rte_unused,
1895 struct instruction *instr,
1896 struct instruction_data *data __rte_unused)
1900 CHECK(n_tokens == 2, EINVAL);
1902 f = metadata_field_parse(p, tokens[1]);
1905 instr->type = INSTR_RECIRCID;
1906 instr->io.io.offset = f->offset / 8;
1907 instr->io.io.n_bits = f->n_bits;
1912 instr_recirculate_exec(struct rte_swx_pipeline *p)
1914 struct thread *t = &p->threads[p->thread_id];
1915 struct instruction *ip = t->ip;
1917 __instr_recirculate_exec(p, t, ip);
1924 instr_recircid_exec(struct rte_swx_pipeline *p)
1926 struct thread *t = &p->threads[p->thread_id];
1927 struct instruction *ip = t->ip;
1929 __instr_recircid_exec(p, t, ip);
1939 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1940 struct action *action,
1943 struct instruction *instr,
1944 struct instruction_data *data __rte_unused)
1948 CHECK(!action, EINVAL);
1949 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1951 h = header_parse(p, tokens[1]);
1954 if (n_tokens == 2) {
1955 CHECK(!h->st->var_size, EINVAL);
1957 instr->type = INSTR_HDR_EXTRACT;
1958 instr->io.hdr.header_id[0] = h->id;
1959 instr->io.hdr.struct_id[0] = h->struct_id;
1960 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1964 CHECK(h->st->var_size, EINVAL);
1966 mf = metadata_field_parse(p, tokens[2]);
1968 CHECK(!mf->var_size, EINVAL);
1970 instr->type = INSTR_HDR_EXTRACT_M;
1971 instr->io.io.offset = mf->offset / 8;
1972 instr->io.io.n_bits = mf->n_bits;
1973 instr->io.hdr.header_id[0] = h->id;
1974 instr->io.hdr.struct_id[0] = h->struct_id;
1975 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1982 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1983 struct action *action,
1986 struct instruction *instr,
1987 struct instruction_data *data __rte_unused)
1991 CHECK(!action, EINVAL);
1992 CHECK(n_tokens == 2, EINVAL);
1994 h = header_parse(p, tokens[1]);
1996 CHECK(!h->st->var_size, EINVAL);
1998 instr->type = INSTR_HDR_LOOKAHEAD;
1999 instr->io.hdr.header_id[0] = h->id;
2000 instr->io.hdr.struct_id[0] = h->struct_id;
2001 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
2007 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2009 struct thread *t = &p->threads[p->thread_id];
2010 struct instruction *ip = t->ip;
2012 __instr_hdr_extract_exec(p, t, ip);
2019 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2021 struct thread *t = &p->threads[p->thread_id];
2022 struct instruction *ip = t->ip;
2024 __instr_hdr_extract2_exec(p, t, ip);
2031 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2033 struct thread *t = &p->threads[p->thread_id];
2034 struct instruction *ip = t->ip;
2036 __instr_hdr_extract3_exec(p, t, ip);
2043 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2045 struct thread *t = &p->threads[p->thread_id];
2046 struct instruction *ip = t->ip;
2048 __instr_hdr_extract4_exec(p, t, ip);
2055 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2057 struct thread *t = &p->threads[p->thread_id];
2058 struct instruction *ip = t->ip;
2060 __instr_hdr_extract5_exec(p, t, ip);
2067 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2069 struct thread *t = &p->threads[p->thread_id];
2070 struct instruction *ip = t->ip;
2072 __instr_hdr_extract6_exec(p, t, ip);
2079 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2081 struct thread *t = &p->threads[p->thread_id];
2082 struct instruction *ip = t->ip;
2084 __instr_hdr_extract7_exec(p, t, ip);
2091 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2093 struct thread *t = &p->threads[p->thread_id];
2094 struct instruction *ip = t->ip;
2096 __instr_hdr_extract8_exec(p, t, ip);
2103 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2105 struct thread *t = &p->threads[p->thread_id];
2106 struct instruction *ip = t->ip;
2108 __instr_hdr_extract_m_exec(p, t, ip);
2115 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2117 struct thread *t = &p->threads[p->thread_id];
2118 struct instruction *ip = t->ip;
2120 __instr_hdr_lookahead_exec(p, t, ip);
2130 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2131 struct action *action __rte_unused,
2134 struct instruction *instr,
2135 struct instruction_data *data __rte_unused)
2139 CHECK(n_tokens == 2, EINVAL);
2141 h = header_parse(p, tokens[1]);
2144 instr->type = INSTR_HDR_EMIT;
2145 instr->io.hdr.header_id[0] = h->id;
2146 instr->io.hdr.struct_id[0] = h->struct_id;
2147 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2152 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2154 struct thread *t = &p->threads[p->thread_id];
2155 struct instruction *ip = t->ip;
2157 __instr_hdr_emit_exec(p, t, ip);
2164 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2166 struct thread *t = &p->threads[p->thread_id];
2167 struct instruction *ip = t->ip;
2169 __instr_hdr_emit_tx_exec(p, t, ip);
2172 thread_ip_reset(p, t);
2177 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2179 struct thread *t = &p->threads[p->thread_id];
2180 struct instruction *ip = t->ip;
2182 __instr_hdr_emit2_tx_exec(p, t, ip);
2185 thread_ip_reset(p, t);
2190 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2192 struct thread *t = &p->threads[p->thread_id];
2193 struct instruction *ip = t->ip;
2195 __instr_hdr_emit3_tx_exec(p, t, ip);
2198 thread_ip_reset(p, t);
2203 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2205 struct thread *t = &p->threads[p->thread_id];
2206 struct instruction *ip = t->ip;
2208 __instr_hdr_emit4_tx_exec(p, t, ip);
2211 thread_ip_reset(p, t);
2216 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2218 struct thread *t = &p->threads[p->thread_id];
2219 struct instruction *ip = t->ip;
2221 __instr_hdr_emit5_tx_exec(p, t, ip);
2224 thread_ip_reset(p, t);
2229 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2231 struct thread *t = &p->threads[p->thread_id];
2232 struct instruction *ip = t->ip;
2234 __instr_hdr_emit6_tx_exec(p, t, ip);
2237 thread_ip_reset(p, t);
2242 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2244 struct thread *t = &p->threads[p->thread_id];
2245 struct instruction *ip = t->ip;
2247 __instr_hdr_emit7_tx_exec(p, t, ip);
2250 thread_ip_reset(p, t);
2255 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2257 struct thread *t = &p->threads[p->thread_id];
2258 struct instruction *ip = t->ip;
2260 __instr_hdr_emit8_tx_exec(p, t, ip);
2263 thread_ip_reset(p, t);
2271 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2272 struct action *action __rte_unused,
2275 struct instruction *instr,
2276 struct instruction_data *data __rte_unused)
2280 CHECK(n_tokens == 2, EINVAL);
2282 h = header_parse(p, tokens[1]);
2285 instr->type = INSTR_HDR_VALIDATE;
2286 instr->valid.header_id = h->id;
2287 instr->valid.struct_id = h->struct_id;
2292 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2294 struct thread *t = &p->threads[p->thread_id];
2295 struct instruction *ip = t->ip;
2297 __instr_hdr_validate_exec(p, t, ip);
2307 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2308 struct action *action __rte_unused,
2311 struct instruction *instr,
2312 struct instruction_data *data __rte_unused)
2316 CHECK(n_tokens == 2, EINVAL);
2318 h = header_parse(p, tokens[1]);
2321 instr->type = INSTR_HDR_INVALIDATE;
2322 instr->valid.header_id = h->id;
2327 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2329 struct thread *t = &p->threads[p->thread_id];
2330 struct instruction *ip = t->ip;
2332 __instr_hdr_invalidate_exec(p, t, ip);
2341 static struct table *
2342 table_find(struct rte_swx_pipeline *p, const char *name);
2344 static struct selector *
2345 selector_find(struct rte_swx_pipeline *p, const char *name);
2347 static struct learner *
2348 learner_find(struct rte_swx_pipeline *p, const char *name);
2351 instr_table_translate(struct rte_swx_pipeline *p,
2352 struct action *action,
2355 struct instruction *instr,
2356 struct instruction_data *data __rte_unused)
2362 CHECK(!action, EINVAL);
2363 CHECK(n_tokens == 2, EINVAL);
2365 t = table_find(p, tokens[1]);
2367 instr->type = INSTR_TABLE;
2368 instr->table.table_id = t->id;
2372 s = selector_find(p, tokens[1]);
2374 instr->type = INSTR_SELECTOR;
2375 instr->table.table_id = s->id;
2379 l = learner_find(p, tokens[1]);
2381 instr->type = INSTR_LEARNER;
2382 instr->table.table_id = l->id;
2390 instr_table_exec(struct rte_swx_pipeline *p)
2392 struct thread *t = &p->threads[p->thread_id];
2393 struct instruction *ip = t->ip;
2394 uint32_t table_id = ip->table.table_id;
2395 struct rte_swx_table_state *ts = &t->table_state[table_id];
2396 struct table_runtime *table = &t->tables[table_id];
2397 struct table_statistics *stats = &p->table_stats[table_id];
2398 uint64_t action_id, n_pkts_hit, n_pkts_action;
2399 uint8_t *action_data;
2403 done = table->func(ts->obj,
2411 TRACE("[Thread %2u] table %u (not finalized)\n",
2419 action_id = hit ? action_id : ts->default_action_id;
2420 action_data = hit ? action_data : ts->default_action_data;
2421 n_pkts_hit = stats->n_pkts_hit[hit];
2422 n_pkts_action = stats->n_pkts_action[action_id];
2424 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2427 hit ? "hit" : "miss",
2428 (uint32_t)action_id);
2430 t->action_id = action_id;
2431 t->structs[0] = action_data;
2433 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2434 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2437 thread_ip_action_call(p, t, action_id);
2441 instr_table_af_exec(struct rte_swx_pipeline *p)
2443 struct thread *t = &p->threads[p->thread_id];
2444 struct instruction *ip = t->ip;
2445 uint32_t table_id = ip->table.table_id;
2446 struct rte_swx_table_state *ts = &t->table_state[table_id];
2447 struct table_runtime *table = &t->tables[table_id];
2448 struct table_statistics *stats = &p->table_stats[table_id];
2449 uint64_t action_id, n_pkts_hit, n_pkts_action;
2450 uint8_t *action_data;
2451 action_func_t action_func;
2455 done = table->func(ts->obj,
2463 TRACE("[Thread %2u] table %u (not finalized)\n",
2471 action_id = hit ? action_id : ts->default_action_id;
2472 action_data = hit ? action_data : ts->default_action_data;
2473 action_func = p->action_funcs[action_id];
2474 n_pkts_hit = stats->n_pkts_hit[hit];
2475 n_pkts_action = stats->n_pkts_action[action_id];
2477 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2480 hit ? "hit" : "miss",
2481 (uint32_t)action_id);
2483 t->action_id = action_id;
2484 t->structs[0] = action_data;
2486 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2487 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2497 instr_selector_exec(struct rte_swx_pipeline *p)
2499 struct thread *t = &p->threads[p->thread_id];
2500 struct instruction *ip = t->ip;
2501 uint32_t selector_id = ip->table.table_id;
2502 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2503 struct selector_runtime *selector = &t->selectors[selector_id];
2504 struct selector_statistics *stats = &p->selector_stats[selector_id];
2505 uint64_t n_pkts = stats->n_pkts;
2509 done = rte_swx_table_selector_select(ts->obj,
2511 selector->group_id_buffer,
2512 selector->selector_buffer,
2513 selector->member_id_buffer);
2516 TRACE("[Thread %2u] selector %u (not finalized)\n",
2525 TRACE("[Thread %2u] selector %u\n",
2529 stats->n_pkts = n_pkts + 1;
2536 instr_learner_exec(struct rte_swx_pipeline *p)
2538 struct thread *t = &p->threads[p->thread_id];
2539 struct instruction *ip = t->ip;
2540 uint32_t learner_id = ip->table.table_id;
2541 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2542 p->n_selectors + learner_id];
2543 struct learner_runtime *l = &t->learners[learner_id];
2544 struct learner_statistics *stats = &p->learner_stats[learner_id];
2545 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2546 uint8_t *action_data;
2550 time = rte_get_tsc_cycles();
2552 done = rte_swx_table_learner_lookup(ts->obj,
2561 TRACE("[Thread %2u] learner %u (not finalized)\n",
2569 action_id = hit ? action_id : ts->default_action_id;
2570 action_data = hit ? action_data : ts->default_action_data;
2571 n_pkts_hit = stats->n_pkts_hit[hit];
2572 n_pkts_action = stats->n_pkts_action[action_id];
2574 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2577 hit ? "hit" : "miss",
2578 (uint32_t)action_id);
2580 t->action_id = action_id;
2581 t->structs[0] = action_data;
2583 t->learner_id = learner_id;
2585 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2586 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2589 thread_ip_action_call(p, t, action_id);
2593 instr_learner_af_exec(struct rte_swx_pipeline *p)
2595 struct thread *t = &p->threads[p->thread_id];
2596 struct instruction *ip = t->ip;
2597 uint32_t learner_id = ip->table.table_id;
2598 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2599 p->n_selectors + learner_id];
2600 struct learner_runtime *l = &t->learners[learner_id];
2601 struct learner_statistics *stats = &p->learner_stats[learner_id];
2602 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2603 uint8_t *action_data;
2604 action_func_t action_func;
2608 time = rte_get_tsc_cycles();
2610 done = rte_swx_table_learner_lookup(ts->obj,
2619 TRACE("[Thread %2u] learner %u (not finalized)\n",
2627 action_id = hit ? action_id : ts->default_action_id;
2628 action_data = hit ? action_data : ts->default_action_data;
2629 action_func = p->action_funcs[action_id];
2630 n_pkts_hit = stats->n_pkts_hit[hit];
2631 n_pkts_action = stats->n_pkts_action[action_id];
2633 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2636 hit ? "hit" : "miss",
2637 (uint32_t)action_id);
2639 t->action_id = action_id;
2640 t->structs[0] = action_data;
2642 t->learner_id = learner_id;
2644 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2645 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2657 static struct action *
2658 action_find(struct rte_swx_pipeline *p, const char *name);
2661 action_has_nbo_args(struct action *a);
2664 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2667 instr_learn_translate(struct rte_swx_pipeline *p,
2668 struct action *action,
2671 struct instruction *instr,
2672 struct instruction_data *data __rte_unused)
2675 struct field *mf_first_arg = NULL, *mf_timeout_id = NULL;
2676 const char *mf_first_arg_name, *mf_timeout_id_name;
2678 CHECK(action, EINVAL);
2679 CHECK((n_tokens == 3) || (n_tokens == 4), EINVAL);
2682 a = action_find(p, tokens[1]);
2684 CHECK(!action_has_nbo_args(a), EINVAL);
2686 /* Action first argument. */
2687 mf_first_arg_name = (n_tokens == 4) ? tokens[2] : NULL;
2688 CHECK(!learner_action_args_check(p, a, mf_first_arg_name), EINVAL);
2690 if (mf_first_arg_name) {
2691 mf_first_arg = metadata_field_parse(p, mf_first_arg_name);
2692 CHECK(mf_first_arg, EINVAL);
2696 mf_timeout_id_name = (n_tokens == 4) ? tokens[3] : tokens[2];
2697 CHECK_NAME(mf_timeout_id_name, EINVAL);
2698 mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2699 CHECK(mf_timeout_id, EINVAL);
2702 instr->type = INSTR_LEARNER_LEARN;
2703 instr->learn.action_id = a->id;
2704 instr->learn.mf_first_arg_offset = mf_first_arg ? (mf_first_arg->offset / 8) : 0;
2705 instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2706 instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2712 instr_learn_exec(struct rte_swx_pipeline *p)
2714 struct thread *t = &p->threads[p->thread_id];
2715 struct instruction *ip = t->ip;
2717 __instr_learn_exec(p, t, ip);
2727 instr_rearm_translate(struct rte_swx_pipeline *p,
2728 struct action *action,
2731 struct instruction *instr,
2732 struct instruction_data *data __rte_unused)
2734 struct field *mf_timeout_id;
2735 const char *mf_timeout_id_name;
2737 CHECK(action, EINVAL);
2738 CHECK((n_tokens == 1) || (n_tokens == 2), EINVAL);
2740 /* INSTR_LEARNER_REARM. */
2741 if (n_tokens == 1) {
2742 instr->type = INSTR_LEARNER_REARM;
2746 /* INSTR_LEARNER_REARM_NEW. */
2747 mf_timeout_id_name = tokens[1];
2748 CHECK_NAME(mf_timeout_id_name, EINVAL);
2749 mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2750 CHECK(mf_timeout_id, EINVAL);
2752 instr->type = INSTR_LEARNER_REARM_NEW;
2753 instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2754 instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2760 instr_rearm_exec(struct rte_swx_pipeline *p)
2762 struct thread *t = &p->threads[p->thread_id];
2763 struct instruction *ip = t->ip;
2765 __instr_rearm_exec(p, t, ip);
2772 instr_rearm_new_exec(struct rte_swx_pipeline *p)
2774 struct thread *t = &p->threads[p->thread_id];
2775 struct instruction *ip = t->ip;
2777 __instr_rearm_new_exec(p, t, ip);
2787 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2788 struct action *action,
2789 char **tokens __rte_unused,
2791 struct instruction *instr,
2792 struct instruction_data *data __rte_unused)
2794 CHECK(action, EINVAL);
2795 CHECK(n_tokens == 1, EINVAL);
2797 instr->type = INSTR_LEARNER_FORGET;
2803 instr_forget_exec(struct rte_swx_pipeline *p)
2805 struct thread *t = &p->threads[p->thread_id];
2806 struct instruction *ip = t->ip;
2808 __instr_forget_exec(p, t, ip);
2818 instr_extern_translate(struct rte_swx_pipeline *p,
2819 struct action *action __rte_unused,
2822 struct instruction *instr,
2823 struct instruction_data *data __rte_unused)
2825 char *token = tokens[1];
2827 CHECK(n_tokens == 2, EINVAL);
2829 if (token[0] == 'e') {
2830 struct extern_obj *obj;
2831 struct extern_type_member_func *func;
2833 func = extern_obj_member_func_parse(p, token, &obj);
2834 CHECK(func, EINVAL);
2836 instr->type = INSTR_EXTERN_OBJ;
2837 instr->ext_obj.ext_obj_id = obj->id;
2838 instr->ext_obj.func_id = func->id;
2843 if (token[0] == 'f') {
2844 struct extern_func *func;
2846 func = extern_func_parse(p, token);
2847 CHECK(func, EINVAL);
2849 instr->type = INSTR_EXTERN_FUNC;
2850 instr->ext_func.ext_func_id = func->id;
2859 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2861 struct thread *t = &p->threads[p->thread_id];
2862 struct instruction *ip = t->ip;
2865 /* Extern object member function execute. */
2866 done = __instr_extern_obj_exec(p, t, ip);
2869 thread_ip_inc_cond(t, done);
2870 thread_yield_cond(p, done ^ 1);
2874 instr_extern_func_exec(struct rte_swx_pipeline *p)
2876 struct thread *t = &p->threads[p->thread_id];
2877 struct instruction *ip = t->ip;
2880 /* Extern function execute. */
2881 done = __instr_extern_func_exec(p, t, ip);
2884 thread_ip_inc_cond(t, done);
2885 thread_yield_cond(p, done ^ 1);
2892 instr_hash_translate(struct rte_swx_pipeline *p,
2893 struct action *action,
2896 struct instruction *instr,
2897 struct instruction_data *data __rte_unused)
2899 struct hash_func *func;
2900 struct field *dst, *src_first, *src_last;
2901 uint32_t src_struct_id_first = 0, src_struct_id_last = 0;
2903 CHECK(n_tokens == 5, EINVAL);
2905 func = hash_func_find(p, tokens[1]);
2906 CHECK(func, EINVAL);
2908 dst = metadata_field_parse(p, tokens[2]);
2911 src_first = struct_field_parse(p, action, tokens[3], &src_struct_id_first);
2912 CHECK(src_first, EINVAL);
2914 src_last = struct_field_parse(p, action, tokens[4], &src_struct_id_last);
2915 CHECK(src_last, EINVAL);
2916 CHECK(src_struct_id_first == src_struct_id_last, EINVAL);
2918 instr->type = INSTR_HASH_FUNC;
2919 instr->hash_func.hash_func_id = (uint8_t)func->id;
2920 instr->hash_func.dst.offset = (uint8_t)dst->offset / 8;
2921 instr->hash_func.dst.n_bits = (uint8_t)dst->n_bits;
2922 instr->hash_func.src.struct_id = (uint8_t)src_struct_id_first;
2923 instr->hash_func.src.offset = (uint16_t)src_first->offset / 8;
2924 instr->hash_func.src.n_bytes = (uint16_t)((src_last->offset + src_last->n_bits -
2925 src_first->offset) / 8);
2931 instr_hash_func_exec(struct rte_swx_pipeline *p)
2933 struct thread *t = &p->threads[p->thread_id];
2934 struct instruction *ip = t->ip;
2936 /* Extern function execute. */
2937 __instr_hash_func_exec(p, t, ip);
2947 instr_mov_translate(struct rte_swx_pipeline *p,
2948 struct action *action,
2951 struct instruction *instr,
2952 struct instruction_data *data __rte_unused)
2954 char *dst = tokens[1], *src = tokens[2];
2955 struct field *fdst, *fsrc;
2957 uint32_t dst_struct_id = 0, src_struct_id = 0;
2959 CHECK(n_tokens == 3, EINVAL);
2961 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2962 CHECK(fdst, EINVAL);
2963 CHECK(!fdst->var_size, EINVAL);
2965 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2966 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2968 CHECK(!fsrc->var_size, EINVAL);
2970 instr->type = INSTR_MOV;
2971 if (dst[0] != 'h' && src[0] == 'h')
2972 instr->type = INSTR_MOV_MH;
2973 if (dst[0] == 'h' && src[0] != 'h')
2974 instr->type = INSTR_MOV_HM;
2975 if (dst[0] == 'h' && src[0] == 'h')
2976 instr->type = INSTR_MOV_HH;
2978 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2979 instr->mov.dst.n_bits = fdst->n_bits;
2980 instr->mov.dst.offset = fdst->offset / 8;
2981 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2982 instr->mov.src.n_bits = fsrc->n_bits;
2983 instr->mov.src.offset = fsrc->offset / 8;
2988 src_val = strtoull(src, &src, 0);
2989 CHECK(!src[0], EINVAL);
2992 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2994 instr->type = INSTR_MOV_I;
2995 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2996 instr->mov.dst.n_bits = fdst->n_bits;
2997 instr->mov.dst.offset = fdst->offset / 8;
2998 instr->mov.src_val = src_val;
3003 instr_mov_exec(struct rte_swx_pipeline *p)
3005 struct thread *t = &p->threads[p->thread_id];
3006 struct instruction *ip = t->ip;
3008 __instr_mov_exec(p, t, ip);
3015 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3017 struct thread *t = &p->threads[p->thread_id];
3018 struct instruction *ip = t->ip;
3020 __instr_mov_mh_exec(p, t, ip);
3027 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3029 struct thread *t = &p->threads[p->thread_id];
3030 struct instruction *ip = t->ip;
3032 __instr_mov_hm_exec(p, t, ip);
3039 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3041 struct thread *t = &p->threads[p->thread_id];
3042 struct instruction *ip = t->ip;
3044 __instr_mov_hh_exec(p, t, ip);
3051 instr_mov_i_exec(struct rte_swx_pipeline *p)
3053 struct thread *t = &p->threads[p->thread_id];
3054 struct instruction *ip = t->ip;
3056 __instr_mov_i_exec(p, t, ip);
3066 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3068 struct thread *t = &p->threads[p->thread_id];
3069 struct instruction *ip = t->ip;
3071 __instr_dma_ht_exec(p, t, ip);
3078 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3080 struct thread *t = &p->threads[p->thread_id];
3081 struct instruction *ip = t->ip;
3083 __instr_dma_ht2_exec(p, t, ip);
3090 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3092 struct thread *t = &p->threads[p->thread_id];
3093 struct instruction *ip = t->ip;
3095 __instr_dma_ht3_exec(p, t, ip);
3102 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3104 struct thread *t = &p->threads[p->thread_id];
3105 struct instruction *ip = t->ip;
3107 __instr_dma_ht4_exec(p, t, ip);
3114 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3116 struct thread *t = &p->threads[p->thread_id];
3117 struct instruction *ip = t->ip;
3119 __instr_dma_ht5_exec(p, t, ip);
3126 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3128 struct thread *t = &p->threads[p->thread_id];
3129 struct instruction *ip = t->ip;
3131 __instr_dma_ht6_exec(p, t, ip);
3138 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3140 struct thread *t = &p->threads[p->thread_id];
3141 struct instruction *ip = t->ip;
3143 __instr_dma_ht7_exec(p, t, ip);
3150 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3152 struct thread *t = &p->threads[p->thread_id];
3153 struct instruction *ip = t->ip;
3155 __instr_dma_ht8_exec(p, t, ip);
3165 instr_alu_add_translate(struct rte_swx_pipeline *p,
3166 struct action *action,
3169 struct instruction *instr,
3170 struct instruction_data *data __rte_unused)
3172 char *dst = tokens[1], *src = tokens[2];
3173 struct field *fdst, *fsrc;
3175 uint32_t dst_struct_id = 0, src_struct_id = 0;
3177 CHECK(n_tokens == 3, EINVAL);
3179 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3180 CHECK(fdst, EINVAL);
3181 CHECK(!fdst->var_size, EINVAL);
3183 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3184 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3186 CHECK(!fsrc->var_size, EINVAL);
3188 instr->type = INSTR_ALU_ADD;
3189 if (dst[0] == 'h' && src[0] != 'h')
3190 instr->type = INSTR_ALU_ADD_HM;
3191 if (dst[0] != 'h' && src[0] == 'h')
3192 instr->type = INSTR_ALU_ADD_MH;
3193 if (dst[0] == 'h' && src[0] == 'h')
3194 instr->type = INSTR_ALU_ADD_HH;
3196 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3197 instr->alu.dst.n_bits = fdst->n_bits;
3198 instr->alu.dst.offset = fdst->offset / 8;
3199 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3200 instr->alu.src.n_bits = fsrc->n_bits;
3201 instr->alu.src.offset = fsrc->offset / 8;
3205 /* ADD_MI, ADD_HI. */
3206 src_val = strtoull(src, &src, 0);
3207 CHECK(!src[0], EINVAL);
3209 instr->type = INSTR_ALU_ADD_MI;
3211 instr->type = INSTR_ALU_ADD_HI;
3213 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3214 instr->alu.dst.n_bits = fdst->n_bits;
3215 instr->alu.dst.offset = fdst->offset / 8;
3216 instr->alu.src_val = src_val;
3221 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3222 struct action *action,
3225 struct instruction *instr,
3226 struct instruction_data *data __rte_unused)
3228 char *dst = tokens[1], *src = tokens[2];
3229 struct field *fdst, *fsrc;
3231 uint32_t dst_struct_id = 0, src_struct_id = 0;
3233 CHECK(n_tokens == 3, EINVAL);
3235 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3236 CHECK(fdst, EINVAL);
3237 CHECK(!fdst->var_size, EINVAL);
3239 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3240 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3242 CHECK(!fsrc->var_size, EINVAL);
3244 instr->type = INSTR_ALU_SUB;
3245 if (dst[0] == 'h' && src[0] != 'h')
3246 instr->type = INSTR_ALU_SUB_HM;
3247 if (dst[0] != 'h' && src[0] == 'h')
3248 instr->type = INSTR_ALU_SUB_MH;
3249 if (dst[0] == 'h' && src[0] == 'h')
3250 instr->type = INSTR_ALU_SUB_HH;
3252 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3253 instr->alu.dst.n_bits = fdst->n_bits;
3254 instr->alu.dst.offset = fdst->offset / 8;
3255 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3256 instr->alu.src.n_bits = fsrc->n_bits;
3257 instr->alu.src.offset = fsrc->offset / 8;
3261 /* SUB_MI, SUB_HI. */
3262 src_val = strtoull(src, &src, 0);
3263 CHECK(!src[0], EINVAL);
3265 instr->type = INSTR_ALU_SUB_MI;
3267 instr->type = INSTR_ALU_SUB_HI;
3269 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3270 instr->alu.dst.n_bits = fdst->n_bits;
3271 instr->alu.dst.offset = fdst->offset / 8;
3272 instr->alu.src_val = src_val;
3277 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3278 struct action *action __rte_unused,
3281 struct instruction *instr,
3282 struct instruction_data *data __rte_unused)
3284 char *dst = tokens[1], *src = tokens[2];
3285 struct header *hdst, *hsrc;
3286 struct field *fdst, *fsrc;
3288 CHECK(n_tokens == 3, EINVAL);
3290 fdst = header_field_parse(p, dst, &hdst);
3291 CHECK(fdst, EINVAL);
3292 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3295 fsrc = header_field_parse(p, src, &hsrc);
3297 CHECK(!fsrc->var_size, EINVAL);
3299 instr->type = INSTR_ALU_CKADD_FIELD;
3300 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3301 instr->alu.dst.n_bits = fdst->n_bits;
3302 instr->alu.dst.offset = fdst->offset / 8;
3303 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3304 instr->alu.src.n_bits = fsrc->n_bits;
3305 instr->alu.src.offset = fsrc->offset / 8;
3309 /* CKADD_STRUCT, CKADD_STRUCT20. */
3310 hsrc = header_parse(p, src);
3311 CHECK(hsrc, EINVAL);
3313 instr->type = INSTR_ALU_CKADD_STRUCT;
3314 if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3315 instr->type = INSTR_ALU_CKADD_STRUCT20;
3317 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3318 instr->alu.dst.n_bits = fdst->n_bits;
3319 instr->alu.dst.offset = fdst->offset / 8;
3320 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3321 instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3322 instr->alu.src.offset = 0; /* Unused. */
3327 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3328 struct action *action __rte_unused,
3331 struct instruction *instr,
3332 struct instruction_data *data __rte_unused)
3334 char *dst = tokens[1], *src = tokens[2];
3335 struct header *hdst, *hsrc;
3336 struct field *fdst, *fsrc;
3338 CHECK(n_tokens == 3, EINVAL);
3340 fdst = header_field_parse(p, dst, &hdst);
3341 CHECK(fdst, EINVAL);
3342 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3344 fsrc = header_field_parse(p, src, &hsrc);
3345 CHECK(fsrc, EINVAL);
3346 CHECK(!fsrc->var_size, EINVAL);
3348 instr->type = INSTR_ALU_CKSUB_FIELD;
3349 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3350 instr->alu.dst.n_bits = fdst->n_bits;
3351 instr->alu.dst.offset = fdst->offset / 8;
3352 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3353 instr->alu.src.n_bits = fsrc->n_bits;
3354 instr->alu.src.offset = fsrc->offset / 8;
3359 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3360 struct action *action,
3363 struct instruction *instr,
3364 struct instruction_data *data __rte_unused)
3366 char *dst = tokens[1], *src = tokens[2];
3367 struct field *fdst, *fsrc;
3369 uint32_t dst_struct_id = 0, src_struct_id = 0;
3371 CHECK(n_tokens == 3, EINVAL);
3373 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3374 CHECK(fdst, EINVAL);
3375 CHECK(!fdst->var_size, EINVAL);
3377 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3378 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3380 CHECK(!fsrc->var_size, EINVAL);
3382 instr->type = INSTR_ALU_SHL;
3383 if (dst[0] == 'h' && src[0] != 'h')
3384 instr->type = INSTR_ALU_SHL_HM;
3385 if (dst[0] != 'h' && src[0] == 'h')
3386 instr->type = INSTR_ALU_SHL_MH;
3387 if (dst[0] == 'h' && src[0] == 'h')
3388 instr->type = INSTR_ALU_SHL_HH;
3390 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3391 instr->alu.dst.n_bits = fdst->n_bits;
3392 instr->alu.dst.offset = fdst->offset / 8;
3393 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3394 instr->alu.src.n_bits = fsrc->n_bits;
3395 instr->alu.src.offset = fsrc->offset / 8;
3399 /* SHL_MI, SHL_HI. */
3400 src_val = strtoull(src, &src, 0);
3401 CHECK(!src[0], EINVAL);
3403 instr->type = INSTR_ALU_SHL_MI;
3405 instr->type = INSTR_ALU_SHL_HI;
3407 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3408 instr->alu.dst.n_bits = fdst->n_bits;
3409 instr->alu.dst.offset = fdst->offset / 8;
3410 instr->alu.src_val = src_val;
3415 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3416 struct action *action,
3419 struct instruction *instr,
3420 struct instruction_data *data __rte_unused)
3422 char *dst = tokens[1], *src = tokens[2];
3423 struct field *fdst, *fsrc;
3425 uint32_t dst_struct_id = 0, src_struct_id = 0;
3427 CHECK(n_tokens == 3, EINVAL);
3429 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3430 CHECK(fdst, EINVAL);
3431 CHECK(!fdst->var_size, EINVAL);
3433 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3434 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3436 CHECK(!fsrc->var_size, EINVAL);
3438 instr->type = INSTR_ALU_SHR;
3439 if (dst[0] == 'h' && src[0] != 'h')
3440 instr->type = INSTR_ALU_SHR_HM;
3441 if (dst[0] != 'h' && src[0] == 'h')
3442 instr->type = INSTR_ALU_SHR_MH;
3443 if (dst[0] == 'h' && src[0] == 'h')
3444 instr->type = INSTR_ALU_SHR_HH;
3446 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3447 instr->alu.dst.n_bits = fdst->n_bits;
3448 instr->alu.dst.offset = fdst->offset / 8;
3449 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3450 instr->alu.src.n_bits = fsrc->n_bits;
3451 instr->alu.src.offset = fsrc->offset / 8;
3455 /* SHR_MI, SHR_HI. */
3456 src_val = strtoull(src, &src, 0);
3457 CHECK(!src[0], EINVAL);
3459 instr->type = INSTR_ALU_SHR_MI;
3461 instr->type = INSTR_ALU_SHR_HI;
3463 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3464 instr->alu.dst.n_bits = fdst->n_bits;
3465 instr->alu.dst.offset = fdst->offset / 8;
3466 instr->alu.src_val = src_val;
3471 instr_alu_and_translate(struct rte_swx_pipeline *p,
3472 struct action *action,
3475 struct instruction *instr,
3476 struct instruction_data *data __rte_unused)
3478 char *dst = tokens[1], *src = tokens[2];
3479 struct field *fdst, *fsrc;
3481 uint32_t dst_struct_id = 0, src_struct_id = 0;
3483 CHECK(n_tokens == 3, EINVAL);
3485 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3486 CHECK(fdst, EINVAL);
3487 CHECK(!fdst->var_size, EINVAL);
3489 /* AND, AND_MH, AND_HM, AND_HH. */
3490 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3492 CHECK(!fsrc->var_size, EINVAL);
3494 instr->type = INSTR_ALU_AND;
3495 if (dst[0] != 'h' && src[0] == 'h')
3496 instr->type = INSTR_ALU_AND_MH;
3497 if (dst[0] == 'h' && src[0] != 'h')
3498 instr->type = INSTR_ALU_AND_HM;
3499 if (dst[0] == 'h' && src[0] == 'h')
3500 instr->type = INSTR_ALU_AND_HH;
3502 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3503 instr->alu.dst.n_bits = fdst->n_bits;
3504 instr->alu.dst.offset = fdst->offset / 8;
3505 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3506 instr->alu.src.n_bits = fsrc->n_bits;
3507 instr->alu.src.offset = fsrc->offset / 8;
3512 src_val = strtoull(src, &src, 0);
3513 CHECK(!src[0], EINVAL);
3516 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3518 instr->type = INSTR_ALU_AND_I;
3519 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3520 instr->alu.dst.n_bits = fdst->n_bits;
3521 instr->alu.dst.offset = fdst->offset / 8;
3522 instr->alu.src_val = src_val;
3527 instr_alu_or_translate(struct rte_swx_pipeline *p,
3528 struct action *action,
3531 struct instruction *instr,
3532 struct instruction_data *data __rte_unused)
3534 char *dst = tokens[1], *src = tokens[2];
3535 struct field *fdst, *fsrc;
3537 uint32_t dst_struct_id = 0, src_struct_id = 0;
3539 CHECK(n_tokens == 3, EINVAL);
3541 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3542 CHECK(fdst, EINVAL);
3543 CHECK(!fdst->var_size, EINVAL);
3545 /* OR, OR_MH, OR_HM, OR_HH. */
3546 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3548 CHECK(!fsrc->var_size, EINVAL);
3550 instr->type = INSTR_ALU_OR;
3551 if (dst[0] != 'h' && src[0] == 'h')
3552 instr->type = INSTR_ALU_OR_MH;
3553 if (dst[0] == 'h' && src[0] != 'h')
3554 instr->type = INSTR_ALU_OR_HM;
3555 if (dst[0] == 'h' && src[0] == 'h')
3556 instr->type = INSTR_ALU_OR_HH;
3558 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3559 instr->alu.dst.n_bits = fdst->n_bits;
3560 instr->alu.dst.offset = fdst->offset / 8;
3561 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3562 instr->alu.src.n_bits = fsrc->n_bits;
3563 instr->alu.src.offset = fsrc->offset / 8;
3568 src_val = strtoull(src, &src, 0);
3569 CHECK(!src[0], EINVAL);
3572 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3574 instr->type = INSTR_ALU_OR_I;
3575 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3576 instr->alu.dst.n_bits = fdst->n_bits;
3577 instr->alu.dst.offset = fdst->offset / 8;
3578 instr->alu.src_val = src_val;
3583 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3584 struct action *action,
3587 struct instruction *instr,
3588 struct instruction_data *data __rte_unused)
3590 char *dst = tokens[1], *src = tokens[2];
3591 struct field *fdst, *fsrc;
3593 uint32_t dst_struct_id = 0, src_struct_id = 0;
3595 CHECK(n_tokens == 3, EINVAL);
3597 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3598 CHECK(fdst, EINVAL);
3599 CHECK(!fdst->var_size, EINVAL);
3601 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3602 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3604 CHECK(!fsrc->var_size, EINVAL);
3606 instr->type = INSTR_ALU_XOR;
3607 if (dst[0] != 'h' && src[0] == 'h')
3608 instr->type = INSTR_ALU_XOR_MH;
3609 if (dst[0] == 'h' && src[0] != 'h')
3610 instr->type = INSTR_ALU_XOR_HM;
3611 if (dst[0] == 'h' && src[0] == 'h')
3612 instr->type = INSTR_ALU_XOR_HH;
3614 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3615 instr->alu.dst.n_bits = fdst->n_bits;
3616 instr->alu.dst.offset = fdst->offset / 8;
3617 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3618 instr->alu.src.n_bits = fsrc->n_bits;
3619 instr->alu.src.offset = fsrc->offset / 8;
3624 src_val = strtoull(src, &src, 0);
3625 CHECK(!src[0], EINVAL);
3628 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3630 instr->type = INSTR_ALU_XOR_I;
3631 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3632 instr->alu.dst.n_bits = fdst->n_bits;
3633 instr->alu.dst.offset = fdst->offset / 8;
3634 instr->alu.src_val = src_val;
3639 instr_alu_add_exec(struct rte_swx_pipeline *p)
3641 struct thread *t = &p->threads[p->thread_id];
3642 struct instruction *ip = t->ip;
3645 __instr_alu_add_exec(p, t, ip);
3652 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3654 struct thread *t = &p->threads[p->thread_id];
3655 struct instruction *ip = t->ip;
3658 __instr_alu_add_mh_exec(p, t, ip);
3665 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3667 struct thread *t = &p->threads[p->thread_id];
3668 struct instruction *ip = t->ip;
3671 __instr_alu_add_hm_exec(p, t, ip);
3678 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3680 struct thread *t = &p->threads[p->thread_id];
3681 struct instruction *ip = t->ip;
3684 __instr_alu_add_hh_exec(p, t, ip);
3691 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3693 struct thread *t = &p->threads[p->thread_id];
3694 struct instruction *ip = t->ip;
3697 __instr_alu_add_mi_exec(p, t, ip);
3704 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3706 struct thread *t = &p->threads[p->thread_id];
3707 struct instruction *ip = t->ip;
3710 __instr_alu_add_hi_exec(p, t, ip);
3717 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3719 struct thread *t = &p->threads[p->thread_id];
3720 struct instruction *ip = t->ip;
3723 __instr_alu_sub_exec(p, t, ip);
3730 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3732 struct thread *t = &p->threads[p->thread_id];
3733 struct instruction *ip = t->ip;
3736 __instr_alu_sub_mh_exec(p, t, ip);
3743 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3745 struct thread *t = &p->threads[p->thread_id];
3746 struct instruction *ip = t->ip;
3749 __instr_alu_sub_hm_exec(p, t, ip);
3756 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3758 struct thread *t = &p->threads[p->thread_id];
3759 struct instruction *ip = t->ip;
3762 __instr_alu_sub_hh_exec(p, t, ip);
3769 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3771 struct thread *t = &p->threads[p->thread_id];
3772 struct instruction *ip = t->ip;
3775 __instr_alu_sub_mi_exec(p, t, ip);
3782 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3784 struct thread *t = &p->threads[p->thread_id];
3785 struct instruction *ip = t->ip;
3788 __instr_alu_sub_hi_exec(p, t, ip);
3795 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3797 struct thread *t = &p->threads[p->thread_id];
3798 struct instruction *ip = t->ip;
3801 __instr_alu_shl_exec(p, t, ip);
3808 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3810 struct thread *t = &p->threads[p->thread_id];
3811 struct instruction *ip = t->ip;
3814 __instr_alu_shl_mh_exec(p, t, ip);
3821 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3823 struct thread *t = &p->threads[p->thread_id];
3824 struct instruction *ip = t->ip;
3827 __instr_alu_shl_hm_exec(p, t, ip);
3834 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3836 struct thread *t = &p->threads[p->thread_id];
3837 struct instruction *ip = t->ip;
3840 __instr_alu_shl_hh_exec(p, t, ip);
3847 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3849 struct thread *t = &p->threads[p->thread_id];
3850 struct instruction *ip = t->ip;
3853 __instr_alu_shl_mi_exec(p, t, ip);
3860 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3862 struct thread *t = &p->threads[p->thread_id];
3863 struct instruction *ip = t->ip;
3866 __instr_alu_shl_hi_exec(p, t, ip);
3873 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3875 struct thread *t = &p->threads[p->thread_id];
3876 struct instruction *ip = t->ip;
3879 __instr_alu_shr_exec(p, t, ip);
3886 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3888 struct thread *t = &p->threads[p->thread_id];
3889 struct instruction *ip = t->ip;
3892 __instr_alu_shr_mh_exec(p, t, ip);
3899 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3901 struct thread *t = &p->threads[p->thread_id];
3902 struct instruction *ip = t->ip;
3905 __instr_alu_shr_hm_exec(p, t, ip);
3912 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3914 struct thread *t = &p->threads[p->thread_id];
3915 struct instruction *ip = t->ip;
3918 __instr_alu_shr_hh_exec(p, t, ip);
3925 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3927 struct thread *t = &p->threads[p->thread_id];
3928 struct instruction *ip = t->ip;
3931 __instr_alu_shr_mi_exec(p, t, ip);
3938 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3940 struct thread *t = &p->threads[p->thread_id];
3941 struct instruction *ip = t->ip;
3944 __instr_alu_shr_hi_exec(p, t, ip);
3951 instr_alu_and_exec(struct rte_swx_pipeline *p)
3953 struct thread *t = &p->threads[p->thread_id];
3954 struct instruction *ip = t->ip;
3957 __instr_alu_and_exec(p, t, ip);
3964 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3966 struct thread *t = &p->threads[p->thread_id];
3967 struct instruction *ip = t->ip;
3970 __instr_alu_and_mh_exec(p, t, ip);
3977 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3979 struct thread *t = &p->threads[p->thread_id];
3980 struct instruction *ip = t->ip;
3983 __instr_alu_and_hm_exec(p, t, ip);
3990 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3992 struct thread *t = &p->threads[p->thread_id];
3993 struct instruction *ip = t->ip;
3996 __instr_alu_and_hh_exec(p, t, ip);
4003 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4005 struct thread *t = &p->threads[p->thread_id];
4006 struct instruction *ip = t->ip;
4009 __instr_alu_and_i_exec(p, t, ip);
4016 instr_alu_or_exec(struct rte_swx_pipeline *p)
4018 struct thread *t = &p->threads[p->thread_id];
4019 struct instruction *ip = t->ip;
4022 __instr_alu_or_exec(p, t, ip);
4029 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4031 struct thread *t = &p->threads[p->thread_id];
4032 struct instruction *ip = t->ip;
4035 __instr_alu_or_mh_exec(p, t, ip);
4042 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4044 struct thread *t = &p->threads[p->thread_id];
4045 struct instruction *ip = t->ip;
4048 __instr_alu_or_hm_exec(p, t, ip);
4055 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4057 struct thread *t = &p->threads[p->thread_id];
4058 struct instruction *ip = t->ip;
4061 __instr_alu_or_hh_exec(p, t, ip);
4068 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4070 struct thread *t = &p->threads[p->thread_id];
4071 struct instruction *ip = t->ip;
4074 __instr_alu_or_i_exec(p, t, ip);
4081 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4083 struct thread *t = &p->threads[p->thread_id];
4084 struct instruction *ip = t->ip;
4087 __instr_alu_xor_exec(p, t, ip);
4094 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4096 struct thread *t = &p->threads[p->thread_id];
4097 struct instruction *ip = t->ip;
4100 __instr_alu_xor_mh_exec(p, t, ip);
4107 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4109 struct thread *t = &p->threads[p->thread_id];
4110 struct instruction *ip = t->ip;
4113 __instr_alu_xor_hm_exec(p, t, ip);
4120 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4122 struct thread *t = &p->threads[p->thread_id];
4123 struct instruction *ip = t->ip;
4126 __instr_alu_xor_hh_exec(p, t, ip);
4133 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4135 struct thread *t = &p->threads[p->thread_id];
4136 struct instruction *ip = t->ip;
4139 __instr_alu_xor_i_exec(p, t, ip);
4146 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4148 struct thread *t = &p->threads[p->thread_id];
4149 struct instruction *ip = t->ip;
4152 __instr_alu_ckadd_field_exec(p, t, ip);
4159 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4161 struct thread *t = &p->threads[p->thread_id];
4162 struct instruction *ip = t->ip;
4165 __instr_alu_cksub_field_exec(p, t, ip);
4172 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4174 struct thread *t = &p->threads[p->thread_id];
4175 struct instruction *ip = t->ip;
4178 __instr_alu_ckadd_struct20_exec(p, t, ip);
4185 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4187 struct thread *t = &p->threads[p->thread_id];
4188 struct instruction *ip = t->ip;
4191 __instr_alu_ckadd_struct_exec(p, t, ip);
4200 static struct regarray *
4201 regarray_find(struct rte_swx_pipeline *p, const char *name);
4204 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4205 struct action *action,
4208 struct instruction *instr,
4209 struct instruction_data *data __rte_unused)
4211 char *regarray = tokens[1], *idx = tokens[2];
4214 uint32_t idx_struct_id, idx_val;
4216 CHECK(n_tokens == 3, EINVAL);
4218 r = regarray_find(p, regarray);
4221 /* REGPREFETCH_RH, REGPREFETCH_RM. */
4222 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4224 CHECK(!fidx->var_size, EINVAL);
4226 instr->type = INSTR_REGPREFETCH_RM;
4228 instr->type = INSTR_REGPREFETCH_RH;
4230 instr->regarray.regarray_id = r->id;
4231 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4232 instr->regarray.idx.n_bits = fidx->n_bits;
4233 instr->regarray.idx.offset = fidx->offset / 8;
4234 instr->regarray.dstsrc_val = 0; /* Unused. */
4238 /* REGPREFETCH_RI. */
4239 idx_val = strtoul(idx, &idx, 0);
4240 CHECK(!idx[0], EINVAL);
4242 instr->type = INSTR_REGPREFETCH_RI;
4243 instr->regarray.regarray_id = r->id;
4244 instr->regarray.idx_val = idx_val;
4245 instr->regarray.dstsrc_val = 0; /* Unused. */
4250 instr_regrd_translate(struct rte_swx_pipeline *p,
4251 struct action *action,
4254 struct instruction *instr,
4255 struct instruction_data *data __rte_unused)
4257 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4259 struct field *fdst, *fidx;
4260 uint32_t dst_struct_id, idx_struct_id, idx_val;
4262 CHECK(n_tokens == 4, EINVAL);
4264 r = regarray_find(p, regarray);
4267 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4268 CHECK(fdst, EINVAL);
4269 CHECK(!fdst->var_size, EINVAL);
4271 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4272 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4274 CHECK(!fidx->var_size, EINVAL);
4276 instr->type = INSTR_REGRD_MRM;
4277 if (dst[0] == 'h' && idx[0] != 'h')
4278 instr->type = INSTR_REGRD_HRM;
4279 if (dst[0] != 'h' && idx[0] == 'h')
4280 instr->type = INSTR_REGRD_MRH;
4281 if (dst[0] == 'h' && idx[0] == 'h')
4282 instr->type = INSTR_REGRD_HRH;
4284 instr->regarray.regarray_id = r->id;
4285 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4286 instr->regarray.idx.n_bits = fidx->n_bits;
4287 instr->regarray.idx.offset = fidx->offset / 8;
4288 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4289 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4290 instr->regarray.dstsrc.offset = fdst->offset / 8;
4294 /* REGRD_MRI, REGRD_HRI. */
4295 idx_val = strtoul(idx, &idx, 0);
4296 CHECK(!idx[0], EINVAL);
4298 instr->type = INSTR_REGRD_MRI;
4300 instr->type = INSTR_REGRD_HRI;
4302 instr->regarray.regarray_id = r->id;
4303 instr->regarray.idx_val = idx_val;
4304 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4305 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4306 instr->regarray.dstsrc.offset = fdst->offset / 8;
4311 instr_regwr_translate(struct rte_swx_pipeline *p,
4312 struct action *action,
4315 struct instruction *instr,
4316 struct instruction_data *data __rte_unused)
4318 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4320 struct field *fidx, *fsrc;
4322 uint32_t idx_struct_id, idx_val, src_struct_id;
4324 CHECK(n_tokens == 4, EINVAL);
4326 r = regarray_find(p, regarray);
4329 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4330 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4331 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4333 CHECK(!fidx->var_size, EINVAL);
4334 CHECK(!fsrc->var_size, EINVAL);
4336 instr->type = INSTR_REGWR_RMM;
4337 if (idx[0] == 'h' && src[0] != 'h')
4338 instr->type = INSTR_REGWR_RHM;
4339 if (idx[0] != 'h' && src[0] == 'h')
4340 instr->type = INSTR_REGWR_RMH;
4341 if (idx[0] == 'h' && src[0] == 'h')
4342 instr->type = INSTR_REGWR_RHH;
4344 instr->regarray.regarray_id = r->id;
4345 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4346 instr->regarray.idx.n_bits = fidx->n_bits;
4347 instr->regarray.idx.offset = fidx->offset / 8;
4348 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4349 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4350 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4354 /* REGWR_RHI, REGWR_RMI. */
4355 if (fidx && !fsrc) {
4356 CHECK(!fidx->var_size, EINVAL);
4358 src_val = strtoull(src, &src, 0);
4359 CHECK(!src[0], EINVAL);
4361 instr->type = INSTR_REGWR_RMI;
4363 instr->type = INSTR_REGWR_RHI;
4365 instr->regarray.regarray_id = r->id;
4366 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4367 instr->regarray.idx.n_bits = fidx->n_bits;
4368 instr->regarray.idx.offset = fidx->offset / 8;
4369 instr->regarray.dstsrc_val = src_val;
4373 /* REGWR_RIH, REGWR_RIM. */
4374 if (!fidx && fsrc) {
4375 idx_val = strtoul(idx, &idx, 0);
4376 CHECK(!idx[0], EINVAL);
4378 CHECK(!fsrc->var_size, EINVAL);
4380 instr->type = INSTR_REGWR_RIM;
4382 instr->type = INSTR_REGWR_RIH;
4384 instr->regarray.regarray_id = r->id;
4385 instr->regarray.idx_val = idx_val;
4386 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4387 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4388 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4393 src_val = strtoull(src, &src, 0);
4394 CHECK(!src[0], EINVAL);
4396 idx_val = strtoul(idx, &idx, 0);
4397 CHECK(!idx[0], EINVAL);
4399 instr->type = INSTR_REGWR_RII;
4400 instr->regarray.idx_val = idx_val;
4401 instr->regarray.dstsrc_val = src_val;
4407 instr_regadd_translate(struct rte_swx_pipeline *p,
4408 struct action *action,
4411 struct instruction *instr,
4412 struct instruction_data *data __rte_unused)
4414 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4416 struct field *fidx, *fsrc;
4418 uint32_t idx_struct_id, idx_val, src_struct_id;
4420 CHECK(n_tokens == 4, EINVAL);
4422 r = regarray_find(p, regarray);
4425 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4426 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4427 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4429 CHECK(!fidx->var_size, EINVAL);
4430 CHECK(!fsrc->var_size, EINVAL);
4432 instr->type = INSTR_REGADD_RMM;
4433 if (idx[0] == 'h' && src[0] != 'h')
4434 instr->type = INSTR_REGADD_RHM;
4435 if (idx[0] != 'h' && src[0] == 'h')
4436 instr->type = INSTR_REGADD_RMH;
4437 if (idx[0] == 'h' && src[0] == 'h')
4438 instr->type = INSTR_REGADD_RHH;
4440 instr->regarray.regarray_id = r->id;
4441 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4442 instr->regarray.idx.n_bits = fidx->n_bits;
4443 instr->regarray.idx.offset = fidx->offset / 8;
4444 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4445 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4446 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4450 /* REGADD_RHI, REGADD_RMI. */
4451 if (fidx && !fsrc) {
4452 CHECK(!fidx->var_size, EINVAL);
4454 src_val = strtoull(src, &src, 0);
4455 CHECK(!src[0], EINVAL);
4457 instr->type = INSTR_REGADD_RMI;
4459 instr->type = INSTR_REGADD_RHI;
4461 instr->regarray.regarray_id = r->id;
4462 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4463 instr->regarray.idx.n_bits = fidx->n_bits;
4464 instr->regarray.idx.offset = fidx->offset / 8;
4465 instr->regarray.dstsrc_val = src_val;
4469 /* REGADD_RIH, REGADD_RIM. */
4470 if (!fidx && fsrc) {
4471 idx_val = strtoul(idx, &idx, 0);
4472 CHECK(!idx[0], EINVAL);
4474 CHECK(!fsrc->var_size, EINVAL);
4476 instr->type = INSTR_REGADD_RIM;
4478 instr->type = INSTR_REGADD_RIH;
4480 instr->regarray.regarray_id = r->id;
4481 instr->regarray.idx_val = idx_val;
4482 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4483 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4484 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4489 src_val = strtoull(src, &src, 0);
4490 CHECK(!src[0], EINVAL);
4492 idx_val = strtoul(idx, &idx, 0);
4493 CHECK(!idx[0], EINVAL);
4495 instr->type = INSTR_REGADD_RII;
4496 instr->regarray.idx_val = idx_val;
4497 instr->regarray.dstsrc_val = src_val;
4502 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4504 struct thread *t = &p->threads[p->thread_id];
4505 struct instruction *ip = t->ip;
4508 __instr_regprefetch_rh_exec(p, t, ip);
4515 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4517 struct thread *t = &p->threads[p->thread_id];
4518 struct instruction *ip = t->ip;
4521 __instr_regprefetch_rm_exec(p, t, ip);
4528 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4530 struct thread *t = &p->threads[p->thread_id];
4531 struct instruction *ip = t->ip;
4534 __instr_regprefetch_ri_exec(p, t, ip);
4541 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4543 struct thread *t = &p->threads[p->thread_id];
4544 struct instruction *ip = t->ip;
4547 __instr_regrd_hrh_exec(p, t, ip);
4554 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4556 struct thread *t = &p->threads[p->thread_id];
4557 struct instruction *ip = t->ip;
4560 __instr_regrd_hrm_exec(p, t, ip);
4567 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4569 struct thread *t = &p->threads[p->thread_id];
4570 struct instruction *ip = t->ip;
4573 __instr_regrd_mrh_exec(p, t, ip);
4580 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4582 struct thread *t = &p->threads[p->thread_id];
4583 struct instruction *ip = t->ip;
4586 __instr_regrd_mrm_exec(p, t, ip);
4593 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4595 struct thread *t = &p->threads[p->thread_id];
4596 struct instruction *ip = t->ip;
4599 __instr_regrd_hri_exec(p, t, ip);
4606 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4608 struct thread *t = &p->threads[p->thread_id];
4609 struct instruction *ip = t->ip;
4612 __instr_regrd_mri_exec(p, t, ip);
4619 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4621 struct thread *t = &p->threads[p->thread_id];
4622 struct instruction *ip = t->ip;
4625 __instr_regwr_rhh_exec(p, t, ip);
4632 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4634 struct thread *t = &p->threads[p->thread_id];
4635 struct instruction *ip = t->ip;
4638 __instr_regwr_rhm_exec(p, t, ip);
4645 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4647 struct thread *t = &p->threads[p->thread_id];
4648 struct instruction *ip = t->ip;
4651 __instr_regwr_rmh_exec(p, t, ip);
4658 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4660 struct thread *t = &p->threads[p->thread_id];
4661 struct instruction *ip = t->ip;
4664 __instr_regwr_rmm_exec(p, t, ip);
4671 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4673 struct thread *t = &p->threads[p->thread_id];
4674 struct instruction *ip = t->ip;
4677 __instr_regwr_rhi_exec(p, t, ip);
4684 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4686 struct thread *t = &p->threads[p->thread_id];
4687 struct instruction *ip = t->ip;
4690 __instr_regwr_rmi_exec(p, t, ip);
4697 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4699 struct thread *t = &p->threads[p->thread_id];
4700 struct instruction *ip = t->ip;
4703 __instr_regwr_rih_exec(p, t, ip);
4710 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4712 struct thread *t = &p->threads[p->thread_id];
4713 struct instruction *ip = t->ip;
4716 __instr_regwr_rim_exec(p, t, ip);
4723 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4725 struct thread *t = &p->threads[p->thread_id];
4726 struct instruction *ip = t->ip;
4729 __instr_regwr_rii_exec(p, t, ip);
4736 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4738 struct thread *t = &p->threads[p->thread_id];
4739 struct instruction *ip = t->ip;
4742 __instr_regadd_rhh_exec(p, t, ip);
4749 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4751 struct thread *t = &p->threads[p->thread_id];
4752 struct instruction *ip = t->ip;
4755 __instr_regadd_rhm_exec(p, t, ip);
4762 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4764 struct thread *t = &p->threads[p->thread_id];
4765 struct instruction *ip = t->ip;
4768 __instr_regadd_rmh_exec(p, t, ip);
4775 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4777 struct thread *t = &p->threads[p->thread_id];
4778 struct instruction *ip = t->ip;
4781 __instr_regadd_rmm_exec(p, t, ip);
4788 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4790 struct thread *t = &p->threads[p->thread_id];
4791 struct instruction *ip = t->ip;
4794 __instr_regadd_rhi_exec(p, t, ip);
4801 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4803 struct thread *t = &p->threads[p->thread_id];
4804 struct instruction *ip = t->ip;
4807 __instr_regadd_rmi_exec(p, t, ip);
4814 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4816 struct thread *t = &p->threads[p->thread_id];
4817 struct instruction *ip = t->ip;
4820 __instr_regadd_rih_exec(p, t, ip);
4827 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4829 struct thread *t = &p->threads[p->thread_id];
4830 struct instruction *ip = t->ip;
4833 __instr_regadd_rim_exec(p, t, ip);
4840 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4842 struct thread *t = &p->threads[p->thread_id];
4843 struct instruction *ip = t->ip;
4846 __instr_regadd_rii_exec(p, t, ip);
4855 static struct metarray *
4856 metarray_find(struct rte_swx_pipeline *p, const char *name);
4859 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4860 struct action *action,
4863 struct instruction *instr,
4864 struct instruction_data *data __rte_unused)
4866 char *metarray = tokens[1], *idx = tokens[2];
4869 uint32_t idx_struct_id, idx_val;
4871 CHECK(n_tokens == 3, EINVAL);
4873 m = metarray_find(p, metarray);
4876 /* METPREFETCH_H, METPREFETCH_M. */
4877 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4879 CHECK(!fidx->var_size, EINVAL);
4881 instr->type = INSTR_METPREFETCH_M;
4883 instr->type = INSTR_METPREFETCH_H;
4885 instr->meter.metarray_id = m->id;
4886 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4887 instr->meter.idx.n_bits = fidx->n_bits;
4888 instr->meter.idx.offset = fidx->offset / 8;
4892 /* METPREFETCH_I. */
4893 idx_val = strtoul(idx, &idx, 0);
4894 CHECK(!idx[0], EINVAL);
4896 instr->type = INSTR_METPREFETCH_I;
4897 instr->meter.metarray_id = m->id;
4898 instr->meter.idx_val = idx_val;
4903 instr_meter_translate(struct rte_swx_pipeline *p,
4904 struct action *action,
4907 struct instruction *instr,
4908 struct instruction_data *data __rte_unused)
4910 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4911 char *color_in = tokens[4], *color_out = tokens[5];
4913 struct field *fidx, *flength, *fcin, *fcout;
4914 uint32_t idx_struct_id, length_struct_id;
4915 uint32_t color_in_struct_id, color_out_struct_id;
4917 CHECK(n_tokens == 6, EINVAL);
4919 m = metarray_find(p, metarray);
4922 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4924 flength = struct_field_parse(p, action, length, &length_struct_id);
4925 CHECK(flength, EINVAL);
4926 CHECK(!flength->var_size, EINVAL);
4928 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4930 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4931 CHECK(fcout, EINVAL);
4932 CHECK(!fcout->var_size, EINVAL);
4934 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4936 CHECK(!fidx->var_size, EINVAL);
4937 CHECK(!fcin->var_size, EINVAL);
4939 instr->type = INSTR_METER_MMM;
4940 if (idx[0] == 'h' && length[0] == 'h')
4941 instr->type = INSTR_METER_HHM;
4942 if (idx[0] == 'h' && length[0] != 'h')
4943 instr->type = INSTR_METER_HMM;
4944 if (idx[0] != 'h' && length[0] == 'h')
4945 instr->type = INSTR_METER_MHM;
4947 instr->meter.metarray_id = m->id;
4949 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4950 instr->meter.idx.n_bits = fidx->n_bits;
4951 instr->meter.idx.offset = fidx->offset / 8;
4953 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4954 instr->meter.length.n_bits = flength->n_bits;
4955 instr->meter.length.offset = flength->offset / 8;
4957 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4958 instr->meter.color_in.n_bits = fcin->n_bits;
4959 instr->meter.color_in.offset = fcin->offset / 8;
4961 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4962 instr->meter.color_out.n_bits = fcout->n_bits;
4963 instr->meter.color_out.offset = fcout->offset / 8;
4966 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4967 if (fidx && !fcin) {
4968 uint32_t color_in_val;
4970 CHECK(!fidx->var_size, EINVAL);
4972 color_in_val = strtoul(color_in, &color_in, 0);
4973 CHECK(!color_in[0], EINVAL);
4975 instr->type = INSTR_METER_MMI;
4976 if (idx[0] == 'h' && length[0] == 'h')
4977 instr->type = INSTR_METER_HHI;
4978 if (idx[0] == 'h' && length[0] != 'h')
4979 instr->type = INSTR_METER_HMI;
4980 if (idx[0] != 'h' && length[0] == 'h')
4981 instr->type = INSTR_METER_MHI;
4983 instr->meter.metarray_id = m->id;
4985 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4986 instr->meter.idx.n_bits = fidx->n_bits;
4987 instr->meter.idx.offset = fidx->offset / 8;
4989 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4990 instr->meter.length.n_bits = flength->n_bits;
4991 instr->meter.length.offset = flength->offset / 8;
4993 instr->meter.color_in_val = color_in_val;
4995 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4996 instr->meter.color_out.n_bits = fcout->n_bits;
4997 instr->meter.color_out.offset = fcout->offset / 8;
5000 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5001 if (!fidx && fcin) {
5004 idx_val = strtoul(idx, &idx, 0);
5005 CHECK(!idx[0], EINVAL);
5007 CHECK(!fcin->var_size, EINVAL);
5009 instr->type = INSTR_METER_IMM;
5010 if (length[0] == 'h')
5011 instr->type = INSTR_METER_IHM;
5013 instr->meter.metarray_id = m->id;
5015 instr->meter.idx_val = idx_val;
5017 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5018 instr->meter.length.n_bits = flength->n_bits;
5019 instr->meter.length.offset = flength->offset / 8;
5021 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5022 instr->meter.color_in.n_bits = fcin->n_bits;
5023 instr->meter.color_in.offset = fcin->offset / 8;
5025 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5026 instr->meter.color_out.n_bits = fcout->n_bits;
5027 instr->meter.color_out.offset = fcout->offset / 8;
5030 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5031 if (!fidx && !fcin) {
5032 uint32_t idx_val, color_in_val;
5034 idx_val = strtoul(idx, &idx, 0);
5035 CHECK(!idx[0], EINVAL);
5037 color_in_val = strtoul(color_in, &color_in, 0);
5038 CHECK(!color_in[0], EINVAL);
5040 instr->type = INSTR_METER_IMI;
5041 if (length[0] == 'h')
5042 instr->type = INSTR_METER_IHI;
5044 instr->meter.metarray_id = m->id;
5046 instr->meter.idx_val = idx_val;
5048 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5049 instr->meter.length.n_bits = flength->n_bits;
5050 instr->meter.length.offset = flength->offset / 8;
5052 instr->meter.color_in_val = color_in_val;
5054 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5055 instr->meter.color_out.n_bits = fcout->n_bits;
5056 instr->meter.color_out.offset = fcout->offset / 8;
5063 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5065 struct thread *t = &p->threads[p->thread_id];
5066 struct instruction *ip = t->ip;
5069 __instr_metprefetch_h_exec(p, t, ip);
5076 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5078 struct thread *t = &p->threads[p->thread_id];
5079 struct instruction *ip = t->ip;
5082 __instr_metprefetch_m_exec(p, t, ip);
5089 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5091 struct thread *t = &p->threads[p->thread_id];
5092 struct instruction *ip = t->ip;
5095 __instr_metprefetch_i_exec(p, t, ip);
5102 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5104 struct thread *t = &p->threads[p->thread_id];
5105 struct instruction *ip = t->ip;
5108 __instr_meter_hhm_exec(p, t, ip);
5115 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5117 struct thread *t = &p->threads[p->thread_id];
5118 struct instruction *ip = t->ip;
5121 __instr_meter_hhi_exec(p, t, ip);
5128 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5130 struct thread *t = &p->threads[p->thread_id];
5131 struct instruction *ip = t->ip;
5134 __instr_meter_hmm_exec(p, t, ip);
5141 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5143 struct thread *t = &p->threads[p->thread_id];
5144 struct instruction *ip = t->ip;
5147 __instr_meter_hmi_exec(p, t, ip);
5154 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5156 struct thread *t = &p->threads[p->thread_id];
5157 struct instruction *ip = t->ip;
5160 __instr_meter_mhm_exec(p, t, ip);
5167 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5169 struct thread *t = &p->threads[p->thread_id];
5170 struct instruction *ip = t->ip;
5173 __instr_meter_mhi_exec(p, t, ip);
5180 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5182 struct thread *t = &p->threads[p->thread_id];
5183 struct instruction *ip = t->ip;
5186 __instr_meter_mmm_exec(p, t, ip);
5193 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5195 struct thread *t = &p->threads[p->thread_id];
5196 struct instruction *ip = t->ip;
5199 __instr_meter_mmi_exec(p, t, ip);
5206 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5208 struct thread *t = &p->threads[p->thread_id];
5209 struct instruction *ip = t->ip;
5212 __instr_meter_ihm_exec(p, t, ip);
5219 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5221 struct thread *t = &p->threads[p->thread_id];
5222 struct instruction *ip = t->ip;
5225 __instr_meter_ihi_exec(p, t, ip);
5232 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5234 struct thread *t = &p->threads[p->thread_id];
5235 struct instruction *ip = t->ip;
5238 __instr_meter_imm_exec(p, t, ip);
5245 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5247 struct thread *t = &p->threads[p->thread_id];
5248 struct instruction *ip = t->ip;
5251 __instr_meter_imi_exec(p, t, ip);
5261 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5262 struct action *action __rte_unused,
5265 struct instruction *instr,
5266 struct instruction_data *data)
5268 CHECK(n_tokens == 2, EINVAL);
5270 strcpy(data->jmp_label, tokens[1]);
5272 instr->type = INSTR_JMP;
5273 instr->jmp.ip = NULL; /* Resolved later. */
5278 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5279 struct action *action __rte_unused,
5282 struct instruction *instr,
5283 struct instruction_data *data)
5287 CHECK(n_tokens == 3, EINVAL);
5289 strcpy(data->jmp_label, tokens[1]);
5291 h = header_parse(p, tokens[2]);
5294 instr->type = INSTR_JMP_VALID;
5295 instr->jmp.ip = NULL; /* Resolved later. */
5296 instr->jmp.header_id = h->id;
5301 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5302 struct action *action __rte_unused,
5305 struct instruction *instr,
5306 struct instruction_data *data)
5310 CHECK(n_tokens == 3, EINVAL);
5312 strcpy(data->jmp_label, tokens[1]);
5314 h = header_parse(p, tokens[2]);
5317 instr->type = INSTR_JMP_INVALID;
5318 instr->jmp.ip = NULL; /* Resolved later. */
5319 instr->jmp.header_id = h->id;
5324 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5325 struct action *action,
5328 struct instruction *instr,
5329 struct instruction_data *data)
5331 CHECK(!action, EINVAL);
5332 CHECK(n_tokens == 2, EINVAL);
5334 strcpy(data->jmp_label, tokens[1]);
5336 instr->type = INSTR_JMP_HIT;
5337 instr->jmp.ip = NULL; /* Resolved later. */
5342 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5343 struct action *action,
5346 struct instruction *instr,
5347 struct instruction_data *data)
5349 CHECK(!action, EINVAL);
5350 CHECK(n_tokens == 2, EINVAL);
5352 strcpy(data->jmp_label, tokens[1]);
5354 instr->type = INSTR_JMP_MISS;
5355 instr->jmp.ip = NULL; /* Resolved later. */
5360 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5361 struct action *action,
5364 struct instruction *instr,
5365 struct instruction_data *data)
5369 CHECK(!action, EINVAL);
5370 CHECK(n_tokens == 3, EINVAL);
5372 strcpy(data->jmp_label, tokens[1]);
5374 a = action_find(p, tokens[2]);
5377 instr->type = INSTR_JMP_ACTION_HIT;
5378 instr->jmp.ip = NULL; /* Resolved later. */
5379 instr->jmp.action_id = a->id;
5384 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5385 struct action *action,
5388 struct instruction *instr,
5389 struct instruction_data *data)
5393 CHECK(!action, EINVAL);
5394 CHECK(n_tokens == 3, EINVAL);
5396 strcpy(data->jmp_label, tokens[1]);
5398 a = action_find(p, tokens[2]);
5401 instr->type = INSTR_JMP_ACTION_MISS;
5402 instr->jmp.ip = NULL; /* Resolved later. */
5403 instr->jmp.action_id = a->id;
5408 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5409 struct action *action,
5412 struct instruction *instr,
5413 struct instruction_data *data)
5415 char *a = tokens[2], *b = tokens[3];
5416 struct field *fa, *fb;
5418 uint32_t a_struct_id, b_struct_id;
5420 CHECK(n_tokens == 4, EINVAL);
5422 strcpy(data->jmp_label, tokens[1]);
5424 fa = struct_field_parse(p, action, a, &a_struct_id);
5426 CHECK(!fa->var_size, EINVAL);
5428 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5429 fb = struct_field_parse(p, action, b, &b_struct_id);
5431 CHECK(!fb->var_size, EINVAL);
5433 instr->type = INSTR_JMP_EQ;
5434 if (a[0] != 'h' && b[0] == 'h')
5435 instr->type = INSTR_JMP_EQ_MH;
5436 if (a[0] == 'h' && b[0] != 'h')
5437 instr->type = INSTR_JMP_EQ_HM;
5438 if (a[0] == 'h' && b[0] == 'h')
5439 instr->type = INSTR_JMP_EQ_HH;
5440 instr->jmp.ip = NULL; /* Resolved later. */
5442 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5443 instr->jmp.a.n_bits = fa->n_bits;
5444 instr->jmp.a.offset = fa->offset / 8;
5445 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5446 instr->jmp.b.n_bits = fb->n_bits;
5447 instr->jmp.b.offset = fb->offset / 8;
5452 b_val = strtoull(b, &b, 0);
5453 CHECK(!b[0], EINVAL);
5456 b_val = hton64(b_val) >> (64 - fa->n_bits);
5458 instr->type = INSTR_JMP_EQ_I;
5459 instr->jmp.ip = NULL; /* Resolved later. */
5460 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5461 instr->jmp.a.n_bits = fa->n_bits;
5462 instr->jmp.a.offset = fa->offset / 8;
5463 instr->jmp.b_val = b_val;
5468 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5469 struct action *action,
5472 struct instruction *instr,
5473 struct instruction_data *data)
5475 char *a = tokens[2], *b = tokens[3];
5476 struct field *fa, *fb;
5478 uint32_t a_struct_id, b_struct_id;
5480 CHECK(n_tokens == 4, EINVAL);
5482 strcpy(data->jmp_label, tokens[1]);
5484 fa = struct_field_parse(p, action, a, &a_struct_id);
5486 CHECK(!fa->var_size, EINVAL);
5488 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5489 fb = struct_field_parse(p, action, b, &b_struct_id);
5491 CHECK(!fb->var_size, EINVAL);
5493 instr->type = INSTR_JMP_NEQ;
5494 if (a[0] != 'h' && b[0] == 'h')
5495 instr->type = INSTR_JMP_NEQ_MH;
5496 if (a[0] == 'h' && b[0] != 'h')
5497 instr->type = INSTR_JMP_NEQ_HM;
5498 if (a[0] == 'h' && b[0] == 'h')
5499 instr->type = INSTR_JMP_NEQ_HH;
5500 instr->jmp.ip = NULL; /* Resolved later. */
5502 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5503 instr->jmp.a.n_bits = fa->n_bits;
5504 instr->jmp.a.offset = fa->offset / 8;
5505 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5506 instr->jmp.b.n_bits = fb->n_bits;
5507 instr->jmp.b.offset = fb->offset / 8;
5512 b_val = strtoull(b, &b, 0);
5513 CHECK(!b[0], EINVAL);
5516 b_val = hton64(b_val) >> (64 - fa->n_bits);
5518 instr->type = INSTR_JMP_NEQ_I;
5519 instr->jmp.ip = NULL; /* Resolved later. */
5520 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5521 instr->jmp.a.n_bits = fa->n_bits;
5522 instr->jmp.a.offset = fa->offset / 8;
5523 instr->jmp.b_val = b_val;
5528 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5529 struct action *action,
5532 struct instruction *instr,
5533 struct instruction_data *data)
5535 char *a = tokens[2], *b = tokens[3];
5536 struct field *fa, *fb;
5538 uint32_t a_struct_id, b_struct_id;
5540 CHECK(n_tokens == 4, EINVAL);
5542 strcpy(data->jmp_label, tokens[1]);
5544 fa = struct_field_parse(p, action, a, &a_struct_id);
5546 CHECK(!fa->var_size, EINVAL);
5548 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5549 fb = struct_field_parse(p, action, b, &b_struct_id);
5551 CHECK(!fb->var_size, EINVAL);
5553 instr->type = INSTR_JMP_LT;
5554 if (a[0] == 'h' && b[0] != 'h')
5555 instr->type = INSTR_JMP_LT_HM;
5556 if (a[0] != 'h' && b[0] == 'h')
5557 instr->type = INSTR_JMP_LT_MH;
5558 if (a[0] == 'h' && b[0] == 'h')
5559 instr->type = INSTR_JMP_LT_HH;
5560 instr->jmp.ip = NULL; /* Resolved later. */
5562 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5563 instr->jmp.a.n_bits = fa->n_bits;
5564 instr->jmp.a.offset = fa->offset / 8;
5565 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5566 instr->jmp.b.n_bits = fb->n_bits;
5567 instr->jmp.b.offset = fb->offset / 8;
5571 /* JMP_LT_MI, JMP_LT_HI. */
5572 b_val = strtoull(b, &b, 0);
5573 CHECK(!b[0], EINVAL);
5575 instr->type = INSTR_JMP_LT_MI;
5577 instr->type = INSTR_JMP_LT_HI;
5578 instr->jmp.ip = NULL; /* Resolved later. */
5580 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5581 instr->jmp.a.n_bits = fa->n_bits;
5582 instr->jmp.a.offset = fa->offset / 8;
5583 instr->jmp.b_val = b_val;
5588 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5589 struct action *action,
5592 struct instruction *instr,
5593 struct instruction_data *data)
5595 char *a = tokens[2], *b = tokens[3];
5596 struct field *fa, *fb;
5598 uint32_t a_struct_id, b_struct_id;
5600 CHECK(n_tokens == 4, EINVAL);
5602 strcpy(data->jmp_label, tokens[1]);
5604 fa = struct_field_parse(p, action, a, &a_struct_id);
5606 CHECK(!fa->var_size, EINVAL);
5608 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5609 fb = struct_field_parse(p, action, b, &b_struct_id);
5611 CHECK(!fb->var_size, EINVAL);
5613 instr->type = INSTR_JMP_GT;
5614 if (a[0] == 'h' && b[0] != 'h')
5615 instr->type = INSTR_JMP_GT_HM;
5616 if (a[0] != 'h' && b[0] == 'h')
5617 instr->type = INSTR_JMP_GT_MH;
5618 if (a[0] == 'h' && b[0] == 'h')
5619 instr->type = INSTR_JMP_GT_HH;
5620 instr->jmp.ip = NULL; /* Resolved later. */
5622 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5623 instr->jmp.a.n_bits = fa->n_bits;
5624 instr->jmp.a.offset = fa->offset / 8;
5625 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5626 instr->jmp.b.n_bits = fb->n_bits;
5627 instr->jmp.b.offset = fb->offset / 8;
5631 /* JMP_GT_MI, JMP_GT_HI. */
5632 b_val = strtoull(b, &b, 0);
5633 CHECK(!b[0], EINVAL);
5635 instr->type = INSTR_JMP_GT_MI;
5637 instr->type = INSTR_JMP_GT_HI;
5638 instr->jmp.ip = NULL; /* Resolved later. */
5640 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5641 instr->jmp.a.n_bits = fa->n_bits;
5642 instr->jmp.a.offset = fa->offset / 8;
5643 instr->jmp.b_val = b_val;
5648 instr_jmp_exec(struct rte_swx_pipeline *p)
5650 struct thread *t = &p->threads[p->thread_id];
5651 struct instruction *ip = t->ip;
5653 TRACE("[Thread %2u] jmp\n", p->thread_id);
5655 thread_ip_set(t, ip->jmp.ip);
5659 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5661 struct thread *t = &p->threads[p->thread_id];
5662 struct instruction *ip = t->ip;
5663 uint32_t header_id = ip->jmp.header_id;
5665 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5667 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5671 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5673 struct thread *t = &p->threads[p->thread_id];
5674 struct instruction *ip = t->ip;
5675 uint32_t header_id = ip->jmp.header_id;
5677 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5679 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5683 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5685 struct thread *t = &p->threads[p->thread_id];
5686 struct instruction *ip = t->ip;
5687 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5689 TRACE("[Thread %2u] jmph\n", p->thread_id);
5691 t->ip = ip_next[t->hit];
5695 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5697 struct thread *t = &p->threads[p->thread_id];
5698 struct instruction *ip = t->ip;
5699 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5701 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5703 t->ip = ip_next[t->hit];
5707 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5709 struct thread *t = &p->threads[p->thread_id];
5710 struct instruction *ip = t->ip;
5712 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5714 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5718 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5720 struct thread *t = &p->threads[p->thread_id];
5721 struct instruction *ip = t->ip;
5723 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5725 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5729 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5731 struct thread *t = &p->threads[p->thread_id];
5732 struct instruction *ip = t->ip;
5734 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5740 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5742 struct thread *t = &p->threads[p->thread_id];
5743 struct instruction *ip = t->ip;
5745 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5747 JMP_CMP_MH(t, ip, ==);
5751 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5753 struct thread *t = &p->threads[p->thread_id];
5754 struct instruction *ip = t->ip;
5756 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5758 JMP_CMP_HM(t, ip, ==);
5762 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5764 struct thread *t = &p->threads[p->thread_id];
5765 struct instruction *ip = t->ip;
5767 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5769 JMP_CMP_HH_FAST(t, ip, ==);
5773 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5775 struct thread *t = &p->threads[p->thread_id];
5776 struct instruction *ip = t->ip;
5778 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5780 JMP_CMP_I(t, ip, ==);
5784 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5786 struct thread *t = &p->threads[p->thread_id];
5787 struct instruction *ip = t->ip;
5789 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5795 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5797 struct thread *t = &p->threads[p->thread_id];
5798 struct instruction *ip = t->ip;
5800 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5802 JMP_CMP_MH(t, ip, !=);
5806 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5808 struct thread *t = &p->threads[p->thread_id];
5809 struct instruction *ip = t->ip;
5811 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5813 JMP_CMP_HM(t, ip, !=);
5817 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5819 struct thread *t = &p->threads[p->thread_id];
5820 struct instruction *ip = t->ip;
5822 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5824 JMP_CMP_HH_FAST(t, ip, !=);
5828 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5830 struct thread *t = &p->threads[p->thread_id];
5831 struct instruction *ip = t->ip;
5833 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5835 JMP_CMP_I(t, ip, !=);
5839 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5841 struct thread *t = &p->threads[p->thread_id];
5842 struct instruction *ip = t->ip;
5844 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5850 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5852 struct thread *t = &p->threads[p->thread_id];
5853 struct instruction *ip = t->ip;
5855 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5857 JMP_CMP_MH(t, ip, <);
5861 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5863 struct thread *t = &p->threads[p->thread_id];
5864 struct instruction *ip = t->ip;
5866 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5868 JMP_CMP_HM(t, ip, <);
5872 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5874 struct thread *t = &p->threads[p->thread_id];
5875 struct instruction *ip = t->ip;
5877 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5879 JMP_CMP_HH(t, ip, <);
5883 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5885 struct thread *t = &p->threads[p->thread_id];
5886 struct instruction *ip = t->ip;
5888 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5890 JMP_CMP_MI(t, ip, <);
5894 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5896 struct thread *t = &p->threads[p->thread_id];
5897 struct instruction *ip = t->ip;
5899 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5901 JMP_CMP_HI(t, ip, <);
5905 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5907 struct thread *t = &p->threads[p->thread_id];
5908 struct instruction *ip = t->ip;
5910 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5916 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5918 struct thread *t = &p->threads[p->thread_id];
5919 struct instruction *ip = t->ip;
5921 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5923 JMP_CMP_MH(t, ip, >);
5927 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5929 struct thread *t = &p->threads[p->thread_id];
5930 struct instruction *ip = t->ip;
5932 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5934 JMP_CMP_HM(t, ip, >);
5938 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5940 struct thread *t = &p->threads[p->thread_id];
5941 struct instruction *ip = t->ip;
5943 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5945 JMP_CMP_HH(t, ip, >);
5949 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5951 struct thread *t = &p->threads[p->thread_id];
5952 struct instruction *ip = t->ip;
5954 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5956 JMP_CMP_MI(t, ip, >);
5960 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5962 struct thread *t = &p->threads[p->thread_id];
5963 struct instruction *ip = t->ip;
5965 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5967 JMP_CMP_HI(t, ip, >);
5974 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5975 struct action *action,
5976 char **tokens __rte_unused,
5978 struct instruction *instr,
5979 struct instruction_data *data __rte_unused)
5981 CHECK(action, EINVAL);
5982 CHECK(n_tokens == 1, EINVAL);
5984 instr->type = INSTR_RETURN;
5989 instr_return_exec(struct rte_swx_pipeline *p)
5991 struct thread *t = &p->threads[p->thread_id];
5993 TRACE("[Thread %2u] return\n", p->thread_id);
5999 instr_translate(struct rte_swx_pipeline *p,
6000 struct action *action,
6002 struct instruction *instr,
6003 struct instruction_data *data)
6005 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6006 int n_tokens = 0, tpos = 0;
6008 /* Parse the instruction string into tokens. */
6012 token = strtok_r(string, " \t\v", &string);
6016 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6017 CHECK_NAME(token, EINVAL);
6019 tokens[n_tokens] = token;
6023 CHECK(n_tokens, EINVAL);
6025 /* Handle the optional instruction label. */
6026 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6027 strcpy(data->label, tokens[0]);
6030 CHECK(n_tokens - tpos, EINVAL);
6033 /* Identify the instruction type. */
6034 if (!strcmp(tokens[tpos], "rx"))
6035 return instr_rx_translate(p,
6042 if (!strcmp(tokens[tpos], "tx"))
6043 return instr_tx_translate(p,
6050 if (!strcmp(tokens[tpos], "drop"))
6051 return instr_drop_translate(p,
6058 if (!strcmp(tokens[tpos], "mirror"))
6059 return instr_mirror_translate(p,
6066 if (!strcmp(tokens[tpos], "recirculate"))
6067 return instr_recirculate_translate(p,
6074 if (!strcmp(tokens[tpos], "recircid"))
6075 return instr_recircid_translate(p,
6082 if (!strcmp(tokens[tpos], "extract"))
6083 return instr_hdr_extract_translate(p,
6090 if (!strcmp(tokens[tpos], "lookahead"))
6091 return instr_hdr_lookahead_translate(p,
6098 if (!strcmp(tokens[tpos], "emit"))
6099 return instr_hdr_emit_translate(p,
6106 if (!strcmp(tokens[tpos], "validate"))
6107 return instr_hdr_validate_translate(p,
6114 if (!strcmp(tokens[tpos], "invalidate"))
6115 return instr_hdr_invalidate_translate(p,
6122 if (!strcmp(tokens[tpos], "mov"))
6123 return instr_mov_translate(p,
6130 if (!strcmp(tokens[tpos], "add"))
6131 return instr_alu_add_translate(p,
6138 if (!strcmp(tokens[tpos], "sub"))
6139 return instr_alu_sub_translate(p,
6146 if (!strcmp(tokens[tpos], "ckadd"))
6147 return instr_alu_ckadd_translate(p,
6154 if (!strcmp(tokens[tpos], "cksub"))
6155 return instr_alu_cksub_translate(p,
6162 if (!strcmp(tokens[tpos], "and"))
6163 return instr_alu_and_translate(p,
6170 if (!strcmp(tokens[tpos], "or"))
6171 return instr_alu_or_translate(p,
6178 if (!strcmp(tokens[tpos], "xor"))
6179 return instr_alu_xor_translate(p,
6186 if (!strcmp(tokens[tpos], "shl"))
6187 return instr_alu_shl_translate(p,
6194 if (!strcmp(tokens[tpos], "shr"))
6195 return instr_alu_shr_translate(p,
6202 if (!strcmp(tokens[tpos], "regprefetch"))
6203 return instr_regprefetch_translate(p,
6210 if (!strcmp(tokens[tpos], "regrd"))
6211 return instr_regrd_translate(p,
6218 if (!strcmp(tokens[tpos], "regwr"))
6219 return instr_regwr_translate(p,
6226 if (!strcmp(tokens[tpos], "regadd"))
6227 return instr_regadd_translate(p,
6234 if (!strcmp(tokens[tpos], "metprefetch"))
6235 return instr_metprefetch_translate(p,
6242 if (!strcmp(tokens[tpos], "meter"))
6243 return instr_meter_translate(p,
6250 if (!strcmp(tokens[tpos], "table"))
6251 return instr_table_translate(p,
6258 if (!strcmp(tokens[tpos], "learn"))
6259 return instr_learn_translate(p,
6265 if (!strcmp(tokens[tpos], "rearm"))
6266 return instr_rearm_translate(p,
6273 if (!strcmp(tokens[tpos], "forget"))
6274 return instr_forget_translate(p,
6281 if (!strcmp(tokens[tpos], "extern"))
6282 return instr_extern_translate(p,
6289 if (!strcmp(tokens[tpos], "hash"))
6290 return instr_hash_translate(p,
6297 if (!strcmp(tokens[tpos], "jmp"))
6298 return instr_jmp_translate(p,
6305 if (!strcmp(tokens[tpos], "jmpv"))
6306 return instr_jmp_valid_translate(p,
6313 if (!strcmp(tokens[tpos], "jmpnv"))
6314 return instr_jmp_invalid_translate(p,
6321 if (!strcmp(tokens[tpos], "jmph"))
6322 return instr_jmp_hit_translate(p,
6329 if (!strcmp(tokens[tpos], "jmpnh"))
6330 return instr_jmp_miss_translate(p,
6337 if (!strcmp(tokens[tpos], "jmpa"))
6338 return instr_jmp_action_hit_translate(p,
6345 if (!strcmp(tokens[tpos], "jmpna"))
6346 return instr_jmp_action_miss_translate(p,
6353 if (!strcmp(tokens[tpos], "jmpeq"))
6354 return instr_jmp_eq_translate(p,
6361 if (!strcmp(tokens[tpos], "jmpneq"))
6362 return instr_jmp_neq_translate(p,
6369 if (!strcmp(tokens[tpos], "jmplt"))
6370 return instr_jmp_lt_translate(p,
6377 if (!strcmp(tokens[tpos], "jmpgt"))
6378 return instr_jmp_gt_translate(p,
6385 if (!strcmp(tokens[tpos], "return"))
6386 return instr_return_translate(p,
6396 static struct instruction_data *
6397 label_find(struct instruction_data *data, uint32_t n, const char *label)
6401 for (i = 0; i < n; i++)
6402 if (!strcmp(label, data[i].label))
6409 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6411 uint32_t count = 0, i;
6416 for (i = 0; i < n; i++)
6417 if (!strcmp(label, data[i].jmp_label))
6424 instr_label_check(struct instruction_data *instruction_data,
6425 uint32_t n_instructions)
6429 /* Check that all instruction labels are unique. */
6430 for (i = 0; i < n_instructions; i++) {
6431 struct instruction_data *data = &instruction_data[i];
6432 char *label = data->label;
6438 for (j = i + 1; j < n_instructions; j++)
6439 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6442 /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6443 for (i = 0; i < n_instructions; i++) {
6444 struct instruction_data *data = &instruction_data[i];
6445 char *label = data->label;
6446 char *jmp_label = data->jmp_label;
6448 /* Continue if this instruction does not have a label or it is not a jump. */
6449 if (!label[0] || !jmp_label[0])
6452 CHECK(strcmp(label, jmp_label), EINVAL);
6455 /* Get users for each instruction label. */
6456 for (i = 0; i < n_instructions; i++) {
6457 struct instruction_data *data = &instruction_data[i];
6458 char *label = data->label;
6460 data->n_users = label_is_used(instruction_data,
6469 instr_jmp_resolve(struct instruction *instructions,
6470 struct instruction_data *instruction_data,
6471 uint32_t n_instructions)
6475 for (i = 0; i < n_instructions; i++) {
6476 struct instruction *instr = &instructions[i];
6477 struct instruction_data *data = &instruction_data[i];
6478 struct instruction_data *found;
6480 if (!instruction_is_jmp(instr))
6483 found = label_find(instruction_data,
6486 CHECK(found, EINVAL);
6488 instr->jmp.ip = &instructions[found - instruction_data];
6495 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6497 struct instruction *instr,
6498 struct instruction_data *data __rte_unused,
6499 uint32_t n_instructions)
6502 enum instruction_type type;
6505 /* Check that the first instruction is rx. */
6506 CHECK(instr[0].type == INSTR_RX, EINVAL);
6508 /* Check that there is at least one tx instruction. */
6509 for (i = 0; i < n_instructions; i++) {
6510 type = instr[i].type;
6512 if (instruction_is_tx(type))
6515 CHECK(i < n_instructions, EINVAL);
6517 /* Check that the last instruction is either tx or unconditional
6520 type = instr[n_instructions - 1].type;
6521 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6525 enum instruction_type type;
6528 /* Check that there is at least one return or tx instruction. */
6529 for (i = 0; i < n_instructions; i++) {
6530 type = instr[i].type;
6532 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6535 CHECK(i < n_instructions, EINVAL);
6542 instr_compact(struct instruction *instructions,
6543 struct instruction_data *instruction_data,
6544 uint32_t n_instructions)
6546 uint32_t i, pos = 0;
6548 /* Eliminate the invalid instructions that have been optimized out. */
6549 for (i = 0; i < n_instructions; i++) {
6550 struct instruction *instr = &instructions[i];
6551 struct instruction_data *data = &instruction_data[i];
6557 memcpy(&instructions[pos], instr, sizeof(*instr));
6558 memcpy(&instruction_data[pos], data, sizeof(*data));
6568 instr_pattern_extract_many_search(struct instruction *instr,
6569 struct instruction_data *data,
6571 uint32_t *n_pattern_instr)
6575 for (i = 0; i < n_instr; i++) {
6576 if (data[i].invalid)
6579 if (instr[i].type != INSTR_HDR_EXTRACT)
6582 if (i == RTE_DIM(instr->io.hdr.header_id))
6585 if (i && data[i].n_users)
6592 *n_pattern_instr = i;
6597 instr_pattern_extract_many_replace(struct instruction *instr,
6598 struct instruction_data *data,
6603 for (i = 1; i < n_instr; i++) {
6605 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6606 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6607 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6609 data[i].invalid = 1;
6614 instr_pattern_extract_many_optimize(struct instruction *instructions,
6615 struct instruction_data *instruction_data,
6616 uint32_t n_instructions)
6620 for (i = 0; i < n_instructions; ) {
6621 struct instruction *instr = &instructions[i];
6622 struct instruction_data *data = &instruction_data[i];
6623 uint32_t n_instr = 0;
6627 detected = instr_pattern_extract_many_search(instr,
6632 instr_pattern_extract_many_replace(instr,
6639 /* No pattern starting at the current instruction. */
6643 /* Eliminate the invalid instructions that have been optimized out. */
6644 n_instructions = instr_compact(instructions,
6648 return n_instructions;
6652 instr_pattern_emit_many_tx_search(struct instruction *instr,
6653 struct instruction_data *data,
6655 uint32_t *n_pattern_instr)
6659 for (i = 0; i < n_instr; i++) {
6660 if (data[i].invalid)
6663 if (instr[i].type != INSTR_HDR_EMIT)
6666 if (i == RTE_DIM(instr->io.hdr.header_id))
6669 if (i && data[i].n_users)
6676 if (instr[i].type != INSTR_TX)
6679 if (data[i].n_users)
6684 *n_pattern_instr = i;
6689 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6690 struct instruction_data *data,
6695 /* Any emit instruction in addition to the first one. */
6696 for (i = 1; i < n_instr - 1; i++) {
6698 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6699 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6700 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6702 data[i].invalid = 1;
6705 /* The TX instruction is the last one in the pattern. */
6707 instr[0].io.io.offset = instr[i].io.io.offset;
6708 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6709 data[i].invalid = 1;
6713 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6714 struct instruction_data *instruction_data,
6715 uint32_t n_instructions)
6719 for (i = 0; i < n_instructions; ) {
6720 struct instruction *instr = &instructions[i];
6721 struct instruction_data *data = &instruction_data[i];
6722 uint32_t n_instr = 0;
6725 /* Emit many + TX. */
6726 detected = instr_pattern_emit_many_tx_search(instr,
6731 instr_pattern_emit_many_tx_replace(instr,
6738 /* No pattern starting at the current instruction. */
6742 /* Eliminate the invalid instructions that have been optimized out. */
6743 n_instructions = instr_compact(instructions,
6747 return n_instructions;
6751 action_arg_src_mov_count(struct action *a,
6753 struct instruction *instructions,
6754 struct instruction_data *instruction_data,
6755 uint32_t n_instructions);
6758 instr_pattern_validate_mov_all_search(struct rte_swx_pipeline *p,
6760 struct instruction *instr,
6761 struct instruction_data *data,
6763 struct instruction *instructions,
6764 struct instruction_data *instruction_data,
6765 uint32_t n_instructions,
6766 uint32_t *n_pattern_instr)
6769 uint32_t src_field_id, i, j;
6771 /* Prerequisites. */
6775 /* First instruction: HDR_VALIDATE. Second instruction: MOV_HM. */
6776 if (data[0].invalid ||
6777 (instr[0].type != INSTR_HDR_VALIDATE) ||
6780 (instr[1].type != INSTR_MOV_HM) ||
6781 instr[1].mov.src.struct_id)
6784 h = header_find_by_struct_id(p, instr[0].valid.struct_id);
6787 (n_instr < 1 + h->st->n_fields))
6790 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6791 if (instr[1].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6794 if (src_field_id + h->st->n_fields > a->st->n_fields)
6797 /* Second and subsequent instructions: MOV_HM. */
6798 for (i = 0; i < h->st->n_fields; i++)
6799 if (data[1 + i].invalid ||
6800 data[1 + i].n_users ||
6801 (instr[1 + i].type != INSTR_MOV_HM) ||
6802 (instr[1 + i].mov.dst.struct_id != h->struct_id) ||
6803 (instr[1 + i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6804 (instr[1 + i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6805 instr[1 + i].mov.src.struct_id ||
6806 (instr[1 + i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6807 (instr[1 + i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6808 (instr[1 + i].mov.dst.n_bits != instr[1 + i].mov.src.n_bits))
6811 /* Check that none of the action args that are used as source for this
6812 * DMA transfer are not used as source in any other mov instruction.
6814 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6817 n_users = action_arg_src_mov_count(a,
6826 *n_pattern_instr = 1 + h->st->n_fields;
6831 instr_pattern_validate_mov_all_replace(struct rte_swx_pipeline *p,
6833 struct instruction *instr,
6834 struct instruction_data *data,
6838 uint32_t src_field_id, src_offset, i;
6840 /* Read from the instructions before they are modified. */
6841 h = header_find_by_struct_id(p, instr[1].mov.dst.struct_id);
6845 src_offset = instr[1].mov.src.offset;
6847 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6848 if (src_offset == a->st->fields[src_field_id].offset / 8)
6851 /* Modify the instructions. */
6852 instr[0].type = INSTR_DMA_HT;
6853 instr[0].dma.dst.header_id[0] = h->id;
6854 instr[0].dma.dst.struct_id[0] = h->struct_id;
6855 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6856 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6858 for (i = 1; i < n_instr; i++)
6859 data[i].invalid = 1;
6861 /* Update the endianness of the action arguments to header endianness. */
6862 for (i = 0; i < h->st->n_fields; i++)
6863 a->args_endianness[src_field_id + i] = 1;
6867 instr_pattern_validate_mov_all_optimize(struct rte_swx_pipeline *p,
6869 struct instruction *instructions,
6870 struct instruction_data *instruction_data,
6871 uint32_t n_instructions)
6876 return n_instructions;
6878 for (i = 0; i < n_instructions; ) {
6879 struct instruction *instr = &instructions[i];
6880 struct instruction_data *data = &instruction_data[i];
6881 uint32_t n_instr = 0;
6884 /* Validate + mov all. */
6885 detected = instr_pattern_validate_mov_all_search(p,
6895 instr_pattern_validate_mov_all_replace(p, a, instr, data, n_instr);
6900 /* No pattern starting at the current instruction. */
6904 /* Eliminate the invalid instructions that have been optimized out. */
6905 n_instructions = instr_compact(instructions,
6909 return n_instructions;
6913 instr_pattern_dma_many_search(struct instruction *instr,
6914 struct instruction_data *data,
6916 uint32_t *n_pattern_instr)
6920 for (i = 0; i < n_instr; i++) {
6921 if (data[i].invalid)
6924 if (instr[i].type != INSTR_DMA_HT)
6927 if (i == RTE_DIM(instr->dma.dst.header_id))
6930 if (i && data[i].n_users)
6937 *n_pattern_instr = i;
6942 instr_pattern_dma_many_replace(struct instruction *instr,
6943 struct instruction_data *data,
6948 for (i = 1; i < n_instr; i++) {
6950 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6951 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6952 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6953 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6955 data[i].invalid = 1;
6960 instr_pattern_dma_many_optimize(struct instruction *instructions,
6961 struct instruction_data *instruction_data,
6962 uint32_t n_instructions)
6966 for (i = 0; i < n_instructions; ) {
6967 struct instruction *instr = &instructions[i];
6968 struct instruction_data *data = &instruction_data[i];
6969 uint32_t n_instr = 0;
6973 detected = instr_pattern_dma_many_search(instr,
6978 instr_pattern_dma_many_replace(instr, data, n_instr);
6983 /* No pattern starting at the current instruction. */
6987 /* Eliminate the invalid instructions that have been optimized out. */
6988 n_instructions = instr_compact(instructions,
6992 return n_instructions;
6996 instr_optimize(struct rte_swx_pipeline *p,
6998 struct instruction *instructions,
6999 struct instruction_data *instruction_data,
7000 uint32_t n_instructions)
7003 n_instructions = instr_pattern_extract_many_optimize(instructions,
7007 /* Emit many + TX. */
7008 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7012 /* Validate + mov all. */
7013 n_instructions = instr_pattern_validate_mov_all_optimize(p,
7020 n_instructions = instr_pattern_dma_many_optimize(instructions,
7024 return n_instructions;
7028 instruction_config(struct rte_swx_pipeline *p,
7030 const char **instructions,
7031 uint32_t n_instructions)
7033 struct instruction *instr = NULL;
7034 struct instruction_data *data = NULL;
7038 CHECK(n_instructions, EINVAL);
7039 CHECK(instructions, EINVAL);
7040 for (i = 0; i < n_instructions; i++)
7041 CHECK_INSTRUCTION(instructions[i], EINVAL);
7043 /* Memory allocation. */
7044 instr = calloc(n_instructions, sizeof(struct instruction));
7050 data = calloc(n_instructions, sizeof(struct instruction_data));
7056 for (i = 0; i < n_instructions; i++) {
7057 char *string = strdup(instructions[i]);
7063 err = instr_translate(p, a, string, &instr[i], &data[i]);
7072 err = instr_label_check(data, n_instructions);
7076 err = instr_verify(p, a, instr, data, n_instructions);
7080 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7082 err = instr_jmp_resolve(instr, data, n_instructions);
7087 a->instructions = instr;
7088 a->instruction_data = data;
7089 a->n_instructions = n_instructions;
7091 p->instructions = instr;
7092 p->instruction_data = data;
7093 p->n_instructions = n_instructions;
7104 static instr_exec_t instruction_table[] = {
7105 [INSTR_RX] = instr_rx_exec,
7106 [INSTR_TX] = instr_tx_exec,
7107 [INSTR_TX_I] = instr_tx_i_exec,
7108 [INSTR_DROP] = instr_drop_exec,
7109 [INSTR_MIRROR] = instr_mirror_exec,
7110 [INSTR_RECIRCULATE] = instr_recirculate_exec,
7111 [INSTR_RECIRCID] = instr_recircid_exec,
7113 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7114 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7115 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7116 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7117 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7118 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7119 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7120 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7121 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7122 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7124 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7125 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7126 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7127 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7128 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7129 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7130 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7131 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7132 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7134 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7135 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7137 [INSTR_MOV] = instr_mov_exec,
7138 [INSTR_MOV_MH] = instr_mov_mh_exec,
7139 [INSTR_MOV_HM] = instr_mov_hm_exec,
7140 [INSTR_MOV_HH] = instr_mov_hh_exec,
7141 [INSTR_MOV_I] = instr_mov_i_exec,
7143 [INSTR_DMA_HT] = instr_dma_ht_exec,
7144 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7145 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7146 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7147 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7148 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7149 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7150 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7152 [INSTR_ALU_ADD] = instr_alu_add_exec,
7153 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7154 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7155 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7156 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7157 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7159 [INSTR_ALU_SUB] = instr_alu_sub_exec,
7160 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7161 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7162 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7163 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7164 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7166 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7167 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7168 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7169 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7171 [INSTR_ALU_AND] = instr_alu_and_exec,
7172 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7173 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7174 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7175 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7177 [INSTR_ALU_OR] = instr_alu_or_exec,
7178 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7179 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7180 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7181 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7183 [INSTR_ALU_XOR] = instr_alu_xor_exec,
7184 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7185 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7186 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7187 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7189 [INSTR_ALU_SHL] = instr_alu_shl_exec,
7190 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7191 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7192 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7193 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7194 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7196 [INSTR_ALU_SHR] = instr_alu_shr_exec,
7197 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7198 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7199 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7200 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7201 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7203 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7204 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7205 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7207 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7208 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7209 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7210 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7211 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7212 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7214 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7215 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7216 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7217 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7218 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7219 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7220 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7221 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7222 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7224 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7225 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7226 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7227 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7228 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7229 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7230 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7231 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7232 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7234 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7235 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7236 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7238 [INSTR_METER_HHM] = instr_meter_hhm_exec,
7239 [INSTR_METER_HHI] = instr_meter_hhi_exec,
7240 [INSTR_METER_HMM] = instr_meter_hmm_exec,
7241 [INSTR_METER_HMI] = instr_meter_hmi_exec,
7242 [INSTR_METER_MHM] = instr_meter_mhm_exec,
7243 [INSTR_METER_MHI] = instr_meter_mhi_exec,
7244 [INSTR_METER_MMM] = instr_meter_mmm_exec,
7245 [INSTR_METER_MMI] = instr_meter_mmi_exec,
7246 [INSTR_METER_IHM] = instr_meter_ihm_exec,
7247 [INSTR_METER_IHI] = instr_meter_ihi_exec,
7248 [INSTR_METER_IMM] = instr_meter_imm_exec,
7249 [INSTR_METER_IMI] = instr_meter_imi_exec,
7251 [INSTR_TABLE] = instr_table_exec,
7252 [INSTR_TABLE_AF] = instr_table_af_exec,
7253 [INSTR_SELECTOR] = instr_selector_exec,
7254 [INSTR_LEARNER] = instr_learner_exec,
7255 [INSTR_LEARNER_AF] = instr_learner_af_exec,
7256 [INSTR_LEARNER_LEARN] = instr_learn_exec,
7257 [INSTR_LEARNER_REARM] = instr_rearm_exec,
7258 [INSTR_LEARNER_REARM_NEW] = instr_rearm_new_exec,
7259 [INSTR_LEARNER_FORGET] = instr_forget_exec,
7260 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7261 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7262 [INSTR_HASH_FUNC] = instr_hash_func_exec,
7264 [INSTR_JMP] = instr_jmp_exec,
7265 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7266 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7267 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7268 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7269 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7270 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7272 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7273 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7274 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7275 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7276 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7278 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7279 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7280 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7281 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7282 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7284 [INSTR_JMP_LT] = instr_jmp_lt_exec,
7285 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7286 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7287 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7288 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7289 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7291 [INSTR_JMP_GT] = instr_jmp_gt_exec,
7292 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7293 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7294 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7295 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7296 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7298 [INSTR_RETURN] = instr_return_exec,
7302 instruction_table_build(struct rte_swx_pipeline *p)
7304 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7305 sizeof(struct instr_exec_t *));
7306 if (!p->instruction_table)
7309 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7315 instruction_table_build_free(struct rte_swx_pipeline *p)
7317 if (!p->instruction_table)
7320 free(p->instruction_table);
7321 p->instruction_table = NULL;
7325 instruction_table_free(struct rte_swx_pipeline *p)
7327 instruction_table_build_free(p);
7331 instr_exec(struct rte_swx_pipeline *p)
7333 struct thread *t = &p->threads[p->thread_id];
7334 struct instruction *ip = t->ip;
7335 instr_exec_t instr = p->instruction_table[ip->type];
7343 static struct action *
7344 action_find(struct rte_swx_pipeline *p, const char *name)
7346 struct action *elem;
7351 TAILQ_FOREACH(elem, &p->actions, node)
7352 if (strcmp(elem->name, name) == 0)
7358 static struct action *
7359 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7361 struct action *action = NULL;
7363 TAILQ_FOREACH(action, &p->actions, node)
7364 if (action->id == id)
7370 static struct field *
7371 action_field_find(struct action *a, const char *name)
7373 return a->st ? struct_type_field_find(a->st, name) : NULL;
7376 static struct field *
7377 action_field_parse(struct action *action, const char *name)
7379 if (name[0] != 't' || name[1] != '.')
7382 return action_field_find(action, &name[2]);
7386 action_has_nbo_args(struct action *a)
7390 /* Return if the action does not have any args. */
7392 return 0; /* FALSE */
7394 for (i = 0; i < a->st->n_fields; i++)
7395 if (a->args_endianness[i])
7396 return 1; /* TRUE */
7398 return 0; /* FALSE */
7402 action_does_learning(struct action *a)
7406 for (i = 0; i < a->n_instructions; i++)
7407 switch (a->instructions[i].type) {
7408 case INSTR_LEARNER_LEARN:
7409 return 1; /* TRUE */
7411 case INSTR_LEARNER_FORGET:
7412 return 1; /* TRUE */
7418 return 0; /* FALSE */
7422 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7424 const char *args_struct_type_name,
7425 const char **instructions,
7426 uint32_t n_instructions)
7428 struct struct_type *args_struct_type = NULL;
7429 struct action *a = NULL;
7434 CHECK_NAME(name, EINVAL);
7435 CHECK(!action_find(p, name), EEXIST);
7437 if (args_struct_type_name) {
7438 CHECK_NAME(args_struct_type_name, EINVAL);
7439 args_struct_type = struct_type_find(p, args_struct_type_name);
7440 CHECK(args_struct_type, EINVAL);
7441 CHECK(!args_struct_type->var_size, EINVAL);
7444 /* Node allocation. */
7445 a = calloc(1, sizeof(struct action));
7451 if (args_struct_type) {
7452 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7453 if (!a->args_endianness) {
7459 /* Node initialization. */
7460 strcpy(a->name, name);
7461 a->st = args_struct_type;
7462 a->id = p->n_actions;
7464 /* Instruction translation. */
7465 status = instruction_config(p, a, instructions, n_instructions);
7469 /* Node add to tailq. */
7470 TAILQ_INSERT_TAIL(&p->actions, a, node);
7479 free(a->args_endianness);
7480 free(a->instructions);
7481 free(a->instruction_data);
7488 action_build(struct rte_swx_pipeline *p)
7490 struct action *action;
7492 /* p->action_instructions. */
7493 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7494 CHECK(p->action_instructions, ENOMEM);
7496 TAILQ_FOREACH(action, &p->actions, node)
7497 p->action_instructions[action->id] = action->instructions;
7499 /* p->action_funcs. */
7500 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7501 CHECK(p->action_funcs, ENOMEM);
7507 action_build_free(struct rte_swx_pipeline *p)
7509 free(p->action_funcs);
7510 p->action_funcs = NULL;
7512 free(p->action_instructions);
7513 p->action_instructions = NULL;
7517 action_free(struct rte_swx_pipeline *p)
7519 action_build_free(p);
7522 struct action *action;
7524 action = TAILQ_FIRST(&p->actions);
7528 TAILQ_REMOVE(&p->actions, action, node);
7529 free(action->args_endianness);
7530 free(action->instructions);
7531 free(action->instruction_data);
7537 action_arg_src_mov_count(struct action *a,
7539 struct instruction *instructions,
7540 struct instruction_data *instruction_data,
7541 uint32_t n_instructions)
7543 uint32_t offset, n_users = 0, i;
7546 (arg_id >= a->st->n_fields) ||
7548 !instruction_data ||
7552 offset = a->st->fields[arg_id].offset / 8;
7554 for (i = 0; i < n_instructions; i++) {
7555 struct instruction *instr = &instructions[i];
7556 struct instruction_data *data = &instruction_data[i];
7558 if (data->invalid ||
7559 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7560 instr->mov.src.struct_id ||
7561 (instr->mov.src.offset != offset))
7570 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7571 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7572 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7574 #define field_ntoh(val, n_bits) (val)
7575 #define field_hton(val, n_bits) (val)
7578 #define ACTION_ARGS_TOKENS_MAX 256
7581 action_args_parse(struct action *a, const char *args, uint8_t *data)
7583 char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7584 uint32_t n_tokens = 0, offset = 0, i;
7588 if (!a->st || !args || !args[0]) {
7593 /* Memory allocation. */
7600 /* Parse the string into tokens. */
7604 token = strtok_r(s, " \f\n\r\t\v", &s);
7608 if (n_tokens >= RTE_DIM(tokens)) {
7613 tokens[n_tokens] = token;
7618 if (n_tokens != a->st->n_fields * 2) {
7623 /* Process the action arguments. */
7624 for (i = 0; i < a->st->n_fields; i++) {
7625 struct field *f = &a->st->fields[i];
7626 char *arg_name = tokens[i * 2];
7627 char *arg_val = tokens[i * 2 + 1];
7630 if (strcmp(arg_name, f->name)) {
7635 val = strtoull(arg_val, &arg_val, 0);
7641 /* Endianness conversion. */
7642 if (a->args_endianness[i])
7643 val = field_hton(val, f->n_bits);
7645 /* Copy to entry. */
7646 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7647 offset += f->n_bits / 8;
7658 static struct table_type *
7659 table_type_find(struct rte_swx_pipeline *p, const char *name)
7661 struct table_type *elem;
7663 TAILQ_FOREACH(elem, &p->table_types, node)
7664 if (strcmp(elem->name, name) == 0)
7670 static struct table_type *
7671 table_type_resolve(struct rte_swx_pipeline *p,
7672 const char *recommended_type_name,
7673 enum rte_swx_table_match_type match_type)
7675 struct table_type *elem;
7677 /* Only consider the recommended type if the match type is correct. */
7678 if (recommended_type_name)
7679 TAILQ_FOREACH(elem, &p->table_types, node)
7680 if (!strcmp(elem->name, recommended_type_name) &&
7681 (elem->match_type == match_type))
7684 /* Ignore the recommended type and get the first element with this match
7687 TAILQ_FOREACH(elem, &p->table_types, node)
7688 if (elem->match_type == match_type)
7694 static struct table *
7695 table_find(struct rte_swx_pipeline *p, const char *name)
7699 TAILQ_FOREACH(elem, &p->tables, node)
7700 if (strcmp(elem->name, name) == 0)
7706 static struct table *
7707 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7709 struct table *table = NULL;
7711 TAILQ_FOREACH(table, &p->tables, node)
7712 if (table->id == id)
7719 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7721 enum rte_swx_table_match_type match_type,
7722 struct rte_swx_table_ops *ops)
7724 struct table_type *elem;
7728 CHECK_NAME(name, EINVAL);
7729 CHECK(!table_type_find(p, name), EEXIST);
7732 CHECK(ops->create, EINVAL);
7733 CHECK(ops->lkp, EINVAL);
7734 CHECK(ops->free, EINVAL);
7736 /* Node allocation. */
7737 elem = calloc(1, sizeof(struct table_type));
7738 CHECK(elem, ENOMEM);
7740 /* Node initialization. */
7741 strcpy(elem->name, name);
7742 elem->match_type = match_type;
7743 memcpy(&elem->ops, ops, sizeof(*ops));
7745 /* Node add to tailq. */
7746 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7752 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7754 enum rte_swx_table_match_type *match_type)
7756 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7758 for (i = 0; i < n_fields; i++) {
7759 struct rte_swx_match_field_params *f = &fields[i];
7761 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7764 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7768 if ((n_fields_lpm > 1) ||
7769 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7772 *match_type = (n_fields_em == n_fields) ?
7773 RTE_SWX_TABLE_MATCH_EXACT :
7774 RTE_SWX_TABLE_MATCH_WILDCARD;
7780 table_match_fields_check(struct rte_swx_pipeline *p,
7781 struct rte_swx_pipeline_table_params *params,
7782 struct header **header)
7784 struct header *h0 = NULL;
7785 struct field *hf, *mf;
7786 uint32_t *offset = NULL, i;
7789 /* Return if no match fields. */
7790 if (!params->n_fields) {
7791 if (params->fields) {
7802 /* Memory allocation. */
7803 offset = calloc(params->n_fields, sizeof(uint32_t));
7809 /* Check that all the match fields belong to either the same header or
7812 hf = header_field_parse(p, params->fields[0].name, &h0);
7813 mf = metadata_field_parse(p, params->fields[0].name);
7814 if ((!hf && !mf) || (hf && hf->var_size)) {
7819 offset[0] = h0 ? hf->offset : mf->offset;
7821 for (i = 1; i < params->n_fields; i++)
7825 hf = header_field_parse(p, params->fields[i].name, &h);
7826 if (!hf || (h->id != h0->id) || hf->var_size) {
7831 offset[i] = hf->offset;
7833 mf = metadata_field_parse(p, params->fields[i].name);
7839 offset[i] = mf->offset;
7842 /* Check that there are no duplicated match fields. */
7843 for (i = 0; i < params->n_fields; i++) {
7846 for (j = 0; j < i; j++)
7847 if (offset[j] == offset[i]) {
7863 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7865 struct rte_swx_pipeline_table_params *params,
7866 const char *recommended_table_type_name,
7870 struct table_type *type;
7871 struct table *t = NULL;
7872 struct action *default_action;
7873 struct header *header = NULL;
7874 uint32_t action_data_size_max = 0, i;
7879 CHECK_NAME(name, EINVAL);
7880 CHECK(!table_find(p, name), EEXIST);
7881 CHECK(!selector_find(p, name), EEXIST);
7882 CHECK(!learner_find(p, name), EEXIST);
7884 CHECK(params, EINVAL);
7887 status = table_match_fields_check(p, params, &header);
7891 /* Action checks. */
7892 CHECK(params->n_actions, EINVAL);
7893 CHECK(params->action_names, EINVAL);
7894 for (i = 0; i < params->n_actions; i++) {
7895 const char *action_name = params->action_names[i];
7897 uint32_t action_data_size;
7898 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7900 CHECK_NAME(action_name, EINVAL);
7902 a = action_find(p, action_name);
7904 CHECK(!action_does_learning(a), EINVAL);
7906 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7907 if (action_data_size > action_data_size_max)
7908 action_data_size_max = action_data_size;
7910 if (params->action_is_for_table_entries)
7911 action_is_for_table_entries = params->action_is_for_table_entries[i];
7912 if (params->action_is_for_default_entry)
7913 action_is_for_default_entry = params->action_is_for_default_entry[i];
7914 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7917 CHECK_NAME(params->default_action_name, EINVAL);
7918 for (i = 0; i < p->n_actions; i++)
7919 if (!strcmp(params->action_names[i],
7920 params->default_action_name))
7922 CHECK(i < params->n_actions, EINVAL);
7923 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7926 default_action = action_find(p, params->default_action_name);
7927 CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7930 /* Table type checks. */
7931 if (recommended_table_type_name)
7932 CHECK_NAME(recommended_table_type_name, EINVAL);
7934 if (params->n_fields) {
7935 enum rte_swx_table_match_type match_type;
7937 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7941 type = table_type_resolve(p, recommended_table_type_name, match_type);
7942 CHECK(type, EINVAL);
7947 /* Memory allocation. */
7948 t = calloc(1, sizeof(struct table));
7954 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7960 t->actions = calloc(params->n_actions, sizeof(struct action *));
7966 if (action_data_size_max) {
7967 t->default_action_data = calloc(1, action_data_size_max);
7968 if (!t->default_action_data) {
7974 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7975 if (!t->action_is_for_table_entries) {
7980 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7981 if (!t->action_is_for_default_entry) {
7986 /* Node initialization. */
7987 strcpy(t->name, name);
7988 if (args && args[0])
7989 strcpy(t->args, args);
7992 for (i = 0; i < params->n_fields; i++) {
7993 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7994 struct match_field *f = &t->fields[i];
7996 f->match_type = field->match_type;
7998 header_field_parse(p, field->name, NULL) :
7999 metadata_field_parse(p, field->name);
8001 t->n_fields = params->n_fields;
8004 for (i = 0; i < params->n_actions; i++) {
8005 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8007 if (params->action_is_for_table_entries)
8008 action_is_for_table_entries = params->action_is_for_table_entries[i];
8009 if (params->action_is_for_default_entry)
8010 action_is_for_default_entry = params->action_is_for_default_entry[i];
8012 t->actions[i] = action_find(p, params->action_names[i]);
8013 t->action_is_for_table_entries[i] = action_is_for_table_entries;
8014 t->action_is_for_default_entry[i] = action_is_for_default_entry;
8016 t->default_action = default_action;
8017 if (default_action->st) {
8018 status = action_args_parse(default_action,
8019 params->default_action_args,
8020 t->default_action_data);
8025 t->n_actions = params->n_actions;
8026 t->default_action_is_const = params->default_action_is_const;
8027 t->action_data_size_max = action_data_size_max;
8030 t->id = p->n_tables;
8032 /* Node add to tailq. */
8033 TAILQ_INSERT_TAIL(&p->tables, t, node);
8042 free(t->action_is_for_default_entry);
8043 free(t->action_is_for_table_entries);
8044 free(t->default_action_data);
8052 static struct rte_swx_table_params *
8053 table_params_get(struct table *table)
8055 struct rte_swx_table_params *params;
8056 struct field *first, *last;
8058 uint32_t key_size, key_offset, action_data_size, i;
8060 /* Memory allocation. */
8061 params = calloc(1, sizeof(struct rte_swx_table_params));
8065 /* Find first (smallest offset) and last (biggest offset) match fields. */
8066 first = table->fields[0].field;
8067 last = table->fields[0].field;
8069 for (i = 0; i < table->n_fields; i++) {
8070 struct field *f = table->fields[i].field;
8072 if (f->offset < first->offset)
8075 if (f->offset > last->offset)
8079 /* Key offset and size. */
8080 key_offset = first->offset / 8;
8081 key_size = (last->offset + last->n_bits - first->offset) / 8;
8083 /* Memory allocation. */
8084 key_mask = calloc(1, key_size);
8091 for (i = 0; i < table->n_fields; i++) {
8092 struct field *f = table->fields[i].field;
8093 uint32_t start = (f->offset - first->offset) / 8;
8094 size_t size = f->n_bits / 8;
8096 memset(&key_mask[start], 0xFF, size);
8099 /* Action data size. */
8100 action_data_size = 0;
8101 for (i = 0; i < table->n_actions; i++) {
8102 struct action *action = table->actions[i];
8103 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8105 if (ads > action_data_size)
8106 action_data_size = ads;
8110 params->match_type = table->type->match_type;
8111 params->key_size = key_size;
8112 params->key_offset = key_offset;
8113 params->key_mask0 = key_mask;
8114 params->action_data_size = action_data_size;
8115 params->n_keys_max = table->size;
8121 table_params_free(struct rte_swx_table_params *params)
8126 free(params->key_mask0);
8131 table_stub_lkp(void *table __rte_unused,
8132 void *mailbox __rte_unused,
8133 uint8_t **key __rte_unused,
8134 uint64_t *action_id __rte_unused,
8135 uint8_t **action_data __rte_unused,
8139 return 1; /* DONE. */
8143 table_build(struct rte_swx_pipeline *p)
8147 /* Per pipeline: table statistics. */
8148 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8149 CHECK(p->table_stats, ENOMEM);
8151 for (i = 0; i < p->n_tables; i++) {
8152 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8153 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8156 /* Per thread: table runt-time. */
8157 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8158 struct thread *t = &p->threads[i];
8159 struct table *table;
8161 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8162 CHECK(t->tables, ENOMEM);
8164 TAILQ_FOREACH(table, &p->tables, node) {
8165 struct table_runtime *r = &t->tables[table->id];
8170 size = table->type->ops.mailbox_size_get();
8173 r->func = table->type->ops.lkp;
8177 r->mailbox = calloc(1, size);
8178 CHECK(r->mailbox, ENOMEM);
8182 r->key = table->header ?
8183 &t->structs[table->header->struct_id] :
8184 &t->structs[p->metadata_struct_id];
8186 r->func = table_stub_lkp;
8195 table_build_free(struct rte_swx_pipeline *p)
8199 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8200 struct thread *t = &p->threads[i];
8206 for (j = 0; j < p->n_tables; j++) {
8207 struct table_runtime *r = &t->tables[j];
8216 if (p->table_stats) {
8217 for (i = 0; i < p->n_tables; i++)
8218 free(p->table_stats[i].n_pkts_action);
8220 free(p->table_stats);
8225 table_free(struct rte_swx_pipeline *p)
8227 table_build_free(p);
8233 elem = TAILQ_FIRST(&p->tables);
8237 TAILQ_REMOVE(&p->tables, elem, node);
8239 free(elem->actions);
8240 free(elem->default_action_data);
8246 struct table_type *elem;
8248 elem = TAILQ_FIRST(&p->table_types);
8252 TAILQ_REMOVE(&p->table_types, elem, node);
8260 static struct selector *
8261 selector_find(struct rte_swx_pipeline *p, const char *name)
8265 TAILQ_FOREACH(s, &p->selectors, node)
8266 if (strcmp(s->name, name) == 0)
8272 static struct selector *
8273 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8275 struct selector *s = NULL;
8277 TAILQ_FOREACH(s, &p->selectors, node)
8285 selector_fields_check(struct rte_swx_pipeline *p,
8286 struct rte_swx_pipeline_selector_params *params,
8287 struct header **header)
8289 struct header *h0 = NULL;
8290 struct field *hf, *mf;
8293 /* Return if no selector fields. */
8294 if (!params->n_selector_fields || !params->selector_field_names)
8297 /* Check that all the selector fields either belong to the same header
8298 * or are all meta-data fields.
8300 hf = header_field_parse(p, params->selector_field_names[0], &h0);
8301 mf = metadata_field_parse(p, params->selector_field_names[0]);
8305 for (i = 1; i < params->n_selector_fields; i++)
8309 hf = header_field_parse(p, params->selector_field_names[i], &h);
8310 if (!hf || (h->id != h0->id))
8313 mf = metadata_field_parse(p, params->selector_field_names[i]);
8318 /* Check that there are no duplicated match fields. */
8319 for (i = 0; i < params->n_selector_fields; i++) {
8320 const char *field_name = params->selector_field_names[i];
8323 for (j = i + 1; j < params->n_selector_fields; j++)
8324 if (!strcmp(params->selector_field_names[j], field_name))
8336 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8338 struct rte_swx_pipeline_selector_params *params)
8341 struct header *selector_header = NULL;
8342 struct field *group_id_field, *member_id_field;
8348 CHECK_NAME(name, EINVAL);
8349 CHECK(!table_find(p, name), EEXIST);
8350 CHECK(!selector_find(p, name), EEXIST);
8351 CHECK(!learner_find(p, name), EEXIST);
8353 CHECK(params, EINVAL);
8355 CHECK_NAME(params->group_id_field_name, EINVAL);
8356 group_id_field = metadata_field_parse(p, params->group_id_field_name);
8357 CHECK(group_id_field, EINVAL);
8359 for (i = 0; i < params->n_selector_fields; i++) {
8360 const char *field_name = params->selector_field_names[i];
8362 CHECK_NAME(field_name, EINVAL);
8364 status = selector_fields_check(p, params, &selector_header);
8368 CHECK_NAME(params->member_id_field_name, EINVAL);
8369 member_id_field = metadata_field_parse(p, params->member_id_field_name);
8370 CHECK(member_id_field, EINVAL);
8372 CHECK(params->n_groups_max, EINVAL);
8374 CHECK(params->n_members_per_group_max, EINVAL);
8376 /* Memory allocation. */
8377 s = calloc(1, sizeof(struct selector));
8383 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8384 if (!s->selector_fields) {
8389 /* Node initialization. */
8390 strcpy(s->name, name);
8392 s->group_id_field = group_id_field;
8394 for (i = 0; i < params->n_selector_fields; i++) {
8395 const char *field_name = params->selector_field_names[i];
8397 s->selector_fields[i] = selector_header ?
8398 header_field_parse(p, field_name, NULL) :
8399 metadata_field_parse(p, field_name);
8402 s->n_selector_fields = params->n_selector_fields;
8404 s->selector_header = selector_header;
8406 s->member_id_field = member_id_field;
8408 s->n_groups_max = params->n_groups_max;
8410 s->n_members_per_group_max = params->n_members_per_group_max;
8412 s->id = p->n_selectors;
8414 /* Node add to tailq. */
8415 TAILQ_INSERT_TAIL(&p->selectors, s, node);
8424 free(s->selector_fields);
8432 selector_params_free(struct rte_swx_table_selector_params *params)
8437 free(params->selector_mask);
8442 static struct rte_swx_table_selector_params *
8443 selector_table_params_get(struct selector *s)
8445 struct rte_swx_table_selector_params *params = NULL;
8446 struct field *first, *last;
8449 /* Memory allocation. */
8450 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8455 params->group_id_offset = s->group_id_field->offset / 8;
8457 /* Find first (smallest offset) and last (biggest offset) selector fields. */
8458 first = s->selector_fields[0];
8459 last = s->selector_fields[0];
8461 for (i = 0; i < s->n_selector_fields; i++) {
8462 struct field *f = s->selector_fields[i];
8464 if (f->offset < first->offset)
8467 if (f->offset > last->offset)
8471 /* Selector offset and size. */
8472 params->selector_offset = first->offset / 8;
8473 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8475 /* Memory allocation. */
8476 params->selector_mask = calloc(1, params->selector_size);
8477 if (!params->selector_mask)
8480 /* Selector mask. */
8481 for (i = 0; i < s->n_selector_fields; i++) {
8482 struct field *f = s->selector_fields[i];
8483 uint32_t start = (f->offset - first->offset) / 8;
8484 size_t size = f->n_bits / 8;
8486 memset(¶ms->selector_mask[start], 0xFF, size);
8490 params->member_id_offset = s->member_id_field->offset / 8;
8492 /* Maximum number of groups. */
8493 params->n_groups_max = s->n_groups_max;
8495 /* Maximum number of members per group. */
8496 params->n_members_per_group_max = s->n_members_per_group_max;
8501 selector_params_free(params);
8506 selector_build_free(struct rte_swx_pipeline *p)
8510 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8511 struct thread *t = &p->threads[i];
8517 for (j = 0; j < p->n_selectors; j++) {
8518 struct selector_runtime *r = &t->selectors[j];
8524 t->selectors = NULL;
8527 free(p->selector_stats);
8528 p->selector_stats = NULL;
8532 selector_build(struct rte_swx_pipeline *p)
8537 /* Per pipeline: selector statistics. */
8538 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8539 if (!p->selector_stats) {
8544 /* Per thread: selector run-time. */
8545 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8546 struct thread *t = &p->threads[i];
8549 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8550 if (!t->selectors) {
8555 TAILQ_FOREACH(s, &p->selectors, node) {
8556 struct selector_runtime *r = &t->selectors[s->id];
8560 size = rte_swx_table_selector_mailbox_size_get();
8562 r->mailbox = calloc(1, size);
8569 /* r->group_id_buffer. */
8570 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8572 /* r->selector_buffer. */
8573 r->selector_buffer = s->selector_header ?
8574 &t->structs[s->selector_header->struct_id] :
8575 &t->structs[p->metadata_struct_id];
8577 /* r->member_id_buffer. */
8578 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8585 selector_build_free(p);
8590 selector_free(struct rte_swx_pipeline *p)
8592 selector_build_free(p);
8594 /* Selector tables. */
8596 struct selector *elem;
8598 elem = TAILQ_FIRST(&p->selectors);
8602 TAILQ_REMOVE(&p->selectors, elem, node);
8603 free(elem->selector_fields);
8611 static struct learner *
8612 learner_find(struct rte_swx_pipeline *p, const char *name)
8616 TAILQ_FOREACH(l, &p->learners, node)
8617 if (!strcmp(l->name, name))
8623 static struct learner *
8624 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8626 struct learner *l = NULL;
8628 TAILQ_FOREACH(l, &p->learners, node)
8636 learner_match_fields_check(struct rte_swx_pipeline *p,
8637 struct rte_swx_pipeline_learner_params *params,
8638 struct header **header)
8640 struct header *h0 = NULL;
8641 struct field *hf, *mf;
8644 /* Return if no match fields. */
8645 if (!params->n_fields || !params->field_names)
8648 /* Check that all the match fields either belong to the same header
8649 * or are all meta-data fields.
8651 hf = header_field_parse(p, params->field_names[0], &h0);
8652 mf = metadata_field_parse(p, params->field_names[0]);
8656 for (i = 1; i < params->n_fields; i++)
8660 hf = header_field_parse(p, params->field_names[i], &h);
8661 if (!hf || (h->id != h0->id))
8664 mf = metadata_field_parse(p, params->field_names[i]);
8669 /* Check that there are no duplicated match fields. */
8670 for (i = 0; i < params->n_fields; i++) {
8671 const char *field_name = params->field_names[i];
8674 for (j = i + 1; j < params->n_fields; j++)
8675 if (!strcmp(params->field_names[j], field_name))
8687 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8689 struct struct_type *mst = p->metadata_st, *ast = a->st;
8690 struct field *mf, *af;
8700 /* Check that mf_name is the name of a valid meta-data field. */
8701 CHECK_NAME(mf_name, EINVAL);
8702 mf = metadata_field_parse(p, mf_name);
8705 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8706 * all the action arguments.
8708 mf_pos = mf - mst->fields;
8709 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8711 /* Check that the size of each of the identified meta-data fields matches exactly the size
8712 * of the corresponding action argument.
8714 for (i = 0; i < ast->n_fields; i++) {
8715 mf = &mst->fields[mf_pos + i];
8716 af = &ast->fields[i];
8718 CHECK(mf->n_bits == af->n_bits, EINVAL);
8725 learner_action_learning_check(struct rte_swx_pipeline *p,
8726 struct action *action,
8727 const char **action_names,
8732 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8733 * the action passed as argument to the "learn" instruction) is also enabled for the
8734 * current learner table.
8736 for (i = 0; i < action->n_instructions; i++) {
8737 struct instruction *instr = &action->instructions[i];
8738 uint32_t found = 0, j;
8740 if (instr->type != INSTR_LEARNER_LEARN)
8743 for (j = 0; j < n_actions; j++) {
8746 a = action_find(p, action_names[j]);
8750 if (a->id == instr->learn.action_id)
8762 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8764 struct rte_swx_pipeline_learner_params *params,
8767 uint32_t n_timeouts)
8769 struct learner *l = NULL;
8770 struct action *default_action;
8771 struct header *header = NULL;
8772 uint32_t action_data_size_max = 0, i;
8777 CHECK_NAME(name, EINVAL);
8778 CHECK(!table_find(p, name), EEXIST);
8779 CHECK(!selector_find(p, name), EEXIST);
8780 CHECK(!learner_find(p, name), EEXIST);
8782 CHECK(params, EINVAL);
8785 status = learner_match_fields_check(p, params, &header);
8789 /* Action checks. */
8790 CHECK(params->n_actions, EINVAL);
8791 CHECK(params->action_names, EINVAL);
8792 for (i = 0; i < params->n_actions; i++) {
8793 const char *action_name = params->action_names[i];
8795 uint32_t action_data_size;
8796 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8798 CHECK_NAME(action_name, EINVAL);
8800 a = action_find(p, action_name);
8803 status = learner_action_learning_check(p,
8805 params->action_names,
8810 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8811 if (action_data_size > action_data_size_max)
8812 action_data_size_max = action_data_size;
8814 if (params->action_is_for_table_entries)
8815 action_is_for_table_entries = params->action_is_for_table_entries[i];
8816 if (params->action_is_for_default_entry)
8817 action_is_for_default_entry = params->action_is_for_default_entry[i];
8818 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8821 CHECK_NAME(params->default_action_name, EINVAL);
8822 for (i = 0; i < p->n_actions; i++)
8823 if (!strcmp(params->action_names[i],
8824 params->default_action_name))
8826 CHECK(i < params->n_actions, EINVAL);
8827 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8830 default_action = action_find(p, params->default_action_name);
8831 CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8834 /* Any other checks. */
8835 CHECK(size, EINVAL);
8836 CHECK(timeout, EINVAL);
8837 CHECK(n_timeouts && (n_timeouts <= RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX), EINVAL);
8839 /* Memory allocation. */
8840 l = calloc(1, sizeof(struct learner));
8846 l->fields = calloc(params->n_fields, sizeof(struct field *));
8852 l->actions = calloc(params->n_actions, sizeof(struct action *));
8858 if (action_data_size_max) {
8859 l->default_action_data = calloc(1, action_data_size_max);
8860 if (!l->default_action_data) {
8866 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8867 if (!l->action_is_for_table_entries) {
8872 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8873 if (!l->action_is_for_default_entry) {
8878 /* Node initialization. */
8879 strcpy(l->name, name);
8881 for (i = 0; i < params->n_fields; i++) {
8882 const char *field_name = params->field_names[i];
8884 l->fields[i] = header ?
8885 header_field_parse(p, field_name, NULL) :
8886 metadata_field_parse(p, field_name);
8889 l->n_fields = params->n_fields;
8893 for (i = 0; i < params->n_actions; i++) {
8894 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8896 if (params->action_is_for_table_entries)
8897 action_is_for_table_entries = params->action_is_for_table_entries[i];
8898 if (params->action_is_for_default_entry)
8899 action_is_for_default_entry = params->action_is_for_default_entry[i];
8901 l->actions[i] = action_find(p, params->action_names[i]);
8902 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8903 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8906 l->default_action = default_action;
8908 if (default_action->st) {
8909 status = action_args_parse(default_action,
8910 params->default_action_args,
8911 l->default_action_data);
8916 l->n_actions = params->n_actions;
8918 l->default_action_is_const = params->default_action_is_const;
8920 l->action_data_size_max = action_data_size_max;
8924 for (i = 0; i < n_timeouts; i++)
8925 l->timeout[i] = timeout[i];
8927 l->n_timeouts = n_timeouts;
8929 l->id = p->n_learners;
8931 /* Node add to tailq. */
8932 TAILQ_INSERT_TAIL(&p->learners, l, node);
8941 free(l->action_is_for_default_entry);
8942 free(l->action_is_for_table_entries);
8943 free(l->default_action_data);
8952 learner_params_free(struct rte_swx_table_learner_params *params)
8957 free(params->key_mask0);
8959 free(params->key_timeout);
8964 static struct rte_swx_table_learner_params *
8965 learner_params_get(struct learner *l)
8967 struct rte_swx_table_learner_params *params = NULL;
8968 struct field *first, *last;
8971 /* Memory allocation. */
8972 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8976 /* Find first (smallest offset) and last (biggest offset) match fields. */
8977 first = l->fields[0];
8978 last = l->fields[0];
8980 for (i = 0; i < l->n_fields; i++) {
8981 struct field *f = l->fields[i];
8983 if (f->offset < first->offset)
8986 if (f->offset > last->offset)
8990 /* Key offset and size. */
8991 params->key_offset = first->offset / 8;
8992 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8994 /* Memory allocation. */
8995 params->key_mask0 = calloc(1, params->key_size);
8996 if (!params->key_mask0)
9000 for (i = 0; i < l->n_fields; i++) {
9001 struct field *f = l->fields[i];
9002 uint32_t start = (f->offset - first->offset) / 8;
9003 size_t size = f->n_bits / 8;
9005 memset(¶ms->key_mask0[start], 0xFF, size);
9008 /* Action data size. */
9009 params->action_data_size = l->action_data_size_max;
9011 /* Maximum number of keys. */
9012 params->n_keys_max = l->size;
9014 /* Memory allocation. */
9015 params->key_timeout = calloc(l->n_timeouts, sizeof(uint32_t));
9016 if (!params->key_timeout)
9020 for (i = 0; i < l->n_timeouts; i++)
9021 params->key_timeout[i] = l->timeout[i];
9023 params->n_key_timeouts = l->n_timeouts;
9028 learner_params_free(params);
9033 learner_build_free(struct rte_swx_pipeline *p)
9037 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9038 struct thread *t = &p->threads[i];
9044 for (j = 0; j < p->n_learners; j++) {
9045 struct learner_runtime *r = &t->learners[j];
9054 if (p->learner_stats) {
9055 for (i = 0; i < p->n_learners; i++)
9056 free(p->learner_stats[i].n_pkts_action);
9058 free(p->learner_stats);
9063 learner_build(struct rte_swx_pipeline *p)
9068 /* Per pipeline: learner statistics. */
9069 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9070 CHECK(p->learner_stats, ENOMEM);
9072 for (i = 0; i < p->n_learners; i++) {
9073 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9074 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9077 /* Per thread: learner run-time. */
9078 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9079 struct thread *t = &p->threads[i];
9082 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9088 TAILQ_FOREACH(l, &p->learners, node) {
9089 struct learner_runtime *r = &t->learners[l->id];
9093 size = rte_swx_table_learner_mailbox_size_get();
9095 r->mailbox = calloc(1, size);
9103 r->key = l->header ?
9104 &t->structs[l->header->struct_id] :
9105 &t->structs[p->metadata_struct_id];
9112 learner_build_free(p);
9117 learner_free(struct rte_swx_pipeline *p)
9119 learner_build_free(p);
9121 /* Learner tables. */
9125 l = TAILQ_FIRST(&p->learners);
9129 TAILQ_REMOVE(&p->learners, l, node);
9132 free(l->default_action_data);
9141 table_state_build(struct rte_swx_pipeline *p)
9143 struct table *table;
9147 p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
9148 sizeof(struct rte_swx_table_state));
9149 CHECK(p->table_state, ENOMEM);
9151 TAILQ_FOREACH(table, &p->tables, node) {
9152 struct rte_swx_table_state *ts = &p->table_state[table->id];
9155 struct rte_swx_table_params *params;
9158 params = table_params_get(table);
9159 CHECK(params, ENOMEM);
9161 ts->obj = table->type->ops.create(params,
9166 table_params_free(params);
9167 CHECK(ts->obj, ENODEV);
9170 /* ts->default_action_data. */
9171 if (table->action_data_size_max) {
9172 ts->default_action_data =
9173 malloc(table->action_data_size_max);
9174 CHECK(ts->default_action_data, ENOMEM);
9176 memcpy(ts->default_action_data,
9177 table->default_action_data,
9178 table->action_data_size_max);
9181 /* ts->default_action_id. */
9182 ts->default_action_id = table->default_action->id;
9185 TAILQ_FOREACH(s, &p->selectors, node) {
9186 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9187 struct rte_swx_table_selector_params *params;
9190 params = selector_table_params_get(s);
9191 CHECK(params, ENOMEM);
9193 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9195 selector_params_free(params);
9196 CHECK(ts->obj, ENODEV);
9199 TAILQ_FOREACH(l, &p->learners, node) {
9200 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9201 p->n_selectors + l->id];
9202 struct rte_swx_table_learner_params *params;
9205 params = learner_params_get(l);
9206 CHECK(params, ENOMEM);
9208 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9209 learner_params_free(params);
9210 CHECK(ts->obj, ENODEV);
9212 /* ts->default_action_data. */
9213 if (l->action_data_size_max) {
9214 ts->default_action_data = malloc(l->action_data_size_max);
9215 CHECK(ts->default_action_data, ENOMEM);
9217 memcpy(ts->default_action_data,
9218 l->default_action_data,
9219 l->action_data_size_max);
9222 /* ts->default_action_id. */
9223 ts->default_action_id = l->default_action->id;
9230 table_state_build_free(struct rte_swx_pipeline *p)
9234 if (!p->table_state)
9237 for (i = 0; i < p->n_tables; i++) {
9238 struct rte_swx_table_state *ts = &p->table_state[i];
9239 struct table *table = table_find_by_id(p, i);
9242 if (table->type && ts->obj)
9243 table->type->ops.free(ts->obj);
9245 /* ts->default_action_data. */
9246 free(ts->default_action_data);
9249 for (i = 0; i < p->n_selectors; i++) {
9250 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9254 rte_swx_table_selector_free(ts->obj);
9257 for (i = 0; i < p->n_learners; i++) {
9258 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9262 rte_swx_table_learner_free(ts->obj);
9264 /* ts->default_action_data. */
9265 free(ts->default_action_data);
9268 free(p->table_state);
9269 p->table_state = NULL;
9273 table_state_free(struct rte_swx_pipeline *p)
9275 table_state_build_free(p);
9281 static struct regarray *
9282 regarray_find(struct rte_swx_pipeline *p, const char *name)
9284 struct regarray *elem;
9286 TAILQ_FOREACH(elem, &p->regarrays, node)
9287 if (!strcmp(elem->name, name))
9293 static struct regarray *
9294 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9296 struct regarray *elem = NULL;
9298 TAILQ_FOREACH(elem, &p->regarrays, node)
9306 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9315 CHECK_NAME(name, EINVAL);
9316 CHECK(!regarray_find(p, name), EEXIST);
9318 CHECK(size, EINVAL);
9319 size = rte_align32pow2(size);
9321 /* Memory allocation. */
9322 r = calloc(1, sizeof(struct regarray));
9325 /* Node initialization. */
9326 strcpy(r->name, name);
9327 r->init_val = init_val;
9329 r->id = p->n_regarrays;
9331 /* Node add to tailq. */
9332 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9339 regarray_build(struct rte_swx_pipeline *p)
9341 struct regarray *regarray;
9343 if (!p->n_regarrays)
9346 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9347 CHECK(p->regarray_runtime, ENOMEM);
9349 TAILQ_FOREACH(regarray, &p->regarrays, node) {
9350 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9353 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9354 RTE_CACHE_LINE_SIZE,
9356 CHECK(r->regarray, ENOMEM);
9358 if (regarray->init_val)
9359 for (i = 0; i < regarray->size; i++)
9360 r->regarray[i] = regarray->init_val;
9362 r->size_mask = regarray->size - 1;
9369 regarray_build_free(struct rte_swx_pipeline *p)
9373 if (!p->regarray_runtime)
9376 for (i = 0; i < p->n_regarrays; i++) {
9377 struct regarray *regarray = regarray_find_by_id(p, i);
9378 struct regarray_runtime *r = &p->regarray_runtime[i];
9380 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9383 free(p->regarray_runtime);
9384 p->regarray_runtime = NULL;
9388 regarray_free(struct rte_swx_pipeline *p)
9390 regarray_build_free(p);
9393 struct regarray *elem;
9395 elem = TAILQ_FIRST(&p->regarrays);
9399 TAILQ_REMOVE(&p->regarrays, elem, node);
9407 static struct meter_profile *
9408 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9410 struct meter_profile *elem;
9412 TAILQ_FOREACH(elem, &p->meter_profiles, node)
9413 if (!strcmp(elem->name, name))
9419 static struct metarray *
9420 metarray_find(struct rte_swx_pipeline *p, const char *name)
9422 struct metarray *elem;
9424 TAILQ_FOREACH(elem, &p->metarrays, node)
9425 if (!strcmp(elem->name, name))
9431 static struct metarray *
9432 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9434 struct metarray *elem = NULL;
9436 TAILQ_FOREACH(elem, &p->metarrays, node)
9444 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9452 CHECK_NAME(name, EINVAL);
9453 CHECK(!metarray_find(p, name), EEXIST);
9455 CHECK(size, EINVAL);
9456 size = rte_align32pow2(size);
9458 /* Memory allocation. */
9459 m = calloc(1, sizeof(struct metarray));
9462 /* Node initialization. */
9463 strcpy(m->name, name);
9465 m->id = p->n_metarrays;
9467 /* Node add to tailq. */
9468 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9474 struct meter_profile meter_profile_default = {
9483 .cir_bytes_per_period = 1,
9485 .pir_bytes_per_period = 1,
9492 meter_init(struct meter *m)
9494 memset(m, 0, sizeof(struct meter));
9495 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9496 m->profile = &meter_profile_default;
9497 m->color_mask = RTE_COLOR_GREEN;
9499 meter_profile_default.n_users++;
9503 metarray_build(struct rte_swx_pipeline *p)
9507 if (!p->n_metarrays)
9510 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9511 CHECK(p->metarray_runtime, ENOMEM);
9513 TAILQ_FOREACH(m, &p->metarrays, node) {
9514 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9517 r->metarray = env_malloc(m->size * sizeof(struct meter),
9518 RTE_CACHE_LINE_SIZE,
9520 CHECK(r->metarray, ENOMEM);
9522 for (i = 0; i < m->size; i++)
9523 meter_init(&r->metarray[i]);
9525 r->size_mask = m->size - 1;
9532 metarray_build_free(struct rte_swx_pipeline *p)
9536 if (!p->metarray_runtime)
9539 for (i = 0; i < p->n_metarrays; i++) {
9540 struct metarray *m = metarray_find_by_id(p, i);
9541 struct metarray_runtime *r = &p->metarray_runtime[i];
9543 env_free(r->metarray, m->size * sizeof(struct meter));
9546 free(p->metarray_runtime);
9547 p->metarray_runtime = NULL;
9551 metarray_free(struct rte_swx_pipeline *p)
9553 metarray_build_free(p);
9557 struct metarray *elem;
9559 elem = TAILQ_FIRST(&p->metarrays);
9563 TAILQ_REMOVE(&p->metarrays, elem, node);
9567 /* Meter profiles. */
9569 struct meter_profile *elem;
9571 elem = TAILQ_FIRST(&p->meter_profiles);
9575 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9584 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9593 free(p->instruction_data);
9594 free(p->instructions);
9598 table_state_free(p);
9603 instruction_table_free(p);
9607 extern_func_free(p);
9621 port_in_types_register(struct rte_swx_pipeline *p)
9625 status = rte_swx_pipeline_port_in_type_register(p,
9627 &rte_swx_port_ethdev_reader_ops);
9631 status = rte_swx_pipeline_port_in_type_register(p,
9633 &rte_swx_port_ring_reader_ops);
9637 #ifdef RTE_PORT_PCAP
9638 status = rte_swx_pipeline_port_in_type_register(p,
9640 &rte_swx_port_source_ops);
9645 status = rte_swx_pipeline_port_in_type_register(p,
9647 &rte_swx_port_fd_reader_ops);
9655 port_out_types_register(struct rte_swx_pipeline *p)
9659 status = rte_swx_pipeline_port_out_type_register(p,
9661 &rte_swx_port_ethdev_writer_ops);
9665 status = rte_swx_pipeline_port_out_type_register(p,
9667 &rte_swx_port_ring_writer_ops);
9671 status = rte_swx_pipeline_port_out_type_register(p,
9673 &rte_swx_port_sink_ops);
9677 status = rte_swx_pipeline_port_out_type_register(p,
9679 &rte_swx_port_fd_writer_ops);
9687 table_types_register(struct rte_swx_pipeline *p)
9691 status = rte_swx_pipeline_table_type_register(p,
9693 RTE_SWX_TABLE_MATCH_EXACT,
9694 &rte_swx_table_exact_match_ops);
9698 status = rte_swx_pipeline_table_type_register(p,
9700 RTE_SWX_TABLE_MATCH_WILDCARD,
9701 &rte_swx_table_wildcard_match_ops);
9709 hash_funcs_register(struct rte_swx_pipeline *p)
9713 status = rte_swx_pipeline_hash_func_register(p, "jhash", rte_jhash);
9717 status = rte_swx_pipeline_hash_func_register(p, "crc32", rte_hash_crc);
9725 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9727 struct rte_swx_pipeline *pipeline = NULL;
9730 /* Check input parameters. */
9733 /* Memory allocation. */
9734 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9740 /* Initialization. */
9741 TAILQ_INIT(&pipeline->struct_types);
9742 TAILQ_INIT(&pipeline->port_in_types);
9743 TAILQ_INIT(&pipeline->ports_in);
9744 TAILQ_INIT(&pipeline->port_out_types);
9745 TAILQ_INIT(&pipeline->ports_out);
9746 TAILQ_INIT(&pipeline->extern_types);
9747 TAILQ_INIT(&pipeline->extern_objs);
9748 TAILQ_INIT(&pipeline->extern_funcs);
9749 TAILQ_INIT(&pipeline->hash_funcs);
9750 TAILQ_INIT(&pipeline->headers);
9751 TAILQ_INIT(&pipeline->actions);
9752 TAILQ_INIT(&pipeline->table_types);
9753 TAILQ_INIT(&pipeline->tables);
9754 TAILQ_INIT(&pipeline->selectors);
9755 TAILQ_INIT(&pipeline->learners);
9756 TAILQ_INIT(&pipeline->regarrays);
9757 TAILQ_INIT(&pipeline->meter_profiles);
9758 TAILQ_INIT(&pipeline->metarrays);
9760 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9761 pipeline->n_mirroring_slots = RTE_SWX_PACKET_MIRRORING_SLOTS_DEFAULT;
9762 pipeline->n_mirroring_sessions = RTE_SWX_PACKET_MIRRORING_SESSIONS_DEFAULT;
9763 pipeline->numa_node = numa_node;
9765 status = port_in_types_register(pipeline);
9769 status = port_out_types_register(pipeline);
9773 status = table_types_register(pipeline);
9777 status = hash_funcs_register(pipeline);
9785 rte_swx_pipeline_free(pipeline);
9790 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9791 const char **instructions,
9792 uint32_t n_instructions)
9797 err = instruction_config(p, NULL, instructions, n_instructions);
9801 /* Thread instruction pointer reset. */
9802 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9803 struct thread *t = &p->threads[i];
9805 thread_ip_reset(p, t);
9812 pipeline_compile(struct rte_swx_pipeline *p);
9815 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9817 struct rte_swx_port_sink_params drop_port_params = {
9823 CHECK(p->build_done == 0, EEXIST);
9825 status = port_in_build(p);
9830 status = rte_swx_pipeline_port_out_config(p,
9837 status = port_out_build(p);
9841 status = mirroring_build(p);
9845 status = struct_build(p);
9849 status = extern_obj_build(p);
9853 status = extern_func_build(p);
9857 status = hash_func_build(p);
9861 status = header_build(p);
9865 status = metadata_build(p);
9869 status = instruction_table_build(p);
9873 status = action_build(p);
9877 status = table_build(p);
9881 status = selector_build(p);
9885 status = learner_build(p);
9889 status = table_state_build(p);
9893 status = regarray_build(p);
9897 status = metarray_build(p);
9903 pipeline_compile(p);
9908 metarray_build_free(p);
9909 regarray_build_free(p);
9910 table_state_build_free(p);
9911 learner_build_free(p);
9912 selector_build_free(p);
9913 table_build_free(p);
9914 action_build_free(p);
9915 instruction_table_build_free(p);
9916 metadata_build_free(p);
9917 header_build_free(p);
9918 hash_func_build_free(p);
9919 extern_func_build_free(p);
9920 extern_obj_build_free(p);
9921 mirroring_build_free(p);
9922 port_out_build_free(p);
9923 port_in_build_free(p);
9924 struct_build_free(p);
9930 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9934 for (i = 0; i < n_instructions; i++)
9939 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9943 for (i = 0; i < p->n_ports_out; i++) {
9944 struct port_out_runtime *port = &p->out[i];
9947 port->flush(port->obj);
9955 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9956 struct rte_swx_ctl_pipeline_info *pipeline)
9958 struct action *action;
9959 struct table *table;
9960 uint32_t n_actions = 0, n_tables = 0;
9962 if (!p || !pipeline)
9965 TAILQ_FOREACH(action, &p->actions, node)
9968 TAILQ_FOREACH(table, &p->tables, node)
9971 pipeline->n_ports_in = p->n_ports_in;
9972 pipeline->n_ports_out = p->n_ports_out;
9973 pipeline->n_mirroring_slots = p->n_mirroring_slots;
9974 pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9975 pipeline->n_actions = n_actions;
9976 pipeline->n_tables = n_tables;
9977 pipeline->n_selectors = p->n_selectors;
9978 pipeline->n_learners = p->n_learners;
9979 pipeline->n_regarrays = p->n_regarrays;
9980 pipeline->n_metarrays = p->n_metarrays;
9986 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9988 if (!p || !numa_node)
9991 *numa_node = p->numa_node;
9996 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9998 struct rte_swx_ctl_action_info *action)
10000 struct action *a = NULL;
10002 if (!p || (action_id >= p->n_actions) || !action)
10005 a = action_find_by_id(p, action_id);
10009 strcpy(action->name, a->name);
10010 action->n_args = a->st ? a->st->n_fields : 0;
10015 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10016 uint32_t action_id,
10017 uint32_t action_arg_id,
10018 struct rte_swx_ctl_action_arg_info *action_arg)
10020 struct action *a = NULL;
10021 struct field *arg = NULL;
10023 if (!p || (action_id >= p->n_actions) || !action_arg)
10026 a = action_find_by_id(p, action_id);
10027 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10030 arg = &a->st->fields[action_arg_id];
10031 strcpy(action_arg->name, arg->name);
10032 action_arg->n_bits = arg->n_bits;
10033 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10039 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10041 struct rte_swx_ctl_table_info *table)
10043 struct table *t = NULL;
10048 t = table_find_by_id(p, table_id);
10052 strcpy(table->name, t->name);
10053 strcpy(table->args, t->args);
10054 table->n_match_fields = t->n_fields;
10055 table->n_actions = t->n_actions;
10056 table->default_action_is_const = t->default_action_is_const;
10057 table->size = t->size;
10062 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10064 uint32_t match_field_id,
10065 struct rte_swx_ctl_table_match_field_info *match_field)
10068 struct match_field *f;
10070 if (!p || (table_id >= p->n_tables) || !match_field)
10073 t = table_find_by_id(p, table_id);
10074 if (!t || (match_field_id >= t->n_fields))
10077 f = &t->fields[match_field_id];
10078 match_field->match_type = f->match_type;
10079 match_field->is_header = t->header ? 1 : 0;
10080 match_field->n_bits = f->field->n_bits;
10081 match_field->offset = f->field->offset;
10087 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10089 uint32_t table_action_id,
10090 struct rte_swx_ctl_table_action_info *table_action)
10094 if (!p || (table_id >= p->n_tables) || !table_action)
10097 t = table_find_by_id(p, table_id);
10098 if (!t || (table_action_id >= t->n_actions))
10101 table_action->action_id = t->actions[table_action_id]->id;
10103 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
10104 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
10110 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10112 struct rte_swx_table_ops *table_ops,
10117 if (!p || (table_id >= p->n_tables))
10120 t = table_find_by_id(p, table_id);
10126 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10136 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10137 uint32_t selector_id,
10138 struct rte_swx_ctl_selector_info *selector)
10140 struct selector *s = NULL;
10142 if (!p || !selector)
10145 s = selector_find_by_id(p, selector_id);
10149 strcpy(selector->name, s->name);
10151 selector->n_selector_fields = s->n_selector_fields;
10152 selector->n_groups_max = s->n_groups_max;
10153 selector->n_members_per_group_max = s->n_members_per_group_max;
10159 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10160 uint32_t selector_id,
10161 struct rte_swx_ctl_table_match_field_info *field)
10163 struct selector *s;
10165 if (!p || (selector_id >= p->n_selectors) || !field)
10168 s = selector_find_by_id(p, selector_id);
10172 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10173 field->is_header = 0;
10174 field->n_bits = s->group_id_field->n_bits;
10175 field->offset = s->group_id_field->offset;
10181 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10182 uint32_t selector_id,
10183 uint32_t selector_field_id,
10184 struct rte_swx_ctl_table_match_field_info *field)
10186 struct selector *s;
10189 if (!p || (selector_id >= p->n_selectors) || !field)
10192 s = selector_find_by_id(p, selector_id);
10193 if (!s || (selector_field_id >= s->n_selector_fields))
10196 f = s->selector_fields[selector_field_id];
10197 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10198 field->is_header = s->selector_header ? 1 : 0;
10199 field->n_bits = f->n_bits;
10200 field->offset = f->offset;
10206 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10207 uint32_t selector_id,
10208 struct rte_swx_ctl_table_match_field_info *field)
10210 struct selector *s;
10212 if (!p || (selector_id >= p->n_selectors) || !field)
10215 s = selector_find_by_id(p, selector_id);
10219 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10220 field->is_header = 0;
10221 field->n_bits = s->member_id_field->n_bits;
10222 field->offset = s->member_id_field->offset;
10228 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10229 uint32_t learner_id,
10230 struct rte_swx_ctl_learner_info *learner)
10232 struct learner *l = NULL;
10234 if (!p || !learner)
10237 l = learner_find_by_id(p, learner_id);
10241 strcpy(learner->name, l->name);
10243 learner->n_match_fields = l->n_fields;
10244 learner->n_actions = l->n_actions;
10245 learner->default_action_is_const = l->default_action_is_const;
10246 learner->size = l->size;
10247 learner->n_key_timeouts = l->n_timeouts;
10253 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10254 uint32_t learner_id,
10255 uint32_t match_field_id,
10256 struct rte_swx_ctl_table_match_field_info *match_field)
10261 if (!p || (learner_id >= p->n_learners) || !match_field)
10264 l = learner_find_by_id(p, learner_id);
10265 if (!l || (match_field_id >= l->n_fields))
10268 f = l->fields[match_field_id];
10269 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10270 match_field->is_header = l->header ? 1 : 0;
10271 match_field->n_bits = f->n_bits;
10272 match_field->offset = f->offset;
10278 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10279 uint32_t learner_id,
10280 uint32_t learner_action_id,
10281 struct rte_swx_ctl_table_action_info *learner_action)
10285 if (!p || (learner_id >= p->n_learners) || !learner_action)
10288 l = learner_find_by_id(p, learner_id);
10289 if (!l || (learner_action_id >= l->n_actions))
10292 learner_action->action_id = l->actions[learner_action_id]->id;
10294 learner_action->action_is_for_table_entries =
10295 l->action_is_for_table_entries[learner_action_id];
10297 learner_action->action_is_for_default_entry =
10298 l->action_is_for_default_entry[learner_action_id];
10304 rte_swx_ctl_pipeline_learner_timeout_get(struct rte_swx_pipeline *p,
10305 uint32_t learner_id,
10306 uint32_t timeout_id,
10311 if (!p || (learner_id >= p->n_learners) || !timeout)
10314 l = learner_find_by_id(p, learner_id);
10315 if (!l || (timeout_id >= l->n_timeouts))
10318 *timeout = l->timeout[timeout_id];
10323 rte_swx_ctl_pipeline_learner_timeout_set(struct rte_swx_pipeline *p,
10324 uint32_t learner_id,
10325 uint32_t timeout_id,
10329 struct rte_swx_table_state *ts;
10332 if (!p || (learner_id >= p->n_learners) || !timeout)
10335 l = learner_find_by_id(p, learner_id);
10336 if (!l || (timeout_id >= l->n_timeouts))
10339 if (!p->build_done)
10342 ts = &p->table_state[p->n_tables + p->n_selectors + l->id];
10344 status = rte_swx_table_learner_timeout_update(ts->obj, timeout_id, timeout);
10348 l->timeout[timeout_id] = timeout;
10354 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10355 struct rte_swx_table_state **table_state)
10357 if (!p || !table_state || !p->build_done)
10360 *table_state = p->table_state;
10365 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10366 struct rte_swx_table_state *table_state)
10368 if (!p || !table_state || !p->build_done)
10371 p->table_state = table_state;
10376 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10378 struct rte_swx_port_in_stats *stats)
10380 struct port_in *port;
10385 port = port_in_find(p, port_id);
10389 port->type->ops.stats_read(port->obj, stats);
10394 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10396 struct rte_swx_port_out_stats *stats)
10398 struct port_out *port;
10403 port = port_out_find(p, port_id);
10407 port->type->ops.stats_read(port->obj, stats);
10412 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10413 const char *table_name,
10414 struct rte_swx_table_stats *stats)
10416 struct table *table;
10417 struct table_statistics *table_stats;
10419 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10422 table = table_find(p, table_name);
10426 table_stats = &p->table_stats[table->id];
10428 memcpy(stats->n_pkts_action,
10429 table_stats->n_pkts_action,
10430 p->n_actions * sizeof(uint64_t));
10432 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10433 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10439 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10440 const char *selector_name,
10441 struct rte_swx_pipeline_selector_stats *stats)
10443 struct selector *s;
10445 if (!p || !selector_name || !selector_name[0] || !stats)
10448 s = selector_find(p, selector_name);
10452 stats->n_pkts = p->selector_stats[s->id].n_pkts;
10458 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10459 const char *learner_name,
10460 struct rte_swx_learner_stats *stats)
10463 struct learner_statistics *learner_stats;
10465 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10468 l = learner_find(p, learner_name);
10472 learner_stats = &p->learner_stats[l->id];
10474 memcpy(stats->n_pkts_action,
10475 learner_stats->n_pkts_action,
10476 p->n_actions * sizeof(uint64_t));
10478 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10479 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10481 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10482 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10484 stats->n_pkts_rearm = learner_stats->n_pkts_rearm;
10485 stats->n_pkts_forget = learner_stats->n_pkts_forget;
10491 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10492 uint32_t regarray_id,
10493 struct rte_swx_ctl_regarray_info *regarray)
10495 struct regarray *r;
10497 if (!p || !regarray)
10500 r = regarray_find_by_id(p, regarray_id);
10504 strcpy(regarray->name, r->name);
10505 regarray->size = r->size;
10510 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10511 const char *regarray_name,
10512 uint32_t regarray_index,
10515 struct regarray *regarray;
10516 struct regarray_runtime *r;
10518 if (!p || !regarray_name || !value)
10521 regarray = regarray_find(p, regarray_name);
10522 if (!regarray || (regarray_index >= regarray->size))
10525 r = &p->regarray_runtime[regarray->id];
10526 *value = r->regarray[regarray_index];
10531 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10532 const char *regarray_name,
10533 uint32_t regarray_index,
10536 struct regarray *regarray;
10537 struct regarray_runtime *r;
10539 if (!p || !regarray_name)
10542 regarray = regarray_find(p, regarray_name);
10543 if (!regarray || (regarray_index >= regarray->size))
10546 r = &p->regarray_runtime[regarray->id];
10547 r->regarray[regarray_index] = value;
10552 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10553 uint32_t metarray_id,
10554 struct rte_swx_ctl_metarray_info *metarray)
10556 struct metarray *m;
10558 if (!p || !metarray)
10561 m = metarray_find_by_id(p, metarray_id);
10565 strcpy(metarray->name, m->name);
10566 metarray->size = m->size;
10571 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10573 struct rte_meter_trtcm_params *params)
10575 struct meter_profile *mp;
10579 CHECK_NAME(name, EINVAL);
10580 CHECK(params, EINVAL);
10581 CHECK(!meter_profile_find(p, name), EEXIST);
10583 /* Node allocation. */
10584 mp = calloc(1, sizeof(struct meter_profile));
10587 /* Node initialization. */
10588 strcpy(mp->name, name);
10589 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10590 status = rte_meter_trtcm_profile_config(&mp->profile, params);
10596 /* Node add to tailq. */
10597 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10603 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10606 struct meter_profile *mp;
10609 CHECK_NAME(name, EINVAL);
10611 mp = meter_profile_find(p, name);
10613 CHECK(!mp->n_users, EBUSY);
10615 /* Remove node from tailq. */
10616 TAILQ_REMOVE(&p->meter_profiles, mp, node);
10623 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10624 const char *metarray_name,
10625 uint32_t metarray_index)
10627 struct meter_profile *mp_old;
10628 struct metarray *metarray;
10629 struct metarray_runtime *metarray_runtime;
10633 CHECK_NAME(metarray_name, EINVAL);
10635 metarray = metarray_find(p, metarray_name);
10636 CHECK(metarray, EINVAL);
10637 CHECK(metarray_index < metarray->size, EINVAL);
10639 metarray_runtime = &p->metarray_runtime[metarray->id];
10640 m = &metarray_runtime->metarray[metarray_index];
10641 mp_old = m->profile;
10651 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10652 const char *metarray_name,
10653 uint32_t metarray_index,
10654 const char *profile_name)
10656 struct meter_profile *mp, *mp_old;
10657 struct metarray *metarray;
10658 struct metarray_runtime *metarray_runtime;
10662 CHECK_NAME(metarray_name, EINVAL);
10664 metarray = metarray_find(p, metarray_name);
10665 CHECK(metarray, EINVAL);
10666 CHECK(metarray_index < metarray->size, EINVAL);
10668 mp = meter_profile_find(p, profile_name);
10671 metarray_runtime = &p->metarray_runtime[metarray->id];
10672 m = &metarray_runtime->metarray[metarray_index];
10673 mp_old = m->profile;
10675 memset(m, 0, sizeof(struct meter));
10676 rte_meter_trtcm_config(&m->m, &mp->profile);
10678 m->color_mask = RTE_COLORS;
10687 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10688 const char *metarray_name,
10689 uint32_t metarray_index,
10690 struct rte_swx_ctl_meter_stats *stats)
10692 struct metarray *metarray;
10693 struct metarray_runtime *metarray_runtime;
10697 CHECK_NAME(metarray_name, EINVAL);
10699 metarray = metarray_find(p, metarray_name);
10700 CHECK(metarray, EINVAL);
10701 CHECK(metarray_index < metarray->size, EINVAL);
10703 CHECK(stats, EINVAL);
10705 metarray_runtime = &p->metarray_runtime[metarray->id];
10706 m = &metarray_runtime->metarray[metarray_index];
10708 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10709 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10715 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10716 uint32_t session_id,
10717 struct rte_swx_pipeline_mirroring_session_params *params)
10719 struct mirroring_session *s;
10722 CHECK(p->build_done, EEXIST);
10723 CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10724 CHECK(params, EINVAL);
10725 CHECK(params->port_id < p->n_ports_out, EINVAL);
10727 s = &p->mirroring_sessions[session_id];
10728 s->port_id = params->port_id;
10729 s->fast_clone = params->fast_clone;
10730 s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10736 * Pipeline compilation.
10738 static const char *
10739 instr_type_to_name(struct instruction *instr)
10741 switch (instr->type) {
10742 case INSTR_RX: return "INSTR_RX";
10744 case INSTR_TX: return "INSTR_TX";
10745 case INSTR_TX_I: return "INSTR_TX_I";
10746 case INSTR_DROP: return "INSTR_DROP";
10747 case INSTR_MIRROR: return "INSTR_MIRROR";
10748 case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10749 case INSTR_RECIRCID: return "INSTR_RECIRCID";
10751 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10752 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10753 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10754 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10755 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10756 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10757 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10758 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10760 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10762 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10764 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10765 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10766 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10767 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10768 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10769 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10770 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10771 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10772 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10774 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10775 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10777 case INSTR_MOV: return "INSTR_MOV";
10778 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10779 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10780 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10781 case INSTR_MOV_I: return "INSTR_MOV_I";
10783 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10784 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10785 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10786 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10787 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10788 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10789 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10790 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10792 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10793 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10794 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10795 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10796 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10797 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10799 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10800 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10801 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10802 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10803 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10804 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10806 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10807 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10808 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10809 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10811 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10812 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10813 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10814 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10815 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10817 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10818 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10819 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10820 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10821 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10823 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10824 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10825 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10826 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10827 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10829 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10830 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10831 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10832 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10833 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10834 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10836 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10837 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10838 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10839 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10840 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10841 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10843 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10844 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10845 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10847 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10848 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10849 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10850 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10851 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10852 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10854 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10855 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10856 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10857 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10858 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10859 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10860 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10861 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10862 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10864 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10865 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10866 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10867 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10868 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10869 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10870 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10871 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10872 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10874 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10875 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10876 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10878 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10879 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10880 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10881 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10882 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10883 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10884 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10885 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10886 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10887 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10888 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10889 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10891 case INSTR_TABLE: return "INSTR_TABLE";
10892 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10893 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10894 case INSTR_LEARNER: return "INSTR_LEARNER";
10895 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10897 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10898 case INSTR_LEARNER_REARM: return "INSTR_LEARNER_REARM";
10899 case INSTR_LEARNER_REARM_NEW: return "INSTR_LEARNER_REARM_NEW";
10900 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10902 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10903 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10904 case INSTR_HASH_FUNC: return "INSTR_HASH_FUNC";
10906 case INSTR_JMP: return "INSTR_JMP";
10907 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10908 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10909 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10910 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10911 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10912 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10913 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10914 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10915 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10916 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10917 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10918 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10919 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10920 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10921 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10922 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10923 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10924 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10925 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10926 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10927 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10928 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10929 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10930 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10931 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10932 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10933 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10934 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10936 case INSTR_RETURN: return "INSTR_RETURN";
10938 default: return "INSTR_UNKNOWN";
10943 (*instruction_export_t)(struct instruction *, FILE *);
10946 instr_io_export(struct instruction *instr, FILE *f)
10948 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10950 /* n_io, n_io_imm, n_hdrs. */
10951 if (instr->type == INSTR_RX ||
10952 instr->type == INSTR_TX ||
10953 instr->type == INSTR_HDR_EXTRACT_M ||
10954 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10957 if (instr->type == INSTR_TX_I)
10960 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10961 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10963 if (instr->type == INSTR_HDR_EXTRACT_M ||
10964 instr->type == INSTR_HDR_LOOKAHEAD ||
10965 instr->type == INSTR_HDR_EMIT)
10968 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10969 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10974 "\t\t.type = %s,\n",
10975 instr_type_to_name(instr));
10978 if (n_io || n_io_imm || n_hdrs)
10986 "\t\t\t\t.offset = %u,\n"
10987 "\t\t\t\t.n_bits = %u,\n"
10989 instr->io.io.offset,
10990 instr->io.io.n_bits);
10995 "\t\t\t\t.val = %u,\n"
10999 /* instr.io.hdr. */
11004 /* instr.io.hdr.header_id. */
11006 "\t\t\t.header_id = {");
11008 for (i = 0; i < n_hdrs; i++)
11011 instr->io.hdr.header_id[i]);
11016 /* instr.io.hdr.struct_id. */
11018 "\t\t\t.struct_id = {");
11020 for (i = 0; i < n_hdrs; i++)
11023 instr->io.hdr.struct_id[i]);
11028 /* instr.io.hdr.n_bytes. */
11030 "\t\t\t.n_bytes = {");
11032 for (i = 0; i < n_hdrs; i++)
11035 instr->io.hdr.n_bytes[i]);
11040 /* instr.io.hdr - closing curly brace. */
11045 /* instr.io - closing curly brace. */
11046 if (n_io || n_io_imm || n_hdrs)
11050 /* instr - closing curly brace. */
11056 instr_mirror_export(struct instruction *instr, FILE *f)
11060 "\t\t.type = %s,\n"
11061 "\t\t.mirror = {\n"
11063 "\t\t\t\t.struct_id = %u,\n"
11064 "\t\t\t\t.n_bits = %u,\n"
11065 "\t\t\t\t.offset = %u,\n"
11068 "\t\t\t\t.struct_id = %u,\n"
11069 "\t\t\t\t.n_bits = %u,\n"
11070 "\t\t\t\t.offset = %u,\n"
11074 instr_type_to_name(instr),
11075 instr->mirror.dst.struct_id,
11076 instr->mirror.dst.n_bits,
11077 instr->mirror.dst.offset,
11078 instr->mirror.src.struct_id,
11079 instr->mirror.src.n_bits,
11080 instr->mirror.src.offset);
11084 instr_recirculate_export(struct instruction *instr, FILE *f)
11088 "\t\t.type = %s,\n"
11090 instr_type_to_name(instr));
11094 instr_recircid_export(struct instruction *instr, FILE *f)
11098 "\t\t.type = %s,\n"
11100 "\t\t\t.offset = %u,\n"
11101 "\t\t\t.n_bits = %u,\n"
11104 instr_type_to_name(instr),
11105 instr->io.io.offset,
11106 instr->io.io.n_bits);
11110 instr_hdr_validate_export(struct instruction *instr, FILE *f)
11114 "\t\t.type = %s,\n"
11116 "\t\t\t.header_id = %u,\n"
11119 instr_type_to_name(instr),
11120 instr->valid.header_id);
11124 instr_mov_export(struct instruction *instr, FILE *f)
11126 if (instr->type != INSTR_MOV_I)
11129 "\t\t.type = %s,\n"
11132 "\t\t\t\t.struct_id = %u,\n"
11133 "\t\t\t\t.n_bits = %u,\n"
11134 "\t\t\t\t.offset = %u,\n"
11137 "\t\t\t\t.struct_id = %u,\n"
11138 "\t\t\t\t.n_bits = %u,\n"
11139 "\t\t\t\t.offset = %u,\n"
11143 instr_type_to_name(instr),
11144 instr->mov.dst.struct_id,
11145 instr->mov.dst.n_bits,
11146 instr->mov.dst.offset,
11147 instr->mov.src.struct_id,
11148 instr->mov.src.n_bits,
11149 instr->mov.src.offset);
11153 "\t\t.type = %s,\n"
11156 "\t\t\t\t.struct_id = %u,\n"
11157 "\t\t\t\t.n_bits = %u,\n"
11158 "\t\t\t\t.offset = %u,\n"
11160 "\t\t\t.src_val = %" PRIu64 ",\n"
11163 instr_type_to_name(instr),
11164 instr->mov.dst.struct_id,
11165 instr->mov.dst.n_bits,
11166 instr->mov.dst.offset,
11167 instr->mov.src_val);
11171 instr_dma_ht_export(struct instruction *instr, FILE *f)
11173 uint32_t n_dma = 0, i;
11176 n_dma = 1 + (instr->type - INSTR_DMA_HT);
11181 "\t\t.type = %s,\n",
11182 instr_type_to_name(instr));
11188 /* instr.dma.dst. */
11190 "\t\t\t.dst = {\n");
11192 /* instr.dma.dst.header_id. */
11194 "\t\t\t\t.header_id = {");
11196 for (i = 0; i < n_dma; i++)
11199 instr->dma.dst.header_id[i]);
11204 /* instr.dma.dst.struct_id. */
11206 "\t\t\t\t.struct_id = {");
11208 for (i = 0; i < n_dma; i++)
11211 instr->dma.dst.struct_id[i]);
11216 /* instr.dma.dst - closing curly brace. */
11220 /* instr.dma.src. */
11222 "\t\t\t.src = {\n");
11224 /* instr.dma.src.offset. */
11226 "\t\t\t\t.offset = {");
11228 for (i = 0; i < n_dma; i++)
11231 instr->dma.src.offset[i]);
11236 /* instr.dma.src - closing curly brace. */
11240 /* instr.dma.n_bytes. */
11242 "\t\t\t.n_bytes = {");
11244 for (i = 0; i < n_dma; i++)
11247 instr->dma.n_bytes[i]);
11252 /* instr.dma - closing curly brace. */
11256 /* instr - closing curly brace. */
11262 instr_alu_export(struct instruction *instr, FILE *f)
11266 if (instr->type == INSTR_ALU_ADD_MI ||
11267 instr->type == INSTR_ALU_ADD_HI ||
11268 instr->type == INSTR_ALU_SUB_MI ||
11269 instr->type == INSTR_ALU_SUB_HI ||
11270 instr->type == INSTR_ALU_SHL_MI ||
11271 instr->type == INSTR_ALU_SHL_HI ||
11272 instr->type == INSTR_ALU_SHR_MI ||
11273 instr->type == INSTR_ALU_SHR_HI ||
11274 instr->type == INSTR_ALU_AND_I ||
11275 instr->type == INSTR_ALU_OR_I ||
11276 instr->type == INSTR_ALU_XOR_I)
11282 "\t\t.type = %s,\n"
11285 "\t\t\t\t.struct_id = %u,\n"
11286 "\t\t\t\t.n_bits = %u,\n"
11287 "\t\t\t\t.offset = %u,\n"
11290 "\t\t\t\t.struct_id = %u,\n"
11291 "\t\t\t\t.n_bits = %u,\n"
11292 "\t\t\t\t.offset = %u,\n"
11296 instr_type_to_name(instr),
11297 instr->alu.dst.struct_id,
11298 instr->alu.dst.n_bits,
11299 instr->alu.dst.offset,
11300 instr->alu.src.struct_id,
11301 instr->alu.src.n_bits,
11302 instr->alu.src.offset);
11306 "\t\t.type = %s,\n"
11309 "\t\t\t\t.struct_id = %u,\n"
11310 "\t\t\t\t.n_bits = %u,\n"
11311 "\t\t\t\t.offset = %u,\n"
11313 "\t\t\t.src_val = %" PRIu64 ",\n"
11316 instr_type_to_name(instr),
11317 instr->alu.dst.struct_id,
11318 instr->alu.dst.n_bits,
11319 instr->alu.dst.offset,
11320 instr->alu.src_val);
11324 instr_hash_export(struct instruction *instr, FILE *f)
11328 "\t\t.type = %s,\n"
11329 "\t\t.hash_func = {\n"
11330 "\t\t\t.hash_func_id = %u,\n"
11332 "\t\t\t\t.offset = %u,\n"
11333 "\t\t\t\t.n_bits = %u,\n"
11336 "\t\t\t\t.struct_id = %u,\n"
11337 "\t\t\t\t.offset = %u,\n"
11338 "\t\t\t\t.n_bytes = %u,\n"
11342 instr_type_to_name(instr),
11343 instr->hash_func.hash_func_id,
11344 instr->hash_func.dst.offset,
11345 instr->hash_func.dst.n_bits,
11346 instr->hash_func.src.struct_id,
11347 instr->hash_func.src.offset,
11348 instr->hash_func.src.n_bytes);
11352 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11354 int prefetch = 0, idx_imm = 0, src_imm = 0;
11356 if (instr->type == INSTR_REGPREFETCH_RH ||
11357 instr->type == INSTR_REGPREFETCH_RM ||
11358 instr->type == INSTR_REGPREFETCH_RI)
11361 /* index is the 3rd operand for the regrd instruction and the 2nd
11362 * operand for the regwr and regadd instructions.
11364 if (instr->type == INSTR_REGPREFETCH_RI ||
11365 instr->type == INSTR_REGRD_HRI ||
11366 instr->type == INSTR_REGRD_MRI ||
11367 instr->type == INSTR_REGWR_RIH ||
11368 instr->type == INSTR_REGWR_RIM ||
11369 instr->type == INSTR_REGWR_RII ||
11370 instr->type == INSTR_REGADD_RIH ||
11371 instr->type == INSTR_REGADD_RIM ||
11372 instr->type == INSTR_REGADD_RII)
11375 /* src is the 3rd operand for the regwr and regadd instructions. */
11376 if (instr->type == INSTR_REGWR_RHI ||
11377 instr->type == INSTR_REGWR_RMI ||
11378 instr->type == INSTR_REGWR_RII ||
11379 instr->type == INSTR_REGADD_RHI ||
11380 instr->type == INSTR_REGADD_RMI ||
11381 instr->type == INSTR_REGADD_RII)
11384 /* instr.regarray.regarray_id. */
11387 "\t\t.type = %s,\n"
11388 "\t\t.regarray = {\n"
11389 "\t\t\t.regarray_id = %u,\n",
11390 instr_type_to_name(instr),
11391 instr->regarray.regarray_id);
11393 /* instr.regarray.idx / instr.regarray.idx_val. */
11396 "\t\t\t\t.idx = {\n"
11397 "\t\t\t\t\t.struct_id = %u,\n"
11398 "\t\t\t\t\t.n_bits = %u,\n"
11399 "\t\t\t\t\t.offset = %u,\n"
11401 instr->regarray.idx.struct_id,
11402 instr->regarray.idx.n_bits,
11403 instr->regarray.idx.offset);
11406 "\t\t\t\t.idx_val = %u,\n",
11407 instr->regarray.idx_val);
11409 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11413 "\t\t\t\t.dstsrc = {\n"
11414 "\t\t\t\t\t.struct_id = %u,\n"
11415 "\t\t\t\t\t.n_bits = %u,\n"
11416 "\t\t\t\t\t.offset = %u,\n"
11418 instr->regarray.dstsrc.struct_id,
11419 instr->regarray.dstsrc.n_bits,
11420 instr->regarray.dstsrc.offset);
11423 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11424 instr->regarray.dstsrc_val);
11427 /* instr.regarray and instr - closing curly braces. */
11434 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11436 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
11438 if (instr->type == INSTR_METPREFETCH_H ||
11439 instr->type == INSTR_METPREFETCH_M ||
11440 instr->type == INSTR_METPREFETCH_I)
11444 if (instr->type == INSTR_METPREFETCH_I ||
11445 instr->type == INSTR_METER_IHM ||
11446 instr->type == INSTR_METER_IHI ||
11447 instr->type == INSTR_METER_IMM ||
11448 instr->type == INSTR_METER_IMI)
11451 /* color_in_imm. */
11452 if (instr->type == INSTR_METER_HHI ||
11453 instr->type == INSTR_METER_HMI ||
11454 instr->type == INSTR_METER_MHI ||
11455 instr->type == INSTR_METER_MMI ||
11456 instr->type == INSTR_METER_IHI ||
11457 instr->type == INSTR_METER_IMI)
11460 /* instr.meter.metarray_id. */
11463 "\t\t.type = %s,\n"
11465 "\t\t\t.metarray_id = %u,\n",
11466 instr_type_to_name(instr),
11467 instr->meter.metarray_id);
11469 /* instr.meter.idx / instr.meter.idx_val. */
11473 "\t\t\t\t.struct_id = %u,\n"
11474 "\t\t\t\t.n_bits = %u,\n"
11475 "\t\t\t\t.offset = %u,\n"
11477 instr->meter.idx.struct_id,
11478 instr->meter.idx.n_bits,
11479 instr->meter.idx.offset);
11482 "\t\t\t.idx_val = %u,\n",
11483 instr->meter.idx_val);
11486 /* instr.meter.length. */
11488 "\t\t\t.length = {\n"
11489 "\t\t\t\t.struct_id = %u,\n"
11490 "\t\t\t\t.n_bits = %u,\n"
11491 "\t\t\t\t.offset = %u,\n"
11493 instr->meter.length.struct_id,
11494 instr->meter.length.n_bits,
11495 instr->meter.length.offset);
11497 /* instr.meter.color_in / instr.meter.color_in_val. */
11500 "\t\t\t.color_in = {\n"
11501 "\t\t\t\t.struct_id = %u,\n"
11502 "\t\t\t\t.n_bits = %u,\n"
11503 "\t\t\t\t.offset = %u,\n"
11505 instr->meter.color_in.struct_id,
11506 instr->meter.color_in.n_bits,
11507 instr->meter.color_in.offset);
11510 "\t\t\t.color_in_val = %u,\n",
11511 (uint32_t)instr->meter.color_in_val);
11513 /* instr.meter.color_out. */
11515 "\t\t\t.color_out = {\n"
11516 "\t\t\t\t.struct_id = %u,\n"
11517 "\t\t\t\t.n_bits = %u,\n"
11518 "\t\t\t\t.offset = %u,\n"
11520 instr->meter.color_out.struct_id,
11521 instr->meter.color_out.n_bits,
11522 instr->meter.color_out.offset);
11525 /* instr.meter and instr - closing curly braces. */
11532 instr_table_export(struct instruction *instr,
11537 "\t\t.type = %s,\n"
11539 "\t\t\t.table_id = %u,\n"
11542 instr_type_to_name(instr),
11543 instr->table.table_id);
11547 instr_learn_export(struct instruction *instr, FILE *f)
11551 "\t\t.type = %s,\n"
11553 "\t\t\t.action_id = %u,\n"
11554 "\t\t\t.mf_first_arg_offset = %u,\n"
11555 "\t\t\t.mf_timeout_id_offset = %u,\n"
11556 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11559 instr_type_to_name(instr),
11560 instr->learn.action_id,
11561 instr->learn.mf_first_arg_offset,
11562 instr->learn.mf_timeout_id_offset,
11563 instr->learn.mf_timeout_id_n_bits);
11567 instr_rearm_export(struct instruction *instr, FILE *f)
11569 if (instr->type == INSTR_LEARNER_REARM)
11572 "\t\t.type = %s,\n"
11574 instr_type_to_name(instr));
11578 "\t\t.type = %s,\n"
11580 "\t\t\t.mf_timeout_id_offset = %u,\n"
11581 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11584 instr_type_to_name(instr),
11585 instr->learn.mf_timeout_id_offset,
11586 instr->learn.mf_timeout_id_n_bits);
11590 instr_forget_export(struct instruction *instr, FILE *f)
11594 "\t\t.type = %s,\n"
11596 instr_type_to_name(instr));
11600 instr_extern_export(struct instruction *instr, FILE *f)
11602 if (instr->type == INSTR_EXTERN_OBJ)
11605 "\t\t.type = %s,\n"
11606 "\t\t.ext_obj = {\n"
11607 "\t\t\t.ext_obj_id = %u,\n"
11608 "\t\t\t.func_id = %u,\n"
11611 instr_type_to_name(instr),
11612 instr->ext_obj.ext_obj_id,
11613 instr->ext_obj.func_id);
11617 "\t\t.type = %s,\n"
11618 "\t\t.ext_func = {\n"
11619 "\t\t\t.ext_func_id = %u,\n"
11622 instr_type_to_name(instr),
11623 instr->ext_func.ext_func_id);
11627 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11631 "\t\t.type = %s,\n"
11633 "\t\t\t.ip = NULL,\n",
11634 instr_type_to_name(instr));
11636 switch (instr->type) {
11637 case INSTR_JMP_VALID:
11638 case INSTR_JMP_INVALID:
11640 "\t\t\t.header_id = %u,\n",
11641 instr->jmp.header_id);
11644 case INSTR_JMP_ACTION_HIT:
11645 case INSTR_JMP_ACTION_MISS:
11647 "\t\t\t.action_id = %u,\n",
11648 instr->jmp.action_id);
11652 case INSTR_JMP_EQ_MH:
11653 case INSTR_JMP_EQ_HM:
11654 case INSTR_JMP_EQ_HH:
11655 case INSTR_JMP_NEQ:
11656 case INSTR_JMP_NEQ_MH:
11657 case INSTR_JMP_NEQ_HM:
11658 case INSTR_JMP_NEQ_HH:
11660 case INSTR_JMP_LT_MH:
11661 case INSTR_JMP_LT_HM:
11662 case INSTR_JMP_LT_HH:
11664 case INSTR_JMP_GT_MH:
11665 case INSTR_JMP_GT_HM:
11666 case INSTR_JMP_GT_HH:
11669 "\t\t\t\t.struct_id = %u,\n"
11670 "\t\t\t\t.n_bits = %u,\n"
11671 "\t\t\t\t.offset = %u,\n"
11674 "\t\t\t\t.struct_id = %u,\n"
11675 "\t\t\t\t.n_bits = %u,\n"
11676 "\t\t\t\t.offset = %u,\n"
11678 instr->jmp.a.struct_id,
11679 instr->jmp.a.n_bits,
11680 instr->jmp.a.offset,
11681 instr->jmp.b.struct_id,
11682 instr->jmp.b.n_bits,
11683 instr->jmp.b.offset);
11686 case INSTR_JMP_EQ_I:
11687 case INSTR_JMP_NEQ_I:
11688 case INSTR_JMP_LT_MI:
11689 case INSTR_JMP_LT_HI:
11690 case INSTR_JMP_GT_MI:
11691 case INSTR_JMP_GT_HI:
11694 "\t\t\t\t.struct_id = %u,\n"
11695 "\t\t\t\t.n_bits = %u,\n"
11696 "\t\t\t\t.offset = %u,\n"
11698 "\t\t\t.b_val = %" PRIu64 ",\n",
11699 instr->jmp.a.struct_id,
11700 instr->jmp.a.n_bits,
11701 instr->jmp.a.offset,
11715 instr_return_export(struct instruction *instr,
11720 "\t\t.type = %s,\n",
11721 instr_type_to_name(instr));
11727 static instruction_export_t export_table[] = {
11728 [INSTR_RX] = instr_io_export,
11730 [INSTR_TX] = instr_io_export,
11731 [INSTR_TX_I] = instr_io_export,
11732 [INSTR_DROP] = instr_io_export,
11733 [INSTR_MIRROR] = instr_mirror_export,
11734 [INSTR_RECIRCULATE] = instr_recirculate_export,
11735 [INSTR_RECIRCID] = instr_recircid_export,
11737 [INSTR_HDR_EXTRACT] = instr_io_export,
11738 [INSTR_HDR_EXTRACT2] = instr_io_export,
11739 [INSTR_HDR_EXTRACT3] = instr_io_export,
11740 [INSTR_HDR_EXTRACT4] = instr_io_export,
11741 [INSTR_HDR_EXTRACT5] = instr_io_export,
11742 [INSTR_HDR_EXTRACT6] = instr_io_export,
11743 [INSTR_HDR_EXTRACT7] = instr_io_export,
11744 [INSTR_HDR_EXTRACT8] = instr_io_export,
11746 [INSTR_HDR_EXTRACT_M] = instr_io_export,
11748 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11750 [INSTR_HDR_EMIT] = instr_io_export,
11751 [INSTR_HDR_EMIT_TX] = instr_io_export,
11752 [INSTR_HDR_EMIT2_TX] = instr_io_export,
11753 [INSTR_HDR_EMIT3_TX] = instr_io_export,
11754 [INSTR_HDR_EMIT4_TX] = instr_io_export,
11755 [INSTR_HDR_EMIT5_TX] = instr_io_export,
11756 [INSTR_HDR_EMIT6_TX] = instr_io_export,
11757 [INSTR_HDR_EMIT7_TX] = instr_io_export,
11758 [INSTR_HDR_EMIT8_TX] = instr_io_export,
11760 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11761 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11763 [INSTR_MOV] = instr_mov_export,
11764 [INSTR_MOV_MH] = instr_mov_export,
11765 [INSTR_MOV_HM] = instr_mov_export,
11766 [INSTR_MOV_HH] = instr_mov_export,
11767 [INSTR_MOV_I] = instr_mov_export,
11769 [INSTR_DMA_HT] = instr_dma_ht_export,
11770 [INSTR_DMA_HT2] = instr_dma_ht_export,
11771 [INSTR_DMA_HT3] = instr_dma_ht_export,
11772 [INSTR_DMA_HT4] = instr_dma_ht_export,
11773 [INSTR_DMA_HT5] = instr_dma_ht_export,
11774 [INSTR_DMA_HT6] = instr_dma_ht_export,
11775 [INSTR_DMA_HT7] = instr_dma_ht_export,
11776 [INSTR_DMA_HT8] = instr_dma_ht_export,
11778 [INSTR_ALU_ADD] = instr_alu_export,
11779 [INSTR_ALU_ADD_MH] = instr_alu_export,
11780 [INSTR_ALU_ADD_HM] = instr_alu_export,
11781 [INSTR_ALU_ADD_HH] = instr_alu_export,
11782 [INSTR_ALU_ADD_MI] = instr_alu_export,
11783 [INSTR_ALU_ADD_HI] = instr_alu_export,
11785 [INSTR_ALU_SUB] = instr_alu_export,
11786 [INSTR_ALU_SUB_MH] = instr_alu_export,
11787 [INSTR_ALU_SUB_HM] = instr_alu_export,
11788 [INSTR_ALU_SUB_HH] = instr_alu_export,
11789 [INSTR_ALU_SUB_MI] = instr_alu_export,
11790 [INSTR_ALU_SUB_HI] = instr_alu_export,
11792 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11793 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11794 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11795 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11797 [INSTR_ALU_AND] = instr_alu_export,
11798 [INSTR_ALU_AND_MH] = instr_alu_export,
11799 [INSTR_ALU_AND_HM] = instr_alu_export,
11800 [INSTR_ALU_AND_HH] = instr_alu_export,
11801 [INSTR_ALU_AND_I] = instr_alu_export,
11803 [INSTR_ALU_OR] = instr_alu_export,
11804 [INSTR_ALU_OR_MH] = instr_alu_export,
11805 [INSTR_ALU_OR_HM] = instr_alu_export,
11806 [INSTR_ALU_OR_HH] = instr_alu_export,
11807 [INSTR_ALU_OR_I] = instr_alu_export,
11809 [INSTR_ALU_XOR] = instr_alu_export,
11810 [INSTR_ALU_XOR_MH] = instr_alu_export,
11811 [INSTR_ALU_XOR_HM] = instr_alu_export,
11812 [INSTR_ALU_XOR_HH] = instr_alu_export,
11813 [INSTR_ALU_XOR_I] = instr_alu_export,
11815 [INSTR_ALU_SHL] = instr_alu_export,
11816 [INSTR_ALU_SHL_MH] = instr_alu_export,
11817 [INSTR_ALU_SHL_HM] = instr_alu_export,
11818 [INSTR_ALU_SHL_HH] = instr_alu_export,
11819 [INSTR_ALU_SHL_MI] = instr_alu_export,
11820 [INSTR_ALU_SHL_HI] = instr_alu_export,
11822 [INSTR_ALU_SHR] = instr_alu_export,
11823 [INSTR_ALU_SHR_MH] = instr_alu_export,
11824 [INSTR_ALU_SHR_HM] = instr_alu_export,
11825 [INSTR_ALU_SHR_HH] = instr_alu_export,
11826 [INSTR_ALU_SHR_MI] = instr_alu_export,
11827 [INSTR_ALU_SHR_HI] = instr_alu_export,
11829 [INSTR_REGPREFETCH_RH] = instr_reg_export,
11830 [INSTR_REGPREFETCH_RM] = instr_reg_export,
11831 [INSTR_REGPREFETCH_RI] = instr_reg_export,
11833 [INSTR_REGRD_HRH] = instr_reg_export,
11834 [INSTR_REGRD_HRM] = instr_reg_export,
11835 [INSTR_REGRD_MRH] = instr_reg_export,
11836 [INSTR_REGRD_MRM] = instr_reg_export,
11837 [INSTR_REGRD_HRI] = instr_reg_export,
11838 [INSTR_REGRD_MRI] = instr_reg_export,
11840 [INSTR_REGWR_RHH] = instr_reg_export,
11841 [INSTR_REGWR_RHM] = instr_reg_export,
11842 [INSTR_REGWR_RMH] = instr_reg_export,
11843 [INSTR_REGWR_RMM] = instr_reg_export,
11844 [INSTR_REGWR_RHI] = instr_reg_export,
11845 [INSTR_REGWR_RMI] = instr_reg_export,
11846 [INSTR_REGWR_RIH] = instr_reg_export,
11847 [INSTR_REGWR_RIM] = instr_reg_export,
11848 [INSTR_REGWR_RII] = instr_reg_export,
11850 [INSTR_REGADD_RHH] = instr_reg_export,
11851 [INSTR_REGADD_RHM] = instr_reg_export,
11852 [INSTR_REGADD_RMH] = instr_reg_export,
11853 [INSTR_REGADD_RMM] = instr_reg_export,
11854 [INSTR_REGADD_RHI] = instr_reg_export,
11855 [INSTR_REGADD_RMI] = instr_reg_export,
11856 [INSTR_REGADD_RIH] = instr_reg_export,
11857 [INSTR_REGADD_RIM] = instr_reg_export,
11858 [INSTR_REGADD_RII] = instr_reg_export,
11860 [INSTR_METPREFETCH_H] = instr_meter_export,
11861 [INSTR_METPREFETCH_M] = instr_meter_export,
11862 [INSTR_METPREFETCH_I] = instr_meter_export,
11864 [INSTR_METER_HHM] = instr_meter_export,
11865 [INSTR_METER_HHI] = instr_meter_export,
11866 [INSTR_METER_HMM] = instr_meter_export,
11867 [INSTR_METER_HMI] = instr_meter_export,
11868 [INSTR_METER_MHM] = instr_meter_export,
11869 [INSTR_METER_MHI] = instr_meter_export,
11870 [INSTR_METER_MMM] = instr_meter_export,
11871 [INSTR_METER_MMI] = instr_meter_export,
11872 [INSTR_METER_IHM] = instr_meter_export,
11873 [INSTR_METER_IHI] = instr_meter_export,
11874 [INSTR_METER_IMM] = instr_meter_export,
11875 [INSTR_METER_IMI] = instr_meter_export,
11877 [INSTR_TABLE] = instr_table_export,
11878 [INSTR_TABLE_AF] = instr_table_export,
11879 [INSTR_SELECTOR] = instr_table_export,
11880 [INSTR_LEARNER] = instr_table_export,
11881 [INSTR_LEARNER_AF] = instr_table_export,
11883 [INSTR_LEARNER_LEARN] = instr_learn_export,
11884 [INSTR_LEARNER_REARM] = instr_rearm_export,
11885 [INSTR_LEARNER_REARM_NEW] = instr_rearm_export,
11886 [INSTR_LEARNER_FORGET] = instr_forget_export,
11888 [INSTR_EXTERN_OBJ] = instr_extern_export,
11889 [INSTR_EXTERN_FUNC] = instr_extern_export,
11890 [INSTR_HASH_FUNC] = instr_hash_export,
11892 [INSTR_JMP] = instr_jmp_export,
11893 [INSTR_JMP_VALID] = instr_jmp_export,
11894 [INSTR_JMP_INVALID] = instr_jmp_export,
11895 [INSTR_JMP_HIT] = instr_jmp_export,
11896 [INSTR_JMP_MISS] = instr_jmp_export,
11897 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11898 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11900 [INSTR_JMP_EQ] = instr_jmp_export,
11901 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11902 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11903 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11904 [INSTR_JMP_EQ_I] = instr_jmp_export,
11906 [INSTR_JMP_NEQ] = instr_jmp_export,
11907 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11908 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11909 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11910 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11912 [INSTR_JMP_LT] = instr_jmp_export,
11913 [INSTR_JMP_LT_MH] = instr_jmp_export,
11914 [INSTR_JMP_LT_HM] = instr_jmp_export,
11915 [INSTR_JMP_LT_HH] = instr_jmp_export,
11916 [INSTR_JMP_LT_MI] = instr_jmp_export,
11917 [INSTR_JMP_LT_HI] = instr_jmp_export,
11919 [INSTR_JMP_GT] = instr_jmp_export,
11920 [INSTR_JMP_GT_MH] = instr_jmp_export,
11921 [INSTR_JMP_GT_HM] = instr_jmp_export,
11922 [INSTR_JMP_GT_HH] = instr_jmp_export,
11923 [INSTR_JMP_GT_MI] = instr_jmp_export,
11924 [INSTR_JMP_GT_HI] = instr_jmp_export,
11926 [INSTR_RETURN] = instr_return_export,
11930 action_data_codegen(struct action *a, FILE *f)
11935 "static const struct instruction action_%s_instructions[] = {\n",
11938 for (i = 0; i < a->n_instructions; i++) {
11939 struct instruction *instr = &a->instructions[i];
11940 instruction_export_t func = export_table[instr->type];
11945 fprintf(f, "};\n");
11948 static const char *
11949 instr_type_to_func(struct instruction *instr)
11951 switch (instr->type) {
11952 case INSTR_RX: return NULL;
11954 case INSTR_TX: return "__instr_tx_exec";
11955 case INSTR_TX_I: return "__instr_tx_i_exec";
11956 case INSTR_DROP: return "__instr_drop_exec";
11957 case INSTR_MIRROR: return "__instr_mirror_exec";
11958 case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11959 case INSTR_RECIRCID: return "__instr_recircid_exec";
11961 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11962 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11963 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11964 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11965 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11966 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11967 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11968 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11970 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11972 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11974 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11975 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11976 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11977 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11978 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11979 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11980 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11981 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11982 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11984 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11985 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11987 case INSTR_MOV: return "__instr_mov_exec";
11988 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11989 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11990 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11991 case INSTR_MOV_I: return "__instr_mov_i_exec";
11993 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11994 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11995 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11996 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11997 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11998 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11999 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
12000 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
12002 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
12003 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
12004 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
12005 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
12006 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
12007 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
12009 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
12010 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
12011 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
12012 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
12013 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
12014 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
12016 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
12017 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
12018 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
12019 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
12021 case INSTR_ALU_AND: return "__instr_alu_and_exec";
12022 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
12023 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
12024 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
12025 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
12027 case INSTR_ALU_OR: return "__instr_alu_or_exec";
12028 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
12029 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
12030 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
12031 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
12033 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
12034 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
12035 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
12036 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
12037 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
12039 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
12040 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
12041 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
12042 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
12043 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
12044 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
12046 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
12047 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
12048 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
12049 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
12050 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
12051 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
12053 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
12054 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
12055 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
12057 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
12058 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
12059 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
12060 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
12061 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
12062 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
12064 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
12065 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
12066 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
12067 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
12068 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
12069 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
12070 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
12071 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
12072 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
12074 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
12075 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
12076 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
12077 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
12078 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
12079 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
12080 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
12081 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
12082 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
12084 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
12085 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
12086 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
12088 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
12089 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
12090 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
12091 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
12092 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
12093 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
12094 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
12095 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
12096 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
12097 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
12098 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
12099 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
12101 case INSTR_TABLE: return NULL;
12102 case INSTR_TABLE_AF: return NULL;
12103 case INSTR_SELECTOR: return NULL;
12104 case INSTR_LEARNER: return NULL;
12105 case INSTR_LEARNER_AF: return NULL;
12107 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
12108 case INSTR_LEARNER_REARM: return "__instr_rearm_exec";
12109 case INSTR_LEARNER_REARM_NEW: return "__instr_rearm_new_exec";
12110 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
12112 case INSTR_EXTERN_OBJ: return NULL;
12113 case INSTR_EXTERN_FUNC: return NULL;
12114 case INSTR_HASH_FUNC: return "__instr_hash_func_exec";
12116 case INSTR_JMP: return NULL;
12117 case INSTR_JMP_VALID: return NULL;
12118 case INSTR_JMP_INVALID: return NULL;
12119 case INSTR_JMP_HIT: return NULL;
12120 case INSTR_JMP_MISS: return NULL;
12121 case INSTR_JMP_ACTION_HIT: return NULL;
12122 case INSTR_JMP_ACTION_MISS: return NULL;
12123 case INSTR_JMP_EQ: return NULL;
12124 case INSTR_JMP_EQ_MH: return NULL;
12125 case INSTR_JMP_EQ_HM: return NULL;
12126 case INSTR_JMP_EQ_HH: return NULL;
12127 case INSTR_JMP_EQ_I: return NULL;
12128 case INSTR_JMP_NEQ: return NULL;
12129 case INSTR_JMP_NEQ_MH: return NULL;
12130 case INSTR_JMP_NEQ_HM: return NULL;
12131 case INSTR_JMP_NEQ_HH: return NULL;
12132 case INSTR_JMP_NEQ_I: return NULL;
12133 case INSTR_JMP_LT: return NULL;
12134 case INSTR_JMP_LT_MH: return NULL;
12135 case INSTR_JMP_LT_HM: return NULL;
12136 case INSTR_JMP_LT_HH: return NULL;
12137 case INSTR_JMP_LT_MI: return NULL;
12138 case INSTR_JMP_LT_HI: return NULL;
12139 case INSTR_JMP_GT: return NULL;
12140 case INSTR_JMP_GT_MH: return NULL;
12141 case INSTR_JMP_GT_HM: return NULL;
12142 case INSTR_JMP_GT_HH: return NULL;
12143 case INSTR_JMP_GT_MI: return NULL;
12144 case INSTR_JMP_GT_HI: return NULL;
12146 case INSTR_RETURN: return NULL;
12148 default: return NULL;
12153 action_instr_does_tx_codegen(struct action *a,
12154 uint32_t instr_pos,
12155 struct instruction *instr,
12159 "%s(p, t, &action_%s_instructions[%u]);\n"
12160 "\tthread_ip_reset(p, t);\n"
12161 "\tinstr_rx_exec(p);\n"
12163 instr_type_to_func(instr),
12169 action_instr_extern_obj_codegen(struct action *a,
12170 uint32_t instr_pos,
12174 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
12180 action_instr_extern_func_codegen(struct action *a,
12181 uint32_t instr_pos,
12185 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
12191 action_instr_jmp_codegen(struct action *a,
12192 uint32_t instr_pos,
12193 struct instruction *instr,
12194 struct instruction_data *data,
12197 switch (instr->type) {
12204 case INSTR_JMP_VALID:
12206 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12213 case INSTR_JMP_INVALID:
12215 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12222 case INSTR_JMP_HIT:
12229 case INSTR_JMP_MISS:
12236 case INSTR_JMP_ACTION_HIT:
12238 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
12245 case INSTR_JMP_ACTION_MISS:
12247 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
12256 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12257 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12266 case INSTR_JMP_EQ_MH:
12268 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12269 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12278 case INSTR_JMP_EQ_HM:
12280 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12281 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12290 case INSTR_JMP_EQ_HH:
12292 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12293 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12302 case INSTR_JMP_EQ_I:
12304 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12305 "action_%s_instructions[%u].jmp.b_val)\n"
12314 case INSTR_JMP_NEQ:
12316 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12317 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12326 case INSTR_JMP_NEQ_MH:
12328 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12329 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12338 case INSTR_JMP_NEQ_HM:
12340 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12341 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12350 case INSTR_JMP_NEQ_HH:
12352 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12353 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12362 case INSTR_JMP_NEQ_I:
12364 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12365 "action_%s_instructions[%u].jmp.b_val)\n"
12376 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12377 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12386 case INSTR_JMP_LT_MH:
12388 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12389 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12398 case INSTR_JMP_LT_HM:
12400 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12401 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12410 case INSTR_JMP_LT_HH:
12412 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12413 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12422 case INSTR_JMP_LT_MI:
12424 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12425 "action_%s_instructions[%u].jmp.b_val)\n"
12434 case INSTR_JMP_LT_HI:
12436 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12437 "action_%s_instructions[%u].jmp.b_val)\n"
12448 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12449 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12458 case INSTR_JMP_GT_MH:
12460 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12461 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12470 case INSTR_JMP_GT_HM:
12472 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12473 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12482 case INSTR_JMP_GT_HH:
12484 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12485 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12494 case INSTR_JMP_GT_MI:
12496 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12497 "action_%s_instructions[%u].jmp.b_val)\n"
12506 case INSTR_JMP_GT_HI:
12508 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12509 "action_%s_instructions[%u].jmp.b_val)\n"
12524 action_instr_return_codegen(FILE *f)
12531 action_instr_codegen(struct action *a, FILE *f)
12537 "action_%s_run(struct rte_swx_pipeline *p)\n"
12539 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12543 for (i = 0; i < a->n_instructions; i++) {
12544 struct instruction *instr = &a->instructions[i];
12545 struct instruction_data *data = &a->instruction_data[i];
12547 /* Label, if present. */
12548 if (data->label[0])
12549 fprintf(f, "\n%s : ", data->label);
12551 fprintf(f, "\n\t");
12553 /* TX instruction type. */
12554 if (instruction_does_tx(instr)) {
12555 action_instr_does_tx_codegen(a, i, instr, f);
12559 /* Extern object/function instruction type. */
12560 if (instr->type == INSTR_EXTERN_OBJ) {
12561 action_instr_extern_obj_codegen(a, i, f);
12565 if (instr->type == INSTR_EXTERN_FUNC) {
12566 action_instr_extern_func_codegen(a, i, f);
12570 /* Jump instruction type. */
12571 if (instruction_is_jmp(instr)) {
12572 action_instr_jmp_codegen(a, i, instr, data, f);
12576 /* Return instruction type. */
12577 if (instr->type == INSTR_RETURN) {
12578 action_instr_return_codegen(f);
12582 /* Any other instruction type. */
12584 "%s(p, t, &action_%s_instructions[%u]);\n",
12585 instr_type_to_func(instr),
12590 fprintf(f, "}\n\n");
12593 struct instruction_group {
12594 TAILQ_ENTRY(instruction_group) node;
12598 uint32_t first_instr_id;
12600 uint32_t last_instr_id;
12605 TAILQ_HEAD(instruction_group_list, instruction_group);
12607 static struct instruction_group *
12608 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12610 struct instruction_group *g;
12612 TAILQ_FOREACH(g, igl, node)
12613 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12620 instruction_group_list_free(struct instruction_group_list *igl)
12626 struct instruction_group *g;
12628 g = TAILQ_FIRST(igl);
12632 TAILQ_REMOVE(igl, g, node);
12639 static struct instruction_group_list *
12640 instruction_group_list_create(struct rte_swx_pipeline *p)
12642 struct instruction_group_list *igl = NULL;
12643 struct instruction_group *g = NULL;
12644 uint32_t n_groups = 0, i;
12646 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12650 igl = calloc(1, sizeof(struct instruction_group_list));
12656 /* Allocate the first group. */
12657 g = calloc(1, sizeof(struct instruction_group));
12661 /* Iteration 1: Separate the instructions into groups based on the thread yield
12662 * instructions. Do not worry about the jump instructions at this point.
12664 for (i = 0; i < p->n_instructions; i++) {
12665 struct instruction *instr = &p->instructions[i];
12667 /* Check for thread yield instructions. */
12668 if (!instruction_does_thread_yield(instr))
12671 /* If the current group contains at least one instruction, then finalize it (with
12672 * the previous instruction), add it to the list and allocate a new group (that
12673 * starts with the current instruction).
12675 if (i - g->first_instr_id) {
12676 /* Finalize the group. */
12677 g->last_instr_id = i - 1;
12679 /* Add the group to the list. Advance the number of groups. */
12680 TAILQ_INSERT_TAIL(igl, g, node);
12683 /* Allocate a new group. */
12684 g = calloc(1, sizeof(struct instruction_group));
12688 /* Initialize the new group. */
12689 g->group_id = n_groups;
12690 g->first_instr_id = i;
12693 /* Finalize the current group (with the current instruction, therefore this group
12694 * contains just the current thread yield instruction), add it to the list and
12695 * allocate a new group (that starts with the next instruction).
12698 /* Finalize the group. */
12699 g->last_instr_id = i;
12701 /* Add the group to the list. Advance the number of groups. */
12702 TAILQ_INSERT_TAIL(igl, g, node);
12705 /* Allocate a new group. */
12706 g = calloc(1, sizeof(struct instruction_group));
12710 /* Initialize the new group. */
12711 g->group_id = n_groups;
12712 g->first_instr_id = i + 1;
12715 /* Handle the last group. */
12716 if (i - g->first_instr_id) {
12717 /* Finalize the group. */
12718 g->last_instr_id = i - 1;
12720 /* Add the group to the list. Advance the number of groups. */
12721 TAILQ_INSERT_TAIL(igl, g, node);
12728 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12729 * the destination of a jump instruction located in a different group ("far jump"), then the
12730 * current group has to be split, so that the instruction representing the far jump
12731 * destination is at the start of its group.
12734 int is_modified = 0;
12736 for (i = 0; i < p->n_instructions; i++) {
12737 struct instruction_data *data = &p->instruction_data[i];
12738 struct instruction_group *g;
12741 /* Continue when the current instruction is not a jump destination. */
12742 if (!data->n_users)
12745 g = instruction_group_list_group_find(igl, i);
12749 /* Find out all the jump instructions with this destination. */
12750 for (j = 0; j < p->n_instructions; j++) {
12751 struct instruction *jmp_instr = &p->instructions[j];
12752 struct instruction_data *jmp_data = &p->instruction_data[j];
12753 struct instruction_group *jmp_g, *new_g;
12755 /* Continue when not a jump instruction. Even when jump instruction,
12756 * continue when the jump destination is not this instruction.
12758 if (!instruction_is_jmp(jmp_instr) ||
12759 strcmp(jmp_data->jmp_label, data->label))
12762 jmp_g = instruction_group_list_group_find(igl, j);
12766 /* Continue when both the jump instruction and the jump destination
12767 * instruction are in the same group. Even when in different groups,
12768 * still continue if the jump destination instruction is already the
12769 * first instruction of its group.
12771 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12774 /* Split the group of the current jump destination instruction to
12775 * make this instruction the first instruction of a new group.
12777 new_g = calloc(1, sizeof(struct instruction_group));
12781 new_g->group_id = n_groups;
12782 new_g->first_instr_id = i;
12783 new_g->last_instr_id = g->last_instr_id;
12785 g->last_instr_id = i - 1;
12787 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12791 /* The decision to split this group (to make the current instruction
12792 * the first instruction of a new group) is already taken and fully
12793 * implemented, so no need to search for more reasons to do it.
12799 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12800 * previously considered local (i.e. the jump destination is in the same group as
12801 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12802 * different group than the jump instruction). Wost case scenario: each instruction
12803 * that is a jump destination ends up as the first instruction of its group.
12809 /* Re-assign the group IDs to be in incremental order. */
12811 TAILQ_FOREACH(g, igl, node) {
12820 instruction_group_list_free(igl);
12828 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12829 uint32_t instr_pos,
12830 struct instruction *instr,
12834 "%s(p, t, &pipeline_instructions[%u]);\n"
12835 "\tthread_ip_reset(p, t);\n"
12836 "\tinstr_rx_exec(p);\n"
12838 instr_type_to_func(instr),
12843 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12844 struct instruction_group_list *igl,
12845 uint32_t jmp_instr_id,
12846 struct instruction *jmp_instr,
12847 struct instruction_data *jmp_data,
12850 struct instruction_group *jmp_g, *g;
12851 struct instruction_data *data;
12854 switch (jmp_instr->type) {
12858 case INSTR_JMP_VALID:
12860 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12864 case INSTR_JMP_INVALID:
12866 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12870 case INSTR_JMP_HIT:
12875 case INSTR_JMP_MISS:
12880 case INSTR_JMP_ACTION_HIT:
12882 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12886 case INSTR_JMP_ACTION_MISS:
12888 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12894 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12895 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12900 case INSTR_JMP_EQ_MH:
12902 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12903 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12908 case INSTR_JMP_EQ_HM:
12910 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12911 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12916 case INSTR_JMP_EQ_HH:
12918 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12919 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12924 case INSTR_JMP_EQ_I:
12926 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12927 "pipeline_instructions[%u].jmp.b_val)",
12932 case INSTR_JMP_NEQ:
12934 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12935 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12940 case INSTR_JMP_NEQ_MH:
12942 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12943 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12948 case INSTR_JMP_NEQ_HM:
12950 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12951 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12956 case INSTR_JMP_NEQ_HH:
12958 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12959 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12964 case INSTR_JMP_NEQ_I:
12966 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12967 "pipeline_instructions[%u].jmp.b_val)",
12974 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12975 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12980 case INSTR_JMP_LT_MH:
12982 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12983 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12988 case INSTR_JMP_LT_HM:
12990 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12991 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12996 case INSTR_JMP_LT_HH:
12998 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12999 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13004 case INSTR_JMP_LT_MI:
13006 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
13007 "pipeline_instructions[%u].jmp.b_val)",
13012 case INSTR_JMP_LT_HI:
13014 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13015 "pipeline_instructions[%u].jmp.b_val)",
13022 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13023 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13028 case INSTR_JMP_GT_MH:
13030 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13031 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13036 case INSTR_JMP_GT_HM:
13038 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13039 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13044 case INSTR_JMP_GT_HH:
13046 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13047 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13052 case INSTR_JMP_GT_MI:
13054 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13055 "pipeline_instructions[%u].jmp.b_val)",
13060 case INSTR_JMP_GT_HI:
13062 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13063 "pipeline_instructions[%u].jmp.b_val)",
13072 /* Find the instruction group of the jump instruction. */
13073 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
13077 /* Find the instruction group of the jump destination instruction. */
13078 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
13082 instr_id = data - p->instruction_data;
13084 g = instruction_group_list_group_find(igl, instr_id);
13088 /* Code generation for "near" jump (same instruction group) or "far" jump (different
13089 * instruction group).
13091 if (g->group_id == jmp_g->group_id)
13093 "\n\t\tgoto %s;\n",
13094 jmp_data->jmp_label);
13098 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
13107 instruction_group_list_codegen(struct instruction_group_list *igl,
13108 struct rte_swx_pipeline *p,
13111 struct instruction_group *g;
13113 int is_required = 0;
13115 /* Check if code generation is required. */
13116 TAILQ_FOREACH(g, igl, node)
13117 if (g->first_instr_id < g->last_instr_id)
13123 /* Generate the code for the pipeline instruction array. */
13125 "static const struct instruction pipeline_instructions[] = {\n");
13127 for (i = 0; i < p->n_instructions; i++) {
13128 struct instruction *instr = &p->instructions[i];
13129 instruction_export_t func = export_table[instr->type];
13134 fprintf(f, "};\n\n");
13136 /* Generate the code for the pipeline functions: one function for each instruction group
13137 * that contains more than one instruction.
13139 TAILQ_FOREACH(g, igl, node) {
13140 struct instruction *last_instr;
13143 /* Skip if group contains a single instruction. */
13144 if (g->last_instr_id == g->first_instr_id)
13147 /* Generate new pipeline function. */
13150 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
13152 "\tstruct thread *t = &p->threads[p->thread_id];\n"
13156 /* Generate the code for each pipeline instruction. */
13157 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
13158 struct instruction *instr = &p->instructions[j];
13159 struct instruction_data *data = &p->instruction_data[j];
13161 /* Label, if present. */
13162 if (data->label[0])
13163 fprintf(f, "\n%s : ", data->label);
13165 fprintf(f, "\n\t");
13167 /* TX instruction type. */
13168 if (instruction_does_tx(instr)) {
13169 pipeline_instr_does_tx_codegen(p, j, instr, f);
13173 /* Jump instruction type. */
13174 if (instruction_is_jmp(instr)) {
13175 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
13179 /* Any other instruction type. */
13181 "%s(p, t, &pipeline_instructions[%u]);\n",
13182 instr_type_to_func(instr),
13186 /* Finalize the generated pipeline function. For some instructions such as TX,
13187 * emit-many-and-TX and unconditional jump, the next instruction has been already
13188 * decided unconditionally and the instruction pointer of the current thread set
13189 * accordingly; for all the other instructions, the instruction pointer must be
13192 last_instr = &p->instructions[g->last_instr_id];
13194 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
13196 "thread_ip_inc(p);\n");
13205 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
13207 struct instruction_group *g;
13208 uint32_t n_custom_instr = 0;
13210 /* Groups with a single instruction: no function is generated for this group, the group
13211 * keeps its current instruction. Groups with more than two instructions: one function and
13212 * the associated custom instruction get generated for each such group.
13214 TAILQ_FOREACH(g, igl, node) {
13215 if (g->first_instr_id == g->last_instr_id)
13221 return n_custom_instr;
13225 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13230 /* Create the .c file. */
13231 f = fopen("/tmp/pipeline.c", "w");
13235 /* Include the .h file. */
13236 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
13238 /* Add the code for each action. */
13239 TAILQ_FOREACH(a, &p->actions, node) {
13240 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
13242 action_data_codegen(a, f);
13246 action_instr_codegen(a, f);
13251 /* Add the pipeline code. */
13252 instruction_group_list_codegen(igl, p, f);
13254 /* Close the .c file. */
13260 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
13261 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
13265 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13268 struct instruction_group *g;
13269 char *dir_in, *buffer = NULL;
13270 const char *dir_out;
13273 /* Get the environment variables. */
13274 dir_in = getenv("RTE_INSTALL_DIR");
13282 /* Memory allocation for the command buffer. */
13283 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
13290 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13291 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
13292 "-I %s/lib/pipeline "
13293 "-I %s/lib/eal/include "
13294 "-I %s/lib/eal/x86/include "
13295 "-I %s/lib/eal/include/generic "
13299 "-I %s/lib/pipeline "
13302 "-I %s/lib/eal/linux/include "
13303 ">%s/pipeline.log 2>&1 "
13305 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
13306 ">>%s/pipeline.log 2>&1",
13325 /* Build the shared object library. */
13326 status = system(buffer);
13330 /* Open library. */
13332 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13333 "%s/libpipeline.so",
13336 p->lib = dlopen(buffer, RTLD_LAZY);
13342 /* Get the action function symbols. */
13343 TAILQ_FOREACH(a, &p->actions, node) {
13344 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
13346 p->action_funcs[a->id] = dlsym(p->lib, buffer);
13347 if (!p->action_funcs[a->id]) {
13353 /* Get the pipeline function symbols. */
13354 TAILQ_FOREACH(g, igl, node) {
13355 if (g->first_instr_id == g->last_instr_id)
13358 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
13360 g->func = dlsym(p->lib, buffer);
13368 if (status && p->lib) {
13379 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13380 struct instruction_group_list *igl)
13382 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13384 /* Check that enough space is available within the pipeline instruction table to store all
13385 * the custom instructions.
13387 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13394 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13396 struct instruction_group *g;
13399 /* Pipeline table instructions. */
13400 for (i = 0; i < p->n_instructions; i++) {
13401 struct instruction *instr = &p->instructions[i];
13403 if (instr->type == INSTR_TABLE)
13404 instr->type = INSTR_TABLE_AF;
13406 if (instr->type == INSTR_LEARNER)
13407 instr->type = INSTR_LEARNER_AF;
13410 /* Pipeline custom instructions. */
13412 TAILQ_FOREACH(g, igl, node) {
13413 struct instruction *instr = &p->instructions[g->first_instr_id];
13416 if (g->first_instr_id == g->last_instr_id)
13419 /* Install a new custom instruction. */
13420 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13422 /* First instruction of the group: change its type to the new custom instruction. */
13423 instr->type = INSTR_CUSTOM_0 + i;
13425 /* All the subsequent instructions of the group: invalidate. */
13426 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13427 struct instruction_data *data = &p->instruction_data[j];
13435 /* Remove the invalidated instructions. */
13436 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13438 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13439 * instructions that are the only instruction within their group, so they were left
13442 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13446 pipeline_compile(struct rte_swx_pipeline *p)
13448 struct instruction_group_list *igl = NULL;
13451 igl = instruction_group_list_create(p);
13457 /* Code generation. */
13458 status = pipeline_codegen(p, igl);
13462 /* Build and load the shared object library. */
13463 status = pipeline_libload(p, igl);
13467 /* Adjust instructions. */
13468 status = pipeline_adjust_check(p, igl);
13472 pipeline_adjust(p, igl);
13475 instruction_group_list_free(igl);