1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <rte_hash_crc.h>
12 #include <rte_swx_port_ethdev.h>
13 #include <rte_swx_port_fd.h>
14 #include <rte_swx_port_ring.h>
15 #include "rte_swx_port_source_sink.h"
17 #include <rte_swx_table_em.h>
18 #include <rte_swx_table_wm.h>
20 #include "rte_swx_pipeline_internal.h"
22 #define CHECK(condition, err_code) \
28 #define CHECK_NAME(name, err_code) \
31 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
34 #define CHECK_INSTRUCTION(instr, err_code) \
37 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
38 RTE_SWX_INSTRUCTION_SIZE), \
44 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
46 #include <rte_malloc.h>
49 env_malloc(size_t size, size_t alignment, int numa_node)
51 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
55 env_free(void *start, size_t size __rte_unused)
65 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
69 if (numa_available() == -1)
72 start = numa_alloc_onnode(size, numa_node);
76 memset(start, 0, size);
81 env_free(void *start, size_t size)
83 if (numa_available() == -1)
86 numa_free(start, size);
94 static struct struct_type *
95 struct_type_find(struct rte_swx_pipeline *p, const char *name)
97 struct struct_type *elem;
99 TAILQ_FOREACH(elem, &p->struct_types, node)
100 if (strcmp(elem->name, name) == 0)
106 static struct field *
107 struct_type_field_find(struct struct_type *st, const char *name)
111 for (i = 0; i < st->n_fields; i++) {
112 struct field *f = &st->fields[i];
114 if (strcmp(f->name, name) == 0)
122 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
124 struct rte_swx_field_params *fields,
126 int last_field_has_variable_size)
128 struct struct_type *st;
132 CHECK_NAME(name, EINVAL);
133 CHECK(fields, EINVAL);
134 CHECK(n_fields, EINVAL);
136 for (i = 0; i < n_fields; i++) {
137 struct rte_swx_field_params *f = &fields[i];
138 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
141 CHECK_NAME(f->name, EINVAL);
142 CHECK(f->n_bits, EINVAL);
143 CHECK((f->n_bits <= 64) || var_size, EINVAL);
144 CHECK((f->n_bits & 7) == 0, EINVAL);
146 for (j = 0; j < i; j++) {
147 struct rte_swx_field_params *f_prev = &fields[j];
149 CHECK(strcmp(f->name, f_prev->name), EINVAL);
153 CHECK(!struct_type_find(p, name), EEXIST);
155 /* Node allocation. */
156 st = calloc(1, sizeof(struct struct_type));
159 st->fields = calloc(n_fields, sizeof(struct field));
165 /* Node initialization. */
166 strcpy(st->name, name);
167 for (i = 0; i < n_fields; i++) {
168 struct field *dst = &st->fields[i];
169 struct rte_swx_field_params *src = &fields[i];
170 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
172 strcpy(dst->name, src->name);
173 dst->n_bits = src->n_bits;
174 dst->offset = st->n_bits;
175 dst->var_size = var_size;
177 st->n_bits += src->n_bits;
178 st->n_bits_min += var_size ? 0 : src->n_bits;
180 st->n_fields = n_fields;
181 st->var_size = last_field_has_variable_size;
183 /* Node add to tailq. */
184 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
190 struct_build(struct rte_swx_pipeline *p)
194 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
195 struct thread *t = &p->threads[i];
197 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
198 CHECK(t->structs, ENOMEM);
205 struct_build_free(struct rte_swx_pipeline *p)
209 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
210 struct thread *t = &p->threads[i];
218 struct_free(struct rte_swx_pipeline *p)
220 struct_build_free(p);
224 struct struct_type *elem;
226 elem = TAILQ_FIRST(&p->struct_types);
230 TAILQ_REMOVE(&p->struct_types, elem, node);
239 static struct port_in_type *
240 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
242 struct port_in_type *elem;
247 TAILQ_FOREACH(elem, &p->port_in_types, node)
248 if (strcmp(elem->name, name) == 0)
255 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
257 struct rte_swx_port_in_ops *ops)
259 struct port_in_type *elem;
262 CHECK_NAME(name, EINVAL);
264 CHECK(ops->create, EINVAL);
265 CHECK(ops->free, EINVAL);
266 CHECK(ops->pkt_rx, EINVAL);
267 CHECK(ops->stats_read, EINVAL);
269 CHECK(!port_in_type_find(p, name), EEXIST);
271 /* Node allocation. */
272 elem = calloc(1, sizeof(struct port_in_type));
275 /* Node initialization. */
276 strcpy(elem->name, name);
277 memcpy(&elem->ops, ops, sizeof(*ops));
279 /* Node add to tailq. */
280 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
285 static struct port_in *
286 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
288 struct port_in *port;
290 TAILQ_FOREACH(port, &p->ports_in, node)
291 if (port->id == port_id)
298 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
300 const char *port_type_name,
303 struct port_in_type *type = NULL;
304 struct port_in *port = NULL;
309 CHECK(!port_in_find(p, port_id), EINVAL);
311 CHECK_NAME(port_type_name, EINVAL);
312 type = port_in_type_find(p, port_type_name);
315 obj = type->ops.create(args);
318 /* Node allocation. */
319 port = calloc(1, sizeof(struct port_in));
322 /* Node initialization. */
327 /* Node add to tailq. */
328 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
329 if (p->n_ports_in < port_id + 1)
330 p->n_ports_in = port_id + 1;
336 port_in_build(struct rte_swx_pipeline *p)
338 struct port_in *port;
341 CHECK(p->n_ports_in, EINVAL);
342 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
344 for (i = 0; i < p->n_ports_in; i++)
345 CHECK(port_in_find(p, i), EINVAL);
347 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
348 CHECK(p->in, ENOMEM);
350 TAILQ_FOREACH(port, &p->ports_in, node) {
351 struct port_in_runtime *in = &p->in[port->id];
353 in->pkt_rx = port->type->ops.pkt_rx;
361 port_in_build_free(struct rte_swx_pipeline *p)
368 port_in_free(struct rte_swx_pipeline *p)
370 port_in_build_free(p);
374 struct port_in *port;
376 port = TAILQ_FIRST(&p->ports_in);
380 TAILQ_REMOVE(&p->ports_in, port, node);
381 port->type->ops.free(port->obj);
385 /* Input port types. */
387 struct port_in_type *elem;
389 elem = TAILQ_FIRST(&p->port_in_types);
393 TAILQ_REMOVE(&p->port_in_types, elem, node);
401 static struct port_out_type *
402 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
404 struct port_out_type *elem;
409 TAILQ_FOREACH(elem, &p->port_out_types, node)
410 if (!strcmp(elem->name, name))
417 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
419 struct rte_swx_port_out_ops *ops)
421 struct port_out_type *elem;
424 CHECK_NAME(name, EINVAL);
426 CHECK(ops->create, EINVAL);
427 CHECK(ops->free, EINVAL);
428 CHECK(ops->pkt_tx, EINVAL);
429 CHECK(ops->pkt_fast_clone_tx, EINVAL);
430 CHECK(ops->pkt_clone_tx, EINVAL);
431 CHECK(ops->stats_read, EINVAL);
433 CHECK(!port_out_type_find(p, name), EEXIST);
435 /* Node allocation. */
436 elem = calloc(1, sizeof(struct port_out_type));
439 /* Node initialization. */
440 strcpy(elem->name, name);
441 memcpy(&elem->ops, ops, sizeof(*ops));
443 /* Node add to tailq. */
444 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
449 static struct port_out *
450 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
452 struct port_out *port;
454 TAILQ_FOREACH(port, &p->ports_out, node)
455 if (port->id == port_id)
462 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
464 const char *port_type_name,
467 struct port_out_type *type = NULL;
468 struct port_out *port = NULL;
473 CHECK(!port_out_find(p, port_id), EINVAL);
475 CHECK_NAME(port_type_name, EINVAL);
476 type = port_out_type_find(p, port_type_name);
479 obj = type->ops.create(args);
482 /* Node allocation. */
483 port = calloc(1, sizeof(struct port_out));
486 /* Node initialization. */
491 /* Node add to tailq. */
492 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
493 if (p->n_ports_out < port_id + 1)
494 p->n_ports_out = port_id + 1;
500 port_out_build(struct rte_swx_pipeline *p)
502 struct port_out *port;
505 CHECK(p->n_ports_out, EINVAL);
507 for (i = 0; i < p->n_ports_out; i++)
508 CHECK(port_out_find(p, i), EINVAL);
510 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
511 CHECK(p->out, ENOMEM);
513 TAILQ_FOREACH(port, &p->ports_out, node) {
514 struct port_out_runtime *out = &p->out[port->id];
516 out->pkt_tx = port->type->ops.pkt_tx;
517 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
518 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
519 out->flush = port->type->ops.flush;
520 out->obj = port->obj;
527 port_out_build_free(struct rte_swx_pipeline *p)
534 port_out_free(struct rte_swx_pipeline *p)
536 port_out_build_free(p);
540 struct port_out *port;
542 port = TAILQ_FIRST(&p->ports_out);
546 TAILQ_REMOVE(&p->ports_out, port, node);
547 port->type->ops.free(port->obj);
551 /* Output port types. */
553 struct port_out_type *elem;
555 elem = TAILQ_FIRST(&p->port_out_types);
559 TAILQ_REMOVE(&p->port_out_types, elem, node);
568 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
569 struct rte_swx_pipeline_mirroring_params *params)
572 CHECK(params, EINVAL);
573 CHECK(params->n_slots, EINVAL);
574 CHECK(params->n_sessions, EINVAL);
575 CHECK(!p->build_done, EEXIST);
577 p->n_mirroring_slots = rte_align32pow2(params->n_slots);
578 if (p->n_mirroring_slots > 64)
579 p->n_mirroring_slots = 64;
581 p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
587 mirroring_build_free(struct rte_swx_pipeline *p)
591 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
592 struct thread *t = &p->threads[i];
594 /* mirroring_slots. */
595 free(t->mirroring_slots);
596 t->mirroring_slots = NULL;
599 /* mirroring_sessions. */
600 free(p->mirroring_sessions);
601 p->mirroring_sessions = NULL;
605 mirroring_free(struct rte_swx_pipeline *p)
607 mirroring_build_free(p);
611 mirroring_build(struct rte_swx_pipeline *p)
615 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
616 struct thread *t = &p->threads[i];
618 /* mirroring_slots. */
619 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
620 if (!t->mirroring_slots)
624 /* mirroring_sessions. */
625 p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
626 if (!p->mirroring_sessions)
632 mirroring_build_free(p);
639 static struct extern_type *
640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
642 struct extern_type *elem;
644 TAILQ_FOREACH(elem, &p->extern_types, node)
645 if (strcmp(elem->name, name) == 0)
651 static struct extern_type_member_func *
652 extern_type_member_func_find(struct extern_type *type, const char *name)
654 struct extern_type_member_func *elem;
656 TAILQ_FOREACH(elem, &type->funcs, node)
657 if (strcmp(elem->name, name) == 0)
663 static struct extern_obj *
664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
666 struct extern_obj *elem;
668 TAILQ_FOREACH(elem, &p->extern_objs, node)
669 if (strcmp(elem->name, name) == 0)
675 static struct extern_type_member_func *
676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
678 struct extern_obj **obj)
680 struct extern_obj *object;
681 struct extern_type_member_func *func;
682 char *object_name, *func_name;
684 if (name[0] != 'e' || name[1] != '.')
687 object_name = strdup(&name[2]);
691 func_name = strchr(object_name, '.');
700 object = extern_obj_find(p, object_name);
706 func = extern_type_member_func_find(object->type, func_name);
719 static struct field *
720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
722 struct extern_obj **object)
724 struct extern_obj *obj;
726 char *obj_name, *field_name;
728 if ((name[0] != 'e') || (name[1] != '.'))
731 obj_name = strdup(&name[2]);
735 field_name = strchr(obj_name, '.');
744 obj = extern_obj_find(p, obj_name);
750 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
766 const char *mailbox_struct_type_name,
767 rte_swx_extern_type_constructor_t constructor,
768 rte_swx_extern_type_destructor_t destructor)
770 struct extern_type *elem;
771 struct struct_type *mailbox_struct_type;
775 CHECK_NAME(name, EINVAL);
776 CHECK(!extern_type_find(p, name), EEXIST);
778 CHECK_NAME(mailbox_struct_type_name, EINVAL);
779 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
780 CHECK(mailbox_struct_type, EINVAL);
781 CHECK(!mailbox_struct_type->var_size, EINVAL);
783 CHECK(constructor, EINVAL);
784 CHECK(destructor, EINVAL);
786 /* Node allocation. */
787 elem = calloc(1, sizeof(struct extern_type));
790 /* Node initialization. */
791 strcpy(elem->name, name);
792 elem->mailbox_struct_type = mailbox_struct_type;
793 elem->constructor = constructor;
794 elem->destructor = destructor;
795 TAILQ_INIT(&elem->funcs);
797 /* Node add to tailq. */
798 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
805 const char *extern_type_name,
807 rte_swx_extern_type_member_func_t member_func)
809 struct extern_type *type;
810 struct extern_type_member_func *type_member;
814 CHECK_NAME(extern_type_name, EINVAL);
815 type = extern_type_find(p, extern_type_name);
817 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
819 CHECK_NAME(name, EINVAL);
820 CHECK(!extern_type_member_func_find(type, name), EEXIST);
822 CHECK(member_func, EINVAL);
824 /* Node allocation. */
825 type_member = calloc(1, sizeof(struct extern_type_member_func));
826 CHECK(type_member, ENOMEM);
828 /* Node initialization. */
829 strcpy(type_member->name, name);
830 type_member->func = member_func;
831 type_member->id = type->n_funcs;
833 /* Node add to tailq. */
834 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
842 const char *extern_type_name,
846 struct extern_type *type;
847 struct extern_obj *obj;
852 CHECK_NAME(extern_type_name, EINVAL);
853 type = extern_type_find(p, extern_type_name);
856 CHECK_NAME(name, EINVAL);
857 CHECK(!extern_obj_find(p, name), EEXIST);
859 /* Node allocation. */
860 obj = calloc(1, sizeof(struct extern_obj));
863 /* Object construction. */
864 obj_handle = type->constructor(args);
870 /* Node initialization. */
871 strcpy(obj->name, name);
873 obj->obj = obj_handle;
874 obj->struct_id = p->n_structs;
875 obj->id = p->n_extern_objs;
877 /* Node add to tailq. */
878 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
886 extern_obj_build(struct rte_swx_pipeline *p)
890 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
891 struct thread *t = &p->threads[i];
892 struct extern_obj *obj;
894 t->extern_objs = calloc(p->n_extern_objs,
895 sizeof(struct extern_obj_runtime));
896 CHECK(t->extern_objs, ENOMEM);
898 TAILQ_FOREACH(obj, &p->extern_objs, node) {
899 struct extern_obj_runtime *r =
900 &t->extern_objs[obj->id];
901 struct extern_type_member_func *func;
902 uint32_t mailbox_size =
903 obj->type->mailbox_struct_type->n_bits / 8;
907 r->mailbox = calloc(1, mailbox_size);
908 CHECK(r->mailbox, ENOMEM);
910 TAILQ_FOREACH(func, &obj->type->funcs, node)
911 r->funcs[func->id] = func->func;
913 t->structs[obj->struct_id] = r->mailbox;
921 extern_obj_build_free(struct rte_swx_pipeline *p)
925 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
926 struct thread *t = &p->threads[i];
932 for (j = 0; j < p->n_extern_objs; j++) {
933 struct extern_obj_runtime *r = &t->extern_objs[j];
938 free(t->extern_objs);
939 t->extern_objs = NULL;
944 extern_obj_free(struct rte_swx_pipeline *p)
946 extern_obj_build_free(p);
948 /* Extern objects. */
950 struct extern_obj *elem;
952 elem = TAILQ_FIRST(&p->extern_objs);
956 TAILQ_REMOVE(&p->extern_objs, elem, node);
958 elem->type->destructor(elem->obj);
964 struct extern_type *elem;
966 elem = TAILQ_FIRST(&p->extern_types);
970 TAILQ_REMOVE(&p->extern_types, elem, node);
973 struct extern_type_member_func *func;
975 func = TAILQ_FIRST(&elem->funcs);
979 TAILQ_REMOVE(&elem->funcs, func, node);
990 static struct extern_func *
991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
993 struct extern_func *elem;
995 TAILQ_FOREACH(elem, &p->extern_funcs, node)
996 if (strcmp(elem->name, name) == 0)
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1006 if (name[0] != 'f' || name[1] != '.')
1009 return extern_func_find(p, &name[2]);
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1015 struct extern_func **function)
1017 struct extern_func *func;
1019 char *func_name, *field_name;
1021 if ((name[0] != 'f') || (name[1] != '.'))
1024 func_name = strdup(&name[2]);
1028 field_name = strchr(func_name, '.');
1037 func = extern_func_find(p, func_name);
1043 f = struct_type_field_find(func->mailbox_struct_type, field_name);
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1059 const char *mailbox_struct_type_name,
1060 rte_swx_extern_func_t func)
1062 struct extern_func *f;
1063 struct struct_type *mailbox_struct_type;
1067 CHECK_NAME(name, EINVAL);
1068 CHECK(!extern_func_find(p, name), EEXIST);
1070 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072 CHECK(mailbox_struct_type, EINVAL);
1073 CHECK(!mailbox_struct_type->var_size, EINVAL);
1075 CHECK(func, EINVAL);
1077 /* Node allocation. */
1078 f = calloc(1, sizeof(struct extern_func));
1079 CHECK(func, ENOMEM);
1081 /* Node initialization. */
1082 strcpy(f->name, name);
1083 f->mailbox_struct_type = mailbox_struct_type;
1085 f->struct_id = p->n_structs;
1086 f->id = p->n_extern_funcs;
1088 /* Node add to tailq. */
1089 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090 p->n_extern_funcs++;
1097 extern_func_build(struct rte_swx_pipeline *p)
1101 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102 struct thread *t = &p->threads[i];
1103 struct extern_func *func;
1105 /* Memory allocation. */
1106 t->extern_funcs = calloc(p->n_extern_funcs,
1107 sizeof(struct extern_func_runtime));
1108 CHECK(t->extern_funcs, ENOMEM);
1110 /* Extern function. */
1111 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112 struct extern_func_runtime *r =
1113 &t->extern_funcs[func->id];
1114 uint32_t mailbox_size =
1115 func->mailbox_struct_type->n_bits / 8;
1117 r->func = func->func;
1119 r->mailbox = calloc(1, mailbox_size);
1120 CHECK(r->mailbox, ENOMEM);
1122 t->structs[func->struct_id] = r->mailbox;
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1134 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135 struct thread *t = &p->threads[i];
1138 if (!t->extern_funcs)
1141 for (j = 0; j < p->n_extern_funcs; j++) {
1142 struct extern_func_runtime *r = &t->extern_funcs[j];
1147 free(t->extern_funcs);
1148 t->extern_funcs = NULL;
1153 extern_func_free(struct rte_swx_pipeline *p)
1155 extern_func_build_free(p);
1158 struct extern_func *elem;
1160 elem = TAILQ_FIRST(&p->extern_funcs);
1164 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1172 static struct hash_func *
1173 hash_func_find(struct rte_swx_pipeline *p, const char *name)
1175 struct hash_func *elem;
1177 TAILQ_FOREACH(elem, &p->hash_funcs, node)
1178 if (strcmp(elem->name, name) == 0)
1185 rte_swx_pipeline_hash_func_register(struct rte_swx_pipeline *p,
1187 rte_swx_hash_func_t func)
1189 struct hash_func *f;
1193 CHECK_NAME(name, EINVAL);
1194 CHECK(!hash_func_find(p, name), EEXIST);
1196 CHECK(func, EINVAL);
1198 /* Node allocation. */
1199 f = calloc(1, sizeof(struct hash_func));
1200 CHECK(func, ENOMEM);
1202 /* Node initialization. */
1203 strcpy(f->name, name);
1205 f->id = p->n_hash_funcs;
1207 /* Node add to tailq. */
1208 TAILQ_INSERT_TAIL(&p->hash_funcs, f, node);
1215 hash_func_build(struct rte_swx_pipeline *p)
1217 struct hash_func *func;
1219 /* Memory allocation. */
1220 p->hash_func_runtime = calloc(p->n_hash_funcs, sizeof(struct hash_func_runtime));
1221 CHECK(p->hash_func_runtime, ENOMEM);
1223 /* Hash function. */
1224 TAILQ_FOREACH(func, &p->hash_funcs, node) {
1225 struct hash_func_runtime *r = &p->hash_func_runtime[func->id];
1227 r->func = func->func;
1234 hash_func_build_free(struct rte_swx_pipeline *p)
1236 free(p->hash_func_runtime);
1237 p->hash_func_runtime = NULL;
1241 hash_func_free(struct rte_swx_pipeline *p)
1243 hash_func_build_free(p);
1246 struct hash_func *elem;
1248 elem = TAILQ_FIRST(&p->hash_funcs);
1252 TAILQ_REMOVE(&p->hash_funcs, elem, node);
1260 static struct header *
1261 header_find(struct rte_swx_pipeline *p, const char *name)
1263 struct header *elem;
1265 TAILQ_FOREACH(elem, &p->headers, node)
1266 if (strcmp(elem->name, name) == 0)
1272 static struct header *
1273 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1275 struct header *elem;
1277 TAILQ_FOREACH(elem, &p->headers, node)
1278 if (elem->struct_id == struct_id)
1284 static struct header *
1285 header_parse(struct rte_swx_pipeline *p,
1288 if (name[0] != 'h' || name[1] != '.')
1291 return header_find(p, &name[2]);
1294 static struct field *
1295 header_field_parse(struct rte_swx_pipeline *p,
1297 struct header **header)
1301 char *header_name, *field_name;
1303 if ((name[0] != 'h') || (name[1] != '.'))
1306 header_name = strdup(&name[2]);
1310 field_name = strchr(header_name, '.');
1319 h = header_find(p, header_name);
1325 f = struct_type_field_find(h->st, field_name);
1339 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1341 const char *struct_type_name)
1343 struct struct_type *st;
1345 size_t n_headers_max;
1348 CHECK_NAME(name, EINVAL);
1349 CHECK_NAME(struct_type_name, EINVAL);
1351 CHECK(!header_find(p, name), EEXIST);
1353 st = struct_type_find(p, struct_type_name);
1356 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1357 CHECK(p->n_headers < n_headers_max, ENOSPC);
1359 /* Node allocation. */
1360 h = calloc(1, sizeof(struct header));
1363 /* Node initialization. */
1364 strcpy(h->name, name);
1366 h->struct_id = p->n_structs;
1367 h->id = p->n_headers;
1369 /* Node add to tailq. */
1370 TAILQ_INSERT_TAIL(&p->headers, h, node);
1378 header_build(struct rte_swx_pipeline *p)
1381 uint32_t n_bytes = 0, i;
1383 TAILQ_FOREACH(h, &p->headers, node) {
1384 n_bytes += h->st->n_bits / 8;
1387 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1388 struct thread *t = &p->threads[i];
1389 uint32_t offset = 0;
1391 t->headers = calloc(p->n_headers,
1392 sizeof(struct header_runtime));
1393 CHECK(t->headers, ENOMEM);
1395 t->headers_out = calloc(p->n_headers,
1396 sizeof(struct header_out_runtime));
1397 CHECK(t->headers_out, ENOMEM);
1399 t->header_storage = calloc(1, n_bytes);
1400 CHECK(t->header_storage, ENOMEM);
1402 t->header_out_storage = calloc(1, n_bytes);
1403 CHECK(t->header_out_storage, ENOMEM);
1405 TAILQ_FOREACH(h, &p->headers, node) {
1406 uint8_t *header_storage;
1407 uint32_t n_bytes = h->st->n_bits / 8;
1409 header_storage = &t->header_storage[offset];
1412 t->headers[h->id].ptr0 = header_storage;
1413 t->headers[h->id].n_bytes = n_bytes;
1415 t->structs[h->struct_id] = header_storage;
1423 header_build_free(struct rte_swx_pipeline *p)
1427 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1428 struct thread *t = &p->threads[i];
1430 free(t->headers_out);
1431 t->headers_out = NULL;
1436 free(t->header_out_storage);
1437 t->header_out_storage = NULL;
1439 free(t->header_storage);
1440 t->header_storage = NULL;
1445 header_free(struct rte_swx_pipeline *p)
1447 header_build_free(p);
1450 struct header *elem;
1452 elem = TAILQ_FIRST(&p->headers);
1456 TAILQ_REMOVE(&p->headers, elem, node);
1464 static struct field *
1465 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1467 if (!p->metadata_st)
1470 if (name[0] != 'm' || name[1] != '.')
1473 return struct_type_field_find(p->metadata_st, &name[2]);
1477 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1478 const char *struct_type_name)
1480 struct struct_type *st = NULL;
1484 CHECK_NAME(struct_type_name, EINVAL);
1485 st = struct_type_find(p, struct_type_name);
1487 CHECK(!st->var_size, EINVAL);
1488 CHECK(!p->metadata_st, EINVAL);
1490 p->metadata_st = st;
1491 p->metadata_struct_id = p->n_structs;
1499 metadata_build(struct rte_swx_pipeline *p)
1501 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1504 /* Thread-level initialization. */
1505 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1506 struct thread *t = &p->threads[i];
1509 metadata = calloc(1, n_bytes);
1510 CHECK(metadata, ENOMEM);
1512 t->metadata = metadata;
1513 t->structs[p->metadata_struct_id] = metadata;
1520 metadata_build_free(struct rte_swx_pipeline *p)
1524 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1525 struct thread *t = &p->threads[i];
1533 metadata_free(struct rte_swx_pipeline *p)
1535 metadata_build_free(p);
1542 instruction_is_tx(enum instruction_type type)
1556 instruction_does_tx(struct instruction *instr)
1558 switch (instr->type) {
1562 case INSTR_HDR_EMIT_TX:
1563 case INSTR_HDR_EMIT2_TX:
1564 case INSTR_HDR_EMIT3_TX:
1565 case INSTR_HDR_EMIT4_TX:
1566 case INSTR_HDR_EMIT5_TX:
1567 case INSTR_HDR_EMIT6_TX:
1568 case INSTR_HDR_EMIT7_TX:
1569 case INSTR_HDR_EMIT8_TX:
1577 instruction_is_jmp(struct instruction *instr)
1579 switch (instr->type) {
1581 case INSTR_JMP_VALID:
1582 case INSTR_JMP_INVALID:
1584 case INSTR_JMP_MISS:
1585 case INSTR_JMP_ACTION_HIT:
1586 case INSTR_JMP_ACTION_MISS:
1588 case INSTR_JMP_EQ_MH:
1589 case INSTR_JMP_EQ_HM:
1590 case INSTR_JMP_EQ_HH:
1591 case INSTR_JMP_EQ_I:
1593 case INSTR_JMP_NEQ_MH:
1594 case INSTR_JMP_NEQ_HM:
1595 case INSTR_JMP_NEQ_HH:
1596 case INSTR_JMP_NEQ_I:
1598 case INSTR_JMP_LT_MH:
1599 case INSTR_JMP_LT_HM:
1600 case INSTR_JMP_LT_HH:
1601 case INSTR_JMP_LT_MI:
1602 case INSTR_JMP_LT_HI:
1604 case INSTR_JMP_GT_MH:
1605 case INSTR_JMP_GT_HM:
1606 case INSTR_JMP_GT_HH:
1607 case INSTR_JMP_GT_MI:
1608 case INSTR_JMP_GT_HI:
1617 instruction_does_thread_yield(struct instruction *instr)
1619 switch (instr->type) {
1622 case INSTR_TABLE_AF:
1623 case INSTR_SELECTOR:
1625 case INSTR_LEARNER_AF:
1626 case INSTR_EXTERN_OBJ:
1627 case INSTR_EXTERN_FUNC:
1634 static struct field *
1635 action_field_parse(struct action *action, const char *name);
1637 static struct field *
1638 struct_field_parse(struct rte_swx_pipeline *p,
1639 struct action *action,
1641 uint32_t *struct_id)
1648 struct header *header;
1650 f = header_field_parse(p, name, &header);
1654 *struct_id = header->struct_id;
1660 f = metadata_field_parse(p, name);
1664 *struct_id = p->metadata_struct_id;
1673 f = action_field_parse(action, name);
1683 struct extern_obj *obj;
1685 f = extern_obj_mailbox_field_parse(p, name, &obj);
1689 *struct_id = obj->struct_id;
1695 struct extern_func *func;
1697 f = extern_func_mailbox_field_parse(p, name, &func);
1701 *struct_id = func->struct_id;
1714 instr_rx_translate(struct rte_swx_pipeline *p,
1715 struct action *action,
1718 struct instruction *instr,
1719 struct instruction_data *data __rte_unused)
1723 CHECK(!action, EINVAL);
1724 CHECK(n_tokens == 2, EINVAL);
1726 f = metadata_field_parse(p, tokens[1]);
1729 instr->type = INSTR_RX;
1730 instr->io.io.offset = f->offset / 8;
1731 instr->io.io.n_bits = f->n_bits;
1739 instr_tx_translate(struct rte_swx_pipeline *p,
1740 struct action *action __rte_unused,
1743 struct instruction *instr,
1744 struct instruction_data *data __rte_unused)
1746 char *port = tokens[1];
1750 CHECK(n_tokens == 2, EINVAL);
1752 f = metadata_field_parse(p, port);
1754 instr->type = INSTR_TX;
1755 instr->io.io.offset = f->offset / 8;
1756 instr->io.io.n_bits = f->n_bits;
1761 port_val = strtoul(port, &port, 0);
1762 CHECK(!port[0], EINVAL);
1764 instr->type = INSTR_TX_I;
1765 instr->io.io.val = port_val;
1770 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1771 struct action *action __rte_unused,
1772 char **tokens __rte_unused,
1774 struct instruction *instr,
1775 struct instruction_data *data __rte_unused)
1777 CHECK(n_tokens == 1, EINVAL);
1780 instr->type = INSTR_DROP;
1785 instr_tx_exec(struct rte_swx_pipeline *p)
1787 struct thread *t = &p->threads[p->thread_id];
1788 struct instruction *ip = t->ip;
1790 __instr_tx_exec(p, t, ip);
1793 thread_ip_reset(p, t);
1798 instr_tx_i_exec(struct rte_swx_pipeline *p)
1800 struct thread *t = &p->threads[p->thread_id];
1801 struct instruction *ip = t->ip;
1803 __instr_tx_i_exec(p, t, ip);
1806 thread_ip_reset(p, t);
1811 instr_drop_exec(struct rte_swx_pipeline *p)
1813 struct thread *t = &p->threads[p->thread_id];
1814 struct instruction *ip = t->ip;
1816 __instr_drop_exec(p, t, ip);
1819 thread_ip_reset(p, t);
1827 instr_mirror_translate(struct rte_swx_pipeline *p,
1828 struct action *action,
1831 struct instruction *instr,
1832 struct instruction_data *data __rte_unused)
1834 char *dst = tokens[1], *src = tokens[2];
1835 struct field *fdst, *fsrc;
1836 uint32_t dst_struct_id = 0, src_struct_id = 0;
1838 CHECK(n_tokens == 3, EINVAL);
1840 fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1841 CHECK(fdst, EINVAL);
1842 CHECK(dst[0] != 'h', EINVAL);
1843 CHECK(!fdst->var_size, EINVAL);
1845 fsrc = struct_field_parse(p, action, src, &src_struct_id);
1846 CHECK(fsrc, EINVAL);
1847 CHECK(src[0] != 'h', EINVAL);
1848 CHECK(!fsrc->var_size, EINVAL);
1850 instr->type = INSTR_MIRROR;
1851 instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1852 instr->mirror.dst.n_bits = fdst->n_bits;
1853 instr->mirror.dst.offset = fdst->offset / 8;
1854 instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1855 instr->mirror.src.n_bits = fsrc->n_bits;
1856 instr->mirror.src.offset = fsrc->offset / 8;
1862 instr_mirror_exec(struct rte_swx_pipeline *p)
1864 struct thread *t = &p->threads[p->thread_id];
1865 struct instruction *ip = t->ip;
1867 __instr_mirror_exec(p, t, ip);
1877 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1878 struct action *action __rte_unused,
1879 char **tokens __rte_unused,
1881 struct instruction *instr,
1882 struct instruction_data *data __rte_unused)
1884 CHECK(n_tokens == 1, EINVAL);
1886 instr->type = INSTR_RECIRCULATE;
1891 instr_recircid_translate(struct rte_swx_pipeline *p,
1892 struct action *action __rte_unused,
1895 struct instruction *instr,
1896 struct instruction_data *data __rte_unused)
1900 CHECK(n_tokens == 2, EINVAL);
1902 f = metadata_field_parse(p, tokens[1]);
1905 instr->type = INSTR_RECIRCID;
1906 instr->io.io.offset = f->offset / 8;
1907 instr->io.io.n_bits = f->n_bits;
1912 instr_recirculate_exec(struct rte_swx_pipeline *p)
1914 struct thread *t = &p->threads[p->thread_id];
1915 struct instruction *ip = t->ip;
1917 __instr_recirculate_exec(p, t, ip);
1924 instr_recircid_exec(struct rte_swx_pipeline *p)
1926 struct thread *t = &p->threads[p->thread_id];
1927 struct instruction *ip = t->ip;
1929 __instr_recircid_exec(p, t, ip);
1939 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1940 struct action *action,
1943 struct instruction *instr,
1944 struct instruction_data *data __rte_unused)
1948 CHECK(!action, EINVAL);
1949 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1951 h = header_parse(p, tokens[1]);
1954 if (n_tokens == 2) {
1955 CHECK(!h->st->var_size, EINVAL);
1957 instr->type = INSTR_HDR_EXTRACT;
1958 instr->io.hdr.header_id[0] = h->id;
1959 instr->io.hdr.struct_id[0] = h->struct_id;
1960 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1964 CHECK(h->st->var_size, EINVAL);
1966 mf = metadata_field_parse(p, tokens[2]);
1968 CHECK(!mf->var_size, EINVAL);
1970 instr->type = INSTR_HDR_EXTRACT_M;
1971 instr->io.io.offset = mf->offset / 8;
1972 instr->io.io.n_bits = mf->n_bits;
1973 instr->io.hdr.header_id[0] = h->id;
1974 instr->io.hdr.struct_id[0] = h->struct_id;
1975 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1982 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1983 struct action *action,
1986 struct instruction *instr,
1987 struct instruction_data *data __rte_unused)
1991 CHECK(!action, EINVAL);
1992 CHECK(n_tokens == 2, EINVAL);
1994 h = header_parse(p, tokens[1]);
1996 CHECK(!h->st->var_size, EINVAL);
1998 instr->type = INSTR_HDR_LOOKAHEAD;
1999 instr->io.hdr.header_id[0] = h->id;
2000 instr->io.hdr.struct_id[0] = h->struct_id;
2001 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
2007 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2009 struct thread *t = &p->threads[p->thread_id];
2010 struct instruction *ip = t->ip;
2012 __instr_hdr_extract_exec(p, t, ip);
2019 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2021 struct thread *t = &p->threads[p->thread_id];
2022 struct instruction *ip = t->ip;
2024 __instr_hdr_extract2_exec(p, t, ip);
2031 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2033 struct thread *t = &p->threads[p->thread_id];
2034 struct instruction *ip = t->ip;
2036 __instr_hdr_extract3_exec(p, t, ip);
2043 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2045 struct thread *t = &p->threads[p->thread_id];
2046 struct instruction *ip = t->ip;
2048 __instr_hdr_extract4_exec(p, t, ip);
2055 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2057 struct thread *t = &p->threads[p->thread_id];
2058 struct instruction *ip = t->ip;
2060 __instr_hdr_extract5_exec(p, t, ip);
2067 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2069 struct thread *t = &p->threads[p->thread_id];
2070 struct instruction *ip = t->ip;
2072 __instr_hdr_extract6_exec(p, t, ip);
2079 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2081 struct thread *t = &p->threads[p->thread_id];
2082 struct instruction *ip = t->ip;
2084 __instr_hdr_extract7_exec(p, t, ip);
2091 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2093 struct thread *t = &p->threads[p->thread_id];
2094 struct instruction *ip = t->ip;
2096 __instr_hdr_extract8_exec(p, t, ip);
2103 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2105 struct thread *t = &p->threads[p->thread_id];
2106 struct instruction *ip = t->ip;
2108 __instr_hdr_extract_m_exec(p, t, ip);
2115 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2117 struct thread *t = &p->threads[p->thread_id];
2118 struct instruction *ip = t->ip;
2120 __instr_hdr_lookahead_exec(p, t, ip);
2130 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2131 struct action *action __rte_unused,
2134 struct instruction *instr,
2135 struct instruction_data *data __rte_unused)
2139 CHECK(n_tokens == 2, EINVAL);
2141 h = header_parse(p, tokens[1]);
2144 instr->type = INSTR_HDR_EMIT;
2145 instr->io.hdr.header_id[0] = h->id;
2146 instr->io.hdr.struct_id[0] = h->struct_id;
2147 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2152 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2154 struct thread *t = &p->threads[p->thread_id];
2155 struct instruction *ip = t->ip;
2157 __instr_hdr_emit_exec(p, t, ip);
2164 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2166 struct thread *t = &p->threads[p->thread_id];
2167 struct instruction *ip = t->ip;
2169 __instr_hdr_emit_tx_exec(p, t, ip);
2172 thread_ip_reset(p, t);
2177 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2179 struct thread *t = &p->threads[p->thread_id];
2180 struct instruction *ip = t->ip;
2182 __instr_hdr_emit2_tx_exec(p, t, ip);
2185 thread_ip_reset(p, t);
2190 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2192 struct thread *t = &p->threads[p->thread_id];
2193 struct instruction *ip = t->ip;
2195 __instr_hdr_emit3_tx_exec(p, t, ip);
2198 thread_ip_reset(p, t);
2203 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2205 struct thread *t = &p->threads[p->thread_id];
2206 struct instruction *ip = t->ip;
2208 __instr_hdr_emit4_tx_exec(p, t, ip);
2211 thread_ip_reset(p, t);
2216 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2218 struct thread *t = &p->threads[p->thread_id];
2219 struct instruction *ip = t->ip;
2221 __instr_hdr_emit5_tx_exec(p, t, ip);
2224 thread_ip_reset(p, t);
2229 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2231 struct thread *t = &p->threads[p->thread_id];
2232 struct instruction *ip = t->ip;
2234 __instr_hdr_emit6_tx_exec(p, t, ip);
2237 thread_ip_reset(p, t);
2242 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2244 struct thread *t = &p->threads[p->thread_id];
2245 struct instruction *ip = t->ip;
2247 __instr_hdr_emit7_tx_exec(p, t, ip);
2250 thread_ip_reset(p, t);
2255 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2257 struct thread *t = &p->threads[p->thread_id];
2258 struct instruction *ip = t->ip;
2260 __instr_hdr_emit8_tx_exec(p, t, ip);
2263 thread_ip_reset(p, t);
2271 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2272 struct action *action __rte_unused,
2275 struct instruction *instr,
2276 struct instruction_data *data __rte_unused)
2280 CHECK(n_tokens == 2, EINVAL);
2282 h = header_parse(p, tokens[1]);
2285 instr->type = INSTR_HDR_VALIDATE;
2286 instr->valid.header_id = h->id;
2291 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2293 struct thread *t = &p->threads[p->thread_id];
2294 struct instruction *ip = t->ip;
2296 __instr_hdr_validate_exec(p, t, ip);
2306 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2307 struct action *action __rte_unused,
2310 struct instruction *instr,
2311 struct instruction_data *data __rte_unused)
2315 CHECK(n_tokens == 2, EINVAL);
2317 h = header_parse(p, tokens[1]);
2320 instr->type = INSTR_HDR_INVALIDATE;
2321 instr->valid.header_id = h->id;
2326 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2328 struct thread *t = &p->threads[p->thread_id];
2329 struct instruction *ip = t->ip;
2331 __instr_hdr_invalidate_exec(p, t, ip);
2340 static struct table *
2341 table_find(struct rte_swx_pipeline *p, const char *name);
2343 static struct selector *
2344 selector_find(struct rte_swx_pipeline *p, const char *name);
2346 static struct learner *
2347 learner_find(struct rte_swx_pipeline *p, const char *name);
2350 instr_table_translate(struct rte_swx_pipeline *p,
2351 struct action *action,
2354 struct instruction *instr,
2355 struct instruction_data *data __rte_unused)
2361 CHECK(!action, EINVAL);
2362 CHECK(n_tokens == 2, EINVAL);
2364 t = table_find(p, tokens[1]);
2366 instr->type = INSTR_TABLE;
2367 instr->table.table_id = t->id;
2371 s = selector_find(p, tokens[1]);
2373 instr->type = INSTR_SELECTOR;
2374 instr->table.table_id = s->id;
2378 l = learner_find(p, tokens[1]);
2380 instr->type = INSTR_LEARNER;
2381 instr->table.table_id = l->id;
2389 instr_table_exec(struct rte_swx_pipeline *p)
2391 struct thread *t = &p->threads[p->thread_id];
2392 struct instruction *ip = t->ip;
2393 uint32_t table_id = ip->table.table_id;
2394 struct rte_swx_table_state *ts = &t->table_state[table_id];
2395 struct table_runtime *table = &t->tables[table_id];
2396 struct table_statistics *stats = &p->table_stats[table_id];
2397 uint64_t action_id, n_pkts_hit, n_pkts_action;
2398 uint8_t *action_data;
2402 done = table->func(ts->obj,
2410 TRACE("[Thread %2u] table %u (not finalized)\n",
2418 action_id = hit ? action_id : ts->default_action_id;
2419 action_data = hit ? action_data : ts->default_action_data;
2420 n_pkts_hit = stats->n_pkts_hit[hit];
2421 n_pkts_action = stats->n_pkts_action[action_id];
2423 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2426 hit ? "hit" : "miss",
2427 (uint32_t)action_id);
2429 t->action_id = action_id;
2430 t->structs[0] = action_data;
2432 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2433 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2436 thread_ip_action_call(p, t, action_id);
2440 instr_table_af_exec(struct rte_swx_pipeline *p)
2442 struct thread *t = &p->threads[p->thread_id];
2443 struct instruction *ip = t->ip;
2444 uint32_t table_id = ip->table.table_id;
2445 struct rte_swx_table_state *ts = &t->table_state[table_id];
2446 struct table_runtime *table = &t->tables[table_id];
2447 struct table_statistics *stats = &p->table_stats[table_id];
2448 uint64_t action_id, n_pkts_hit, n_pkts_action;
2449 uint8_t *action_data;
2450 action_func_t action_func;
2454 done = table->func(ts->obj,
2462 TRACE("[Thread %2u] table %u (not finalized)\n",
2470 action_id = hit ? action_id : ts->default_action_id;
2471 action_data = hit ? action_data : ts->default_action_data;
2472 action_func = p->action_funcs[action_id];
2473 n_pkts_hit = stats->n_pkts_hit[hit];
2474 n_pkts_action = stats->n_pkts_action[action_id];
2476 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2479 hit ? "hit" : "miss",
2480 (uint32_t)action_id);
2482 t->action_id = action_id;
2483 t->structs[0] = action_data;
2485 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2486 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2496 instr_selector_exec(struct rte_swx_pipeline *p)
2498 struct thread *t = &p->threads[p->thread_id];
2499 struct instruction *ip = t->ip;
2500 uint32_t selector_id = ip->table.table_id;
2501 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2502 struct selector_runtime *selector = &t->selectors[selector_id];
2503 struct selector_statistics *stats = &p->selector_stats[selector_id];
2504 uint64_t n_pkts = stats->n_pkts;
2508 done = rte_swx_table_selector_select(ts->obj,
2510 selector->group_id_buffer,
2511 selector->selector_buffer,
2512 selector->member_id_buffer);
2515 TRACE("[Thread %2u] selector %u (not finalized)\n",
2524 TRACE("[Thread %2u] selector %u\n",
2528 stats->n_pkts = n_pkts + 1;
2535 instr_learner_exec(struct rte_swx_pipeline *p)
2537 struct thread *t = &p->threads[p->thread_id];
2538 struct instruction *ip = t->ip;
2539 uint32_t learner_id = ip->table.table_id;
2540 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2541 p->n_selectors + learner_id];
2542 struct learner_runtime *l = &t->learners[learner_id];
2543 struct learner_statistics *stats = &p->learner_stats[learner_id];
2544 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2545 uint8_t *action_data;
2549 time = rte_get_tsc_cycles();
2551 done = rte_swx_table_learner_lookup(ts->obj,
2560 TRACE("[Thread %2u] learner %u (not finalized)\n",
2568 action_id = hit ? action_id : ts->default_action_id;
2569 action_data = hit ? action_data : ts->default_action_data;
2570 n_pkts_hit = stats->n_pkts_hit[hit];
2571 n_pkts_action = stats->n_pkts_action[action_id];
2573 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2576 hit ? "hit" : "miss",
2577 (uint32_t)action_id);
2579 t->action_id = action_id;
2580 t->structs[0] = action_data;
2582 t->learner_id = learner_id;
2584 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2585 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2588 thread_ip_action_call(p, t, action_id);
2592 instr_learner_af_exec(struct rte_swx_pipeline *p)
2594 struct thread *t = &p->threads[p->thread_id];
2595 struct instruction *ip = t->ip;
2596 uint32_t learner_id = ip->table.table_id;
2597 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2598 p->n_selectors + learner_id];
2599 struct learner_runtime *l = &t->learners[learner_id];
2600 struct learner_statistics *stats = &p->learner_stats[learner_id];
2601 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2602 uint8_t *action_data;
2603 action_func_t action_func;
2607 time = rte_get_tsc_cycles();
2609 done = rte_swx_table_learner_lookup(ts->obj,
2618 TRACE("[Thread %2u] learner %u (not finalized)\n",
2626 action_id = hit ? action_id : ts->default_action_id;
2627 action_data = hit ? action_data : ts->default_action_data;
2628 action_func = p->action_funcs[action_id];
2629 n_pkts_hit = stats->n_pkts_hit[hit];
2630 n_pkts_action = stats->n_pkts_action[action_id];
2632 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2635 hit ? "hit" : "miss",
2636 (uint32_t)action_id);
2638 t->action_id = action_id;
2639 t->structs[0] = action_data;
2641 t->learner_id = learner_id;
2643 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2644 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2656 static struct action *
2657 action_find(struct rte_swx_pipeline *p, const char *name);
2660 action_has_nbo_args(struct action *a);
2663 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2666 instr_learn_translate(struct rte_swx_pipeline *p,
2667 struct action *action,
2670 struct instruction *instr,
2671 struct instruction_data *data __rte_unused)
2674 struct field *mf_first_arg = NULL, *mf_timeout_id = NULL;
2675 const char *mf_first_arg_name, *mf_timeout_id_name;
2677 CHECK(action, EINVAL);
2678 CHECK((n_tokens == 3) || (n_tokens == 4), EINVAL);
2681 a = action_find(p, tokens[1]);
2683 CHECK(!action_has_nbo_args(a), EINVAL);
2685 /* Action first argument. */
2686 mf_first_arg_name = (n_tokens == 4) ? tokens[2] : NULL;
2687 CHECK(!learner_action_args_check(p, a, mf_first_arg_name), EINVAL);
2689 if (mf_first_arg_name) {
2690 mf_first_arg = metadata_field_parse(p, mf_first_arg_name);
2691 CHECK(mf_first_arg, EINVAL);
2695 mf_timeout_id_name = (n_tokens == 4) ? tokens[3] : tokens[2];
2696 CHECK_NAME(mf_timeout_id_name, EINVAL);
2697 mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2698 CHECK(mf_timeout_id, EINVAL);
2701 instr->type = INSTR_LEARNER_LEARN;
2702 instr->learn.action_id = a->id;
2703 instr->learn.mf_first_arg_offset = mf_first_arg ? (mf_first_arg->offset / 8) : 0;
2704 instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2705 instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2711 instr_learn_exec(struct rte_swx_pipeline *p)
2713 struct thread *t = &p->threads[p->thread_id];
2714 struct instruction *ip = t->ip;
2716 __instr_learn_exec(p, t, ip);
2726 instr_rearm_translate(struct rte_swx_pipeline *p,
2727 struct action *action,
2730 struct instruction *instr,
2731 struct instruction_data *data __rte_unused)
2733 struct field *mf_timeout_id;
2734 const char *mf_timeout_id_name;
2736 CHECK(action, EINVAL);
2737 CHECK((n_tokens == 1) || (n_tokens == 2), EINVAL);
2739 /* INSTR_LEARNER_REARM. */
2740 if (n_tokens == 1) {
2741 instr->type = INSTR_LEARNER_REARM;
2745 /* INSTR_LEARNER_REARM_NEW. */
2746 mf_timeout_id_name = tokens[1];
2747 CHECK_NAME(mf_timeout_id_name, EINVAL);
2748 mf_timeout_id = metadata_field_parse(p, mf_timeout_id_name);
2749 CHECK(mf_timeout_id, EINVAL);
2751 instr->type = INSTR_LEARNER_REARM_NEW;
2752 instr->learn.mf_timeout_id_offset = mf_timeout_id->offset / 8;
2753 instr->learn.mf_timeout_id_n_bits = mf_timeout_id->n_bits;
2759 instr_rearm_exec(struct rte_swx_pipeline *p)
2761 struct thread *t = &p->threads[p->thread_id];
2762 struct instruction *ip = t->ip;
2764 __instr_rearm_exec(p, t, ip);
2771 instr_rearm_new_exec(struct rte_swx_pipeline *p)
2773 struct thread *t = &p->threads[p->thread_id];
2774 struct instruction *ip = t->ip;
2776 __instr_rearm_new_exec(p, t, ip);
2786 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2787 struct action *action,
2788 char **tokens __rte_unused,
2790 struct instruction *instr,
2791 struct instruction_data *data __rte_unused)
2793 CHECK(action, EINVAL);
2794 CHECK(n_tokens == 1, EINVAL);
2796 instr->type = INSTR_LEARNER_FORGET;
2802 instr_forget_exec(struct rte_swx_pipeline *p)
2804 struct thread *t = &p->threads[p->thread_id];
2805 struct instruction *ip = t->ip;
2807 __instr_forget_exec(p, t, ip);
2817 instr_extern_translate(struct rte_swx_pipeline *p,
2818 struct action *action __rte_unused,
2821 struct instruction *instr,
2822 struct instruction_data *data __rte_unused)
2824 char *token = tokens[1];
2826 CHECK(n_tokens == 2, EINVAL);
2828 if (token[0] == 'e') {
2829 struct extern_obj *obj;
2830 struct extern_type_member_func *func;
2832 func = extern_obj_member_func_parse(p, token, &obj);
2833 CHECK(func, EINVAL);
2835 instr->type = INSTR_EXTERN_OBJ;
2836 instr->ext_obj.ext_obj_id = obj->id;
2837 instr->ext_obj.func_id = func->id;
2842 if (token[0] == 'f') {
2843 struct extern_func *func;
2845 func = extern_func_parse(p, token);
2846 CHECK(func, EINVAL);
2848 instr->type = INSTR_EXTERN_FUNC;
2849 instr->ext_func.ext_func_id = func->id;
2858 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2860 struct thread *t = &p->threads[p->thread_id];
2861 struct instruction *ip = t->ip;
2864 /* Extern object member function execute. */
2865 done = __instr_extern_obj_exec(p, t, ip);
2868 thread_ip_inc_cond(t, done);
2869 thread_yield_cond(p, done ^ 1);
2873 instr_extern_func_exec(struct rte_swx_pipeline *p)
2875 struct thread *t = &p->threads[p->thread_id];
2876 struct instruction *ip = t->ip;
2879 /* Extern function execute. */
2880 done = __instr_extern_func_exec(p, t, ip);
2883 thread_ip_inc_cond(t, done);
2884 thread_yield_cond(p, done ^ 1);
2891 instr_hash_translate(struct rte_swx_pipeline *p,
2892 struct action *action,
2895 struct instruction *instr,
2896 struct instruction_data *data __rte_unused)
2898 struct hash_func *func;
2899 struct field *dst, *src_first, *src_last;
2900 uint32_t src_struct_id_first = 0, src_struct_id_last = 0;
2902 CHECK(n_tokens == 5, EINVAL);
2904 func = hash_func_find(p, tokens[1]);
2905 CHECK(func, EINVAL);
2907 dst = metadata_field_parse(p, tokens[2]);
2910 src_first = struct_field_parse(p, action, tokens[3], &src_struct_id_first);
2911 CHECK(src_first, EINVAL);
2913 src_last = struct_field_parse(p, action, tokens[4], &src_struct_id_last);
2914 CHECK(src_last, EINVAL);
2915 CHECK(src_struct_id_first == src_struct_id_last, EINVAL);
2917 instr->type = INSTR_HASH_FUNC;
2918 instr->hash_func.hash_func_id = (uint8_t)func->id;
2919 instr->hash_func.dst.offset = (uint8_t)dst->offset / 8;
2920 instr->hash_func.dst.n_bits = (uint8_t)dst->n_bits;
2921 instr->hash_func.src.struct_id = (uint8_t)src_struct_id_first;
2922 instr->hash_func.src.offset = (uint16_t)src_first->offset / 8;
2923 instr->hash_func.src.n_bytes = (uint16_t)((src_last->offset + src_last->n_bits -
2924 src_first->offset) / 8);
2930 instr_hash_func_exec(struct rte_swx_pipeline *p)
2932 struct thread *t = &p->threads[p->thread_id];
2933 struct instruction *ip = t->ip;
2935 /* Extern function execute. */
2936 __instr_hash_func_exec(p, t, ip);
2946 instr_mov_translate(struct rte_swx_pipeline *p,
2947 struct action *action,
2950 struct instruction *instr,
2951 struct instruction_data *data __rte_unused)
2953 char *dst = tokens[1], *src = tokens[2];
2954 struct field *fdst, *fsrc;
2956 uint32_t dst_struct_id = 0, src_struct_id = 0;
2958 CHECK(n_tokens == 3, EINVAL);
2960 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2961 CHECK(fdst, EINVAL);
2962 CHECK(!fdst->var_size, EINVAL);
2964 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2965 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2967 CHECK(!fsrc->var_size, EINVAL);
2969 instr->type = INSTR_MOV;
2970 if (dst[0] != 'h' && src[0] == 'h')
2971 instr->type = INSTR_MOV_MH;
2972 if (dst[0] == 'h' && src[0] != 'h')
2973 instr->type = INSTR_MOV_HM;
2974 if (dst[0] == 'h' && src[0] == 'h')
2975 instr->type = INSTR_MOV_HH;
2977 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2978 instr->mov.dst.n_bits = fdst->n_bits;
2979 instr->mov.dst.offset = fdst->offset / 8;
2980 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2981 instr->mov.src.n_bits = fsrc->n_bits;
2982 instr->mov.src.offset = fsrc->offset / 8;
2987 src_val = strtoull(src, &src, 0);
2988 CHECK(!src[0], EINVAL);
2991 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2993 instr->type = INSTR_MOV_I;
2994 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2995 instr->mov.dst.n_bits = fdst->n_bits;
2996 instr->mov.dst.offset = fdst->offset / 8;
2997 instr->mov.src_val = src_val;
3002 instr_mov_exec(struct rte_swx_pipeline *p)
3004 struct thread *t = &p->threads[p->thread_id];
3005 struct instruction *ip = t->ip;
3007 __instr_mov_exec(p, t, ip);
3014 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3016 struct thread *t = &p->threads[p->thread_id];
3017 struct instruction *ip = t->ip;
3019 __instr_mov_mh_exec(p, t, ip);
3026 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3028 struct thread *t = &p->threads[p->thread_id];
3029 struct instruction *ip = t->ip;
3031 __instr_mov_hm_exec(p, t, ip);
3038 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3040 struct thread *t = &p->threads[p->thread_id];
3041 struct instruction *ip = t->ip;
3043 __instr_mov_hh_exec(p, t, ip);
3050 instr_mov_i_exec(struct rte_swx_pipeline *p)
3052 struct thread *t = &p->threads[p->thread_id];
3053 struct instruction *ip = t->ip;
3055 __instr_mov_i_exec(p, t, ip);
3065 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3067 struct thread *t = &p->threads[p->thread_id];
3068 struct instruction *ip = t->ip;
3070 __instr_dma_ht_exec(p, t, ip);
3077 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3079 struct thread *t = &p->threads[p->thread_id];
3080 struct instruction *ip = t->ip;
3082 __instr_dma_ht2_exec(p, t, ip);
3089 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3091 struct thread *t = &p->threads[p->thread_id];
3092 struct instruction *ip = t->ip;
3094 __instr_dma_ht3_exec(p, t, ip);
3101 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3103 struct thread *t = &p->threads[p->thread_id];
3104 struct instruction *ip = t->ip;
3106 __instr_dma_ht4_exec(p, t, ip);
3113 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3115 struct thread *t = &p->threads[p->thread_id];
3116 struct instruction *ip = t->ip;
3118 __instr_dma_ht5_exec(p, t, ip);
3125 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3127 struct thread *t = &p->threads[p->thread_id];
3128 struct instruction *ip = t->ip;
3130 __instr_dma_ht6_exec(p, t, ip);
3137 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3139 struct thread *t = &p->threads[p->thread_id];
3140 struct instruction *ip = t->ip;
3142 __instr_dma_ht7_exec(p, t, ip);
3149 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3151 struct thread *t = &p->threads[p->thread_id];
3152 struct instruction *ip = t->ip;
3154 __instr_dma_ht8_exec(p, t, ip);
3164 instr_alu_add_translate(struct rte_swx_pipeline *p,
3165 struct action *action,
3168 struct instruction *instr,
3169 struct instruction_data *data __rte_unused)
3171 char *dst = tokens[1], *src = tokens[2];
3172 struct field *fdst, *fsrc;
3174 uint32_t dst_struct_id = 0, src_struct_id = 0;
3176 CHECK(n_tokens == 3, EINVAL);
3178 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3179 CHECK(fdst, EINVAL);
3180 CHECK(!fdst->var_size, EINVAL);
3182 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3183 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3185 CHECK(!fsrc->var_size, EINVAL);
3187 instr->type = INSTR_ALU_ADD;
3188 if (dst[0] == 'h' && src[0] != 'h')
3189 instr->type = INSTR_ALU_ADD_HM;
3190 if (dst[0] != 'h' && src[0] == 'h')
3191 instr->type = INSTR_ALU_ADD_MH;
3192 if (dst[0] == 'h' && src[0] == 'h')
3193 instr->type = INSTR_ALU_ADD_HH;
3195 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3196 instr->alu.dst.n_bits = fdst->n_bits;
3197 instr->alu.dst.offset = fdst->offset / 8;
3198 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3199 instr->alu.src.n_bits = fsrc->n_bits;
3200 instr->alu.src.offset = fsrc->offset / 8;
3204 /* ADD_MI, ADD_HI. */
3205 src_val = strtoull(src, &src, 0);
3206 CHECK(!src[0], EINVAL);
3208 instr->type = INSTR_ALU_ADD_MI;
3210 instr->type = INSTR_ALU_ADD_HI;
3212 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3213 instr->alu.dst.n_bits = fdst->n_bits;
3214 instr->alu.dst.offset = fdst->offset / 8;
3215 instr->alu.src_val = src_val;
3220 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3221 struct action *action,
3224 struct instruction *instr,
3225 struct instruction_data *data __rte_unused)
3227 char *dst = tokens[1], *src = tokens[2];
3228 struct field *fdst, *fsrc;
3230 uint32_t dst_struct_id = 0, src_struct_id = 0;
3232 CHECK(n_tokens == 3, EINVAL);
3234 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3235 CHECK(fdst, EINVAL);
3236 CHECK(!fdst->var_size, EINVAL);
3238 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3239 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3241 CHECK(!fsrc->var_size, EINVAL);
3243 instr->type = INSTR_ALU_SUB;
3244 if (dst[0] == 'h' && src[0] != 'h')
3245 instr->type = INSTR_ALU_SUB_HM;
3246 if (dst[0] != 'h' && src[0] == 'h')
3247 instr->type = INSTR_ALU_SUB_MH;
3248 if (dst[0] == 'h' && src[0] == 'h')
3249 instr->type = INSTR_ALU_SUB_HH;
3251 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3252 instr->alu.dst.n_bits = fdst->n_bits;
3253 instr->alu.dst.offset = fdst->offset / 8;
3254 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3255 instr->alu.src.n_bits = fsrc->n_bits;
3256 instr->alu.src.offset = fsrc->offset / 8;
3260 /* SUB_MI, SUB_HI. */
3261 src_val = strtoull(src, &src, 0);
3262 CHECK(!src[0], EINVAL);
3264 instr->type = INSTR_ALU_SUB_MI;
3266 instr->type = INSTR_ALU_SUB_HI;
3268 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3269 instr->alu.dst.n_bits = fdst->n_bits;
3270 instr->alu.dst.offset = fdst->offset / 8;
3271 instr->alu.src_val = src_val;
3276 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3277 struct action *action __rte_unused,
3280 struct instruction *instr,
3281 struct instruction_data *data __rte_unused)
3283 char *dst = tokens[1], *src = tokens[2];
3284 struct header *hdst, *hsrc;
3285 struct field *fdst, *fsrc;
3287 CHECK(n_tokens == 3, EINVAL);
3289 fdst = header_field_parse(p, dst, &hdst);
3290 CHECK(fdst, EINVAL);
3291 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3294 fsrc = header_field_parse(p, src, &hsrc);
3296 CHECK(!fsrc->var_size, EINVAL);
3298 instr->type = INSTR_ALU_CKADD_FIELD;
3299 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3300 instr->alu.dst.n_bits = fdst->n_bits;
3301 instr->alu.dst.offset = fdst->offset / 8;
3302 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3303 instr->alu.src.n_bits = fsrc->n_bits;
3304 instr->alu.src.offset = fsrc->offset / 8;
3308 /* CKADD_STRUCT, CKADD_STRUCT20. */
3309 hsrc = header_parse(p, src);
3310 CHECK(hsrc, EINVAL);
3312 instr->type = INSTR_ALU_CKADD_STRUCT;
3313 if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3314 instr->type = INSTR_ALU_CKADD_STRUCT20;
3316 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3317 instr->alu.dst.n_bits = fdst->n_bits;
3318 instr->alu.dst.offset = fdst->offset / 8;
3319 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3320 instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3321 instr->alu.src.offset = 0; /* Unused. */
3326 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3327 struct action *action __rte_unused,
3330 struct instruction *instr,
3331 struct instruction_data *data __rte_unused)
3333 char *dst = tokens[1], *src = tokens[2];
3334 struct header *hdst, *hsrc;
3335 struct field *fdst, *fsrc;
3337 CHECK(n_tokens == 3, EINVAL);
3339 fdst = header_field_parse(p, dst, &hdst);
3340 CHECK(fdst, EINVAL);
3341 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3343 fsrc = header_field_parse(p, src, &hsrc);
3344 CHECK(fsrc, EINVAL);
3345 CHECK(!fsrc->var_size, EINVAL);
3347 instr->type = INSTR_ALU_CKSUB_FIELD;
3348 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3349 instr->alu.dst.n_bits = fdst->n_bits;
3350 instr->alu.dst.offset = fdst->offset / 8;
3351 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3352 instr->alu.src.n_bits = fsrc->n_bits;
3353 instr->alu.src.offset = fsrc->offset / 8;
3358 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3359 struct action *action,
3362 struct instruction *instr,
3363 struct instruction_data *data __rte_unused)
3365 char *dst = tokens[1], *src = tokens[2];
3366 struct field *fdst, *fsrc;
3368 uint32_t dst_struct_id = 0, src_struct_id = 0;
3370 CHECK(n_tokens == 3, EINVAL);
3372 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3373 CHECK(fdst, EINVAL);
3374 CHECK(!fdst->var_size, EINVAL);
3376 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3377 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3379 CHECK(!fsrc->var_size, EINVAL);
3381 instr->type = INSTR_ALU_SHL;
3382 if (dst[0] == 'h' && src[0] != 'h')
3383 instr->type = INSTR_ALU_SHL_HM;
3384 if (dst[0] != 'h' && src[0] == 'h')
3385 instr->type = INSTR_ALU_SHL_MH;
3386 if (dst[0] == 'h' && src[0] == 'h')
3387 instr->type = INSTR_ALU_SHL_HH;
3389 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3390 instr->alu.dst.n_bits = fdst->n_bits;
3391 instr->alu.dst.offset = fdst->offset / 8;
3392 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3393 instr->alu.src.n_bits = fsrc->n_bits;
3394 instr->alu.src.offset = fsrc->offset / 8;
3398 /* SHL_MI, SHL_HI. */
3399 src_val = strtoull(src, &src, 0);
3400 CHECK(!src[0], EINVAL);
3402 instr->type = INSTR_ALU_SHL_MI;
3404 instr->type = INSTR_ALU_SHL_HI;
3406 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3407 instr->alu.dst.n_bits = fdst->n_bits;
3408 instr->alu.dst.offset = fdst->offset / 8;
3409 instr->alu.src_val = src_val;
3414 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3415 struct action *action,
3418 struct instruction *instr,
3419 struct instruction_data *data __rte_unused)
3421 char *dst = tokens[1], *src = tokens[2];
3422 struct field *fdst, *fsrc;
3424 uint32_t dst_struct_id = 0, src_struct_id = 0;
3426 CHECK(n_tokens == 3, EINVAL);
3428 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3429 CHECK(fdst, EINVAL);
3430 CHECK(!fdst->var_size, EINVAL);
3432 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3433 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3435 CHECK(!fsrc->var_size, EINVAL);
3437 instr->type = INSTR_ALU_SHR;
3438 if (dst[0] == 'h' && src[0] != 'h')
3439 instr->type = INSTR_ALU_SHR_HM;
3440 if (dst[0] != 'h' && src[0] == 'h')
3441 instr->type = INSTR_ALU_SHR_MH;
3442 if (dst[0] == 'h' && src[0] == 'h')
3443 instr->type = INSTR_ALU_SHR_HH;
3445 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3446 instr->alu.dst.n_bits = fdst->n_bits;
3447 instr->alu.dst.offset = fdst->offset / 8;
3448 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3449 instr->alu.src.n_bits = fsrc->n_bits;
3450 instr->alu.src.offset = fsrc->offset / 8;
3454 /* SHR_MI, SHR_HI. */
3455 src_val = strtoull(src, &src, 0);
3456 CHECK(!src[0], EINVAL);
3458 instr->type = INSTR_ALU_SHR_MI;
3460 instr->type = INSTR_ALU_SHR_HI;
3462 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3463 instr->alu.dst.n_bits = fdst->n_bits;
3464 instr->alu.dst.offset = fdst->offset / 8;
3465 instr->alu.src_val = src_val;
3470 instr_alu_and_translate(struct rte_swx_pipeline *p,
3471 struct action *action,
3474 struct instruction *instr,
3475 struct instruction_data *data __rte_unused)
3477 char *dst = tokens[1], *src = tokens[2];
3478 struct field *fdst, *fsrc;
3480 uint32_t dst_struct_id = 0, src_struct_id = 0;
3482 CHECK(n_tokens == 3, EINVAL);
3484 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3485 CHECK(fdst, EINVAL);
3486 CHECK(!fdst->var_size, EINVAL);
3488 /* AND, AND_MH, AND_HM, AND_HH. */
3489 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3491 CHECK(!fsrc->var_size, EINVAL);
3493 instr->type = INSTR_ALU_AND;
3494 if (dst[0] != 'h' && src[0] == 'h')
3495 instr->type = INSTR_ALU_AND_MH;
3496 if (dst[0] == 'h' && src[0] != 'h')
3497 instr->type = INSTR_ALU_AND_HM;
3498 if (dst[0] == 'h' && src[0] == 'h')
3499 instr->type = INSTR_ALU_AND_HH;
3501 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3502 instr->alu.dst.n_bits = fdst->n_bits;
3503 instr->alu.dst.offset = fdst->offset / 8;
3504 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3505 instr->alu.src.n_bits = fsrc->n_bits;
3506 instr->alu.src.offset = fsrc->offset / 8;
3511 src_val = strtoull(src, &src, 0);
3512 CHECK(!src[0], EINVAL);
3515 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3517 instr->type = INSTR_ALU_AND_I;
3518 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3519 instr->alu.dst.n_bits = fdst->n_bits;
3520 instr->alu.dst.offset = fdst->offset / 8;
3521 instr->alu.src_val = src_val;
3526 instr_alu_or_translate(struct rte_swx_pipeline *p,
3527 struct action *action,
3530 struct instruction *instr,
3531 struct instruction_data *data __rte_unused)
3533 char *dst = tokens[1], *src = tokens[2];
3534 struct field *fdst, *fsrc;
3536 uint32_t dst_struct_id = 0, src_struct_id = 0;
3538 CHECK(n_tokens == 3, EINVAL);
3540 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3541 CHECK(fdst, EINVAL);
3542 CHECK(!fdst->var_size, EINVAL);
3544 /* OR, OR_MH, OR_HM, OR_HH. */
3545 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3547 CHECK(!fsrc->var_size, EINVAL);
3549 instr->type = INSTR_ALU_OR;
3550 if (dst[0] != 'h' && src[0] == 'h')
3551 instr->type = INSTR_ALU_OR_MH;
3552 if (dst[0] == 'h' && src[0] != 'h')
3553 instr->type = INSTR_ALU_OR_HM;
3554 if (dst[0] == 'h' && src[0] == 'h')
3555 instr->type = INSTR_ALU_OR_HH;
3557 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3558 instr->alu.dst.n_bits = fdst->n_bits;
3559 instr->alu.dst.offset = fdst->offset / 8;
3560 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3561 instr->alu.src.n_bits = fsrc->n_bits;
3562 instr->alu.src.offset = fsrc->offset / 8;
3567 src_val = strtoull(src, &src, 0);
3568 CHECK(!src[0], EINVAL);
3571 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3573 instr->type = INSTR_ALU_OR_I;
3574 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3575 instr->alu.dst.n_bits = fdst->n_bits;
3576 instr->alu.dst.offset = fdst->offset / 8;
3577 instr->alu.src_val = src_val;
3582 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3583 struct action *action,
3586 struct instruction *instr,
3587 struct instruction_data *data __rte_unused)
3589 char *dst = tokens[1], *src = tokens[2];
3590 struct field *fdst, *fsrc;
3592 uint32_t dst_struct_id = 0, src_struct_id = 0;
3594 CHECK(n_tokens == 3, EINVAL);
3596 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3597 CHECK(fdst, EINVAL);
3598 CHECK(!fdst->var_size, EINVAL);
3600 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3601 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3603 CHECK(!fsrc->var_size, EINVAL);
3605 instr->type = INSTR_ALU_XOR;
3606 if (dst[0] != 'h' && src[0] == 'h')
3607 instr->type = INSTR_ALU_XOR_MH;
3608 if (dst[0] == 'h' && src[0] != 'h')
3609 instr->type = INSTR_ALU_XOR_HM;
3610 if (dst[0] == 'h' && src[0] == 'h')
3611 instr->type = INSTR_ALU_XOR_HH;
3613 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3614 instr->alu.dst.n_bits = fdst->n_bits;
3615 instr->alu.dst.offset = fdst->offset / 8;
3616 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3617 instr->alu.src.n_bits = fsrc->n_bits;
3618 instr->alu.src.offset = fsrc->offset / 8;
3623 src_val = strtoull(src, &src, 0);
3624 CHECK(!src[0], EINVAL);
3627 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3629 instr->type = INSTR_ALU_XOR_I;
3630 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3631 instr->alu.dst.n_bits = fdst->n_bits;
3632 instr->alu.dst.offset = fdst->offset / 8;
3633 instr->alu.src_val = src_val;
3638 instr_alu_add_exec(struct rte_swx_pipeline *p)
3640 struct thread *t = &p->threads[p->thread_id];
3641 struct instruction *ip = t->ip;
3644 __instr_alu_add_exec(p, t, ip);
3651 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3653 struct thread *t = &p->threads[p->thread_id];
3654 struct instruction *ip = t->ip;
3657 __instr_alu_add_mh_exec(p, t, ip);
3664 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3666 struct thread *t = &p->threads[p->thread_id];
3667 struct instruction *ip = t->ip;
3670 __instr_alu_add_hm_exec(p, t, ip);
3677 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3679 struct thread *t = &p->threads[p->thread_id];
3680 struct instruction *ip = t->ip;
3683 __instr_alu_add_hh_exec(p, t, ip);
3690 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3692 struct thread *t = &p->threads[p->thread_id];
3693 struct instruction *ip = t->ip;
3696 __instr_alu_add_mi_exec(p, t, ip);
3703 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3705 struct thread *t = &p->threads[p->thread_id];
3706 struct instruction *ip = t->ip;
3709 __instr_alu_add_hi_exec(p, t, ip);
3716 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3718 struct thread *t = &p->threads[p->thread_id];
3719 struct instruction *ip = t->ip;
3722 __instr_alu_sub_exec(p, t, ip);
3729 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3731 struct thread *t = &p->threads[p->thread_id];
3732 struct instruction *ip = t->ip;
3735 __instr_alu_sub_mh_exec(p, t, ip);
3742 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3744 struct thread *t = &p->threads[p->thread_id];
3745 struct instruction *ip = t->ip;
3748 __instr_alu_sub_hm_exec(p, t, ip);
3755 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3757 struct thread *t = &p->threads[p->thread_id];
3758 struct instruction *ip = t->ip;
3761 __instr_alu_sub_hh_exec(p, t, ip);
3768 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3770 struct thread *t = &p->threads[p->thread_id];
3771 struct instruction *ip = t->ip;
3774 __instr_alu_sub_mi_exec(p, t, ip);
3781 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3783 struct thread *t = &p->threads[p->thread_id];
3784 struct instruction *ip = t->ip;
3787 __instr_alu_sub_hi_exec(p, t, ip);
3794 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3796 struct thread *t = &p->threads[p->thread_id];
3797 struct instruction *ip = t->ip;
3800 __instr_alu_shl_exec(p, t, ip);
3807 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3809 struct thread *t = &p->threads[p->thread_id];
3810 struct instruction *ip = t->ip;
3813 __instr_alu_shl_mh_exec(p, t, ip);
3820 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3822 struct thread *t = &p->threads[p->thread_id];
3823 struct instruction *ip = t->ip;
3826 __instr_alu_shl_hm_exec(p, t, ip);
3833 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3835 struct thread *t = &p->threads[p->thread_id];
3836 struct instruction *ip = t->ip;
3839 __instr_alu_shl_hh_exec(p, t, ip);
3846 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3848 struct thread *t = &p->threads[p->thread_id];
3849 struct instruction *ip = t->ip;
3852 __instr_alu_shl_mi_exec(p, t, ip);
3859 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3861 struct thread *t = &p->threads[p->thread_id];
3862 struct instruction *ip = t->ip;
3865 __instr_alu_shl_hi_exec(p, t, ip);
3872 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3874 struct thread *t = &p->threads[p->thread_id];
3875 struct instruction *ip = t->ip;
3878 __instr_alu_shr_exec(p, t, ip);
3885 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3887 struct thread *t = &p->threads[p->thread_id];
3888 struct instruction *ip = t->ip;
3891 __instr_alu_shr_mh_exec(p, t, ip);
3898 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3900 struct thread *t = &p->threads[p->thread_id];
3901 struct instruction *ip = t->ip;
3904 __instr_alu_shr_hm_exec(p, t, ip);
3911 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3913 struct thread *t = &p->threads[p->thread_id];
3914 struct instruction *ip = t->ip;
3917 __instr_alu_shr_hh_exec(p, t, ip);
3924 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3926 struct thread *t = &p->threads[p->thread_id];
3927 struct instruction *ip = t->ip;
3930 __instr_alu_shr_mi_exec(p, t, ip);
3937 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3939 struct thread *t = &p->threads[p->thread_id];
3940 struct instruction *ip = t->ip;
3943 __instr_alu_shr_hi_exec(p, t, ip);
3950 instr_alu_and_exec(struct rte_swx_pipeline *p)
3952 struct thread *t = &p->threads[p->thread_id];
3953 struct instruction *ip = t->ip;
3956 __instr_alu_and_exec(p, t, ip);
3963 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3965 struct thread *t = &p->threads[p->thread_id];
3966 struct instruction *ip = t->ip;
3969 __instr_alu_and_mh_exec(p, t, ip);
3976 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3978 struct thread *t = &p->threads[p->thread_id];
3979 struct instruction *ip = t->ip;
3982 __instr_alu_and_hm_exec(p, t, ip);
3989 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3991 struct thread *t = &p->threads[p->thread_id];
3992 struct instruction *ip = t->ip;
3995 __instr_alu_and_hh_exec(p, t, ip);
4002 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4004 struct thread *t = &p->threads[p->thread_id];
4005 struct instruction *ip = t->ip;
4008 __instr_alu_and_i_exec(p, t, ip);
4015 instr_alu_or_exec(struct rte_swx_pipeline *p)
4017 struct thread *t = &p->threads[p->thread_id];
4018 struct instruction *ip = t->ip;
4021 __instr_alu_or_exec(p, t, ip);
4028 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4030 struct thread *t = &p->threads[p->thread_id];
4031 struct instruction *ip = t->ip;
4034 __instr_alu_or_mh_exec(p, t, ip);
4041 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4043 struct thread *t = &p->threads[p->thread_id];
4044 struct instruction *ip = t->ip;
4047 __instr_alu_or_hm_exec(p, t, ip);
4054 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4056 struct thread *t = &p->threads[p->thread_id];
4057 struct instruction *ip = t->ip;
4060 __instr_alu_or_hh_exec(p, t, ip);
4067 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4069 struct thread *t = &p->threads[p->thread_id];
4070 struct instruction *ip = t->ip;
4073 __instr_alu_or_i_exec(p, t, ip);
4080 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4082 struct thread *t = &p->threads[p->thread_id];
4083 struct instruction *ip = t->ip;
4086 __instr_alu_xor_exec(p, t, ip);
4093 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4095 struct thread *t = &p->threads[p->thread_id];
4096 struct instruction *ip = t->ip;
4099 __instr_alu_xor_mh_exec(p, t, ip);
4106 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4108 struct thread *t = &p->threads[p->thread_id];
4109 struct instruction *ip = t->ip;
4112 __instr_alu_xor_hm_exec(p, t, ip);
4119 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4121 struct thread *t = &p->threads[p->thread_id];
4122 struct instruction *ip = t->ip;
4125 __instr_alu_xor_hh_exec(p, t, ip);
4132 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4134 struct thread *t = &p->threads[p->thread_id];
4135 struct instruction *ip = t->ip;
4138 __instr_alu_xor_i_exec(p, t, ip);
4145 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4147 struct thread *t = &p->threads[p->thread_id];
4148 struct instruction *ip = t->ip;
4151 __instr_alu_ckadd_field_exec(p, t, ip);
4158 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4160 struct thread *t = &p->threads[p->thread_id];
4161 struct instruction *ip = t->ip;
4164 __instr_alu_cksub_field_exec(p, t, ip);
4171 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4173 struct thread *t = &p->threads[p->thread_id];
4174 struct instruction *ip = t->ip;
4177 __instr_alu_ckadd_struct20_exec(p, t, ip);
4184 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4186 struct thread *t = &p->threads[p->thread_id];
4187 struct instruction *ip = t->ip;
4190 __instr_alu_ckadd_struct_exec(p, t, ip);
4199 static struct regarray *
4200 regarray_find(struct rte_swx_pipeline *p, const char *name);
4203 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4204 struct action *action,
4207 struct instruction *instr,
4208 struct instruction_data *data __rte_unused)
4210 char *regarray = tokens[1], *idx = tokens[2];
4213 uint32_t idx_struct_id, idx_val;
4215 CHECK(n_tokens == 3, EINVAL);
4217 r = regarray_find(p, regarray);
4220 /* REGPREFETCH_RH, REGPREFETCH_RM. */
4221 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4223 CHECK(!fidx->var_size, EINVAL);
4225 instr->type = INSTR_REGPREFETCH_RM;
4227 instr->type = INSTR_REGPREFETCH_RH;
4229 instr->regarray.regarray_id = r->id;
4230 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4231 instr->regarray.idx.n_bits = fidx->n_bits;
4232 instr->regarray.idx.offset = fidx->offset / 8;
4233 instr->regarray.dstsrc_val = 0; /* Unused. */
4237 /* REGPREFETCH_RI. */
4238 idx_val = strtoul(idx, &idx, 0);
4239 CHECK(!idx[0], EINVAL);
4241 instr->type = INSTR_REGPREFETCH_RI;
4242 instr->regarray.regarray_id = r->id;
4243 instr->regarray.idx_val = idx_val;
4244 instr->regarray.dstsrc_val = 0; /* Unused. */
4249 instr_regrd_translate(struct rte_swx_pipeline *p,
4250 struct action *action,
4253 struct instruction *instr,
4254 struct instruction_data *data __rte_unused)
4256 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4258 struct field *fdst, *fidx;
4259 uint32_t dst_struct_id, idx_struct_id, idx_val;
4261 CHECK(n_tokens == 4, EINVAL);
4263 r = regarray_find(p, regarray);
4266 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4267 CHECK(fdst, EINVAL);
4268 CHECK(!fdst->var_size, EINVAL);
4270 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4271 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4273 CHECK(!fidx->var_size, EINVAL);
4275 instr->type = INSTR_REGRD_MRM;
4276 if (dst[0] == 'h' && idx[0] != 'h')
4277 instr->type = INSTR_REGRD_HRM;
4278 if (dst[0] != 'h' && idx[0] == 'h')
4279 instr->type = INSTR_REGRD_MRH;
4280 if (dst[0] == 'h' && idx[0] == 'h')
4281 instr->type = INSTR_REGRD_HRH;
4283 instr->regarray.regarray_id = r->id;
4284 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4285 instr->regarray.idx.n_bits = fidx->n_bits;
4286 instr->regarray.idx.offset = fidx->offset / 8;
4287 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4288 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4289 instr->regarray.dstsrc.offset = fdst->offset / 8;
4293 /* REGRD_MRI, REGRD_HRI. */
4294 idx_val = strtoul(idx, &idx, 0);
4295 CHECK(!idx[0], EINVAL);
4297 instr->type = INSTR_REGRD_MRI;
4299 instr->type = INSTR_REGRD_HRI;
4301 instr->regarray.regarray_id = r->id;
4302 instr->regarray.idx_val = idx_val;
4303 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4304 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4305 instr->regarray.dstsrc.offset = fdst->offset / 8;
4310 instr_regwr_translate(struct rte_swx_pipeline *p,
4311 struct action *action,
4314 struct instruction *instr,
4315 struct instruction_data *data __rte_unused)
4317 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4319 struct field *fidx, *fsrc;
4321 uint32_t idx_struct_id, idx_val, src_struct_id;
4323 CHECK(n_tokens == 4, EINVAL);
4325 r = regarray_find(p, regarray);
4328 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4329 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4330 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4332 CHECK(!fidx->var_size, EINVAL);
4333 CHECK(!fsrc->var_size, EINVAL);
4335 instr->type = INSTR_REGWR_RMM;
4336 if (idx[0] == 'h' && src[0] != 'h')
4337 instr->type = INSTR_REGWR_RHM;
4338 if (idx[0] != 'h' && src[0] == 'h')
4339 instr->type = INSTR_REGWR_RMH;
4340 if (idx[0] == 'h' && src[0] == 'h')
4341 instr->type = INSTR_REGWR_RHH;
4343 instr->regarray.regarray_id = r->id;
4344 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4345 instr->regarray.idx.n_bits = fidx->n_bits;
4346 instr->regarray.idx.offset = fidx->offset / 8;
4347 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4348 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4349 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4353 /* REGWR_RHI, REGWR_RMI. */
4354 if (fidx && !fsrc) {
4355 CHECK(!fidx->var_size, EINVAL);
4357 src_val = strtoull(src, &src, 0);
4358 CHECK(!src[0], EINVAL);
4360 instr->type = INSTR_REGWR_RMI;
4362 instr->type = INSTR_REGWR_RHI;
4364 instr->regarray.regarray_id = r->id;
4365 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4366 instr->regarray.idx.n_bits = fidx->n_bits;
4367 instr->regarray.idx.offset = fidx->offset / 8;
4368 instr->regarray.dstsrc_val = src_val;
4372 /* REGWR_RIH, REGWR_RIM. */
4373 if (!fidx && fsrc) {
4374 idx_val = strtoul(idx, &idx, 0);
4375 CHECK(!idx[0], EINVAL);
4377 CHECK(!fsrc->var_size, EINVAL);
4379 instr->type = INSTR_REGWR_RIM;
4381 instr->type = INSTR_REGWR_RIH;
4383 instr->regarray.regarray_id = r->id;
4384 instr->regarray.idx_val = idx_val;
4385 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4386 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4387 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4392 src_val = strtoull(src, &src, 0);
4393 CHECK(!src[0], EINVAL);
4395 idx_val = strtoul(idx, &idx, 0);
4396 CHECK(!idx[0], EINVAL);
4398 instr->type = INSTR_REGWR_RII;
4399 instr->regarray.idx_val = idx_val;
4400 instr->regarray.dstsrc_val = src_val;
4406 instr_regadd_translate(struct rte_swx_pipeline *p,
4407 struct action *action,
4410 struct instruction *instr,
4411 struct instruction_data *data __rte_unused)
4413 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4415 struct field *fidx, *fsrc;
4417 uint32_t idx_struct_id, idx_val, src_struct_id;
4419 CHECK(n_tokens == 4, EINVAL);
4421 r = regarray_find(p, regarray);
4424 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4425 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4426 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4428 CHECK(!fidx->var_size, EINVAL);
4429 CHECK(!fsrc->var_size, EINVAL);
4431 instr->type = INSTR_REGADD_RMM;
4432 if (idx[0] == 'h' && src[0] != 'h')
4433 instr->type = INSTR_REGADD_RHM;
4434 if (idx[0] != 'h' && src[0] == 'h')
4435 instr->type = INSTR_REGADD_RMH;
4436 if (idx[0] == 'h' && src[0] == 'h')
4437 instr->type = INSTR_REGADD_RHH;
4439 instr->regarray.regarray_id = r->id;
4440 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4441 instr->regarray.idx.n_bits = fidx->n_bits;
4442 instr->regarray.idx.offset = fidx->offset / 8;
4443 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4444 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4445 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4449 /* REGADD_RHI, REGADD_RMI. */
4450 if (fidx && !fsrc) {
4451 CHECK(!fidx->var_size, EINVAL);
4453 src_val = strtoull(src, &src, 0);
4454 CHECK(!src[0], EINVAL);
4456 instr->type = INSTR_REGADD_RMI;
4458 instr->type = INSTR_REGADD_RHI;
4460 instr->regarray.regarray_id = r->id;
4461 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4462 instr->regarray.idx.n_bits = fidx->n_bits;
4463 instr->regarray.idx.offset = fidx->offset / 8;
4464 instr->regarray.dstsrc_val = src_val;
4468 /* REGADD_RIH, REGADD_RIM. */
4469 if (!fidx && fsrc) {
4470 idx_val = strtoul(idx, &idx, 0);
4471 CHECK(!idx[0], EINVAL);
4473 CHECK(!fsrc->var_size, EINVAL);
4475 instr->type = INSTR_REGADD_RIM;
4477 instr->type = INSTR_REGADD_RIH;
4479 instr->regarray.regarray_id = r->id;
4480 instr->regarray.idx_val = idx_val;
4481 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4482 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4483 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4488 src_val = strtoull(src, &src, 0);
4489 CHECK(!src[0], EINVAL);
4491 idx_val = strtoul(idx, &idx, 0);
4492 CHECK(!idx[0], EINVAL);
4494 instr->type = INSTR_REGADD_RII;
4495 instr->regarray.idx_val = idx_val;
4496 instr->regarray.dstsrc_val = src_val;
4501 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4503 struct thread *t = &p->threads[p->thread_id];
4504 struct instruction *ip = t->ip;
4507 __instr_regprefetch_rh_exec(p, t, ip);
4514 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4516 struct thread *t = &p->threads[p->thread_id];
4517 struct instruction *ip = t->ip;
4520 __instr_regprefetch_rm_exec(p, t, ip);
4527 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4529 struct thread *t = &p->threads[p->thread_id];
4530 struct instruction *ip = t->ip;
4533 __instr_regprefetch_ri_exec(p, t, ip);
4540 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4542 struct thread *t = &p->threads[p->thread_id];
4543 struct instruction *ip = t->ip;
4546 __instr_regrd_hrh_exec(p, t, ip);
4553 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4555 struct thread *t = &p->threads[p->thread_id];
4556 struct instruction *ip = t->ip;
4559 __instr_regrd_hrm_exec(p, t, ip);
4566 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4568 struct thread *t = &p->threads[p->thread_id];
4569 struct instruction *ip = t->ip;
4572 __instr_regrd_mrh_exec(p, t, ip);
4579 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4581 struct thread *t = &p->threads[p->thread_id];
4582 struct instruction *ip = t->ip;
4585 __instr_regrd_mrm_exec(p, t, ip);
4592 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4594 struct thread *t = &p->threads[p->thread_id];
4595 struct instruction *ip = t->ip;
4598 __instr_regrd_hri_exec(p, t, ip);
4605 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4607 struct thread *t = &p->threads[p->thread_id];
4608 struct instruction *ip = t->ip;
4611 __instr_regrd_mri_exec(p, t, ip);
4618 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4620 struct thread *t = &p->threads[p->thread_id];
4621 struct instruction *ip = t->ip;
4624 __instr_regwr_rhh_exec(p, t, ip);
4631 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4633 struct thread *t = &p->threads[p->thread_id];
4634 struct instruction *ip = t->ip;
4637 __instr_regwr_rhm_exec(p, t, ip);
4644 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4646 struct thread *t = &p->threads[p->thread_id];
4647 struct instruction *ip = t->ip;
4650 __instr_regwr_rmh_exec(p, t, ip);
4657 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4659 struct thread *t = &p->threads[p->thread_id];
4660 struct instruction *ip = t->ip;
4663 __instr_regwr_rmm_exec(p, t, ip);
4670 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4672 struct thread *t = &p->threads[p->thread_id];
4673 struct instruction *ip = t->ip;
4676 __instr_regwr_rhi_exec(p, t, ip);
4683 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4685 struct thread *t = &p->threads[p->thread_id];
4686 struct instruction *ip = t->ip;
4689 __instr_regwr_rmi_exec(p, t, ip);
4696 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4698 struct thread *t = &p->threads[p->thread_id];
4699 struct instruction *ip = t->ip;
4702 __instr_regwr_rih_exec(p, t, ip);
4709 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4711 struct thread *t = &p->threads[p->thread_id];
4712 struct instruction *ip = t->ip;
4715 __instr_regwr_rim_exec(p, t, ip);
4722 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4724 struct thread *t = &p->threads[p->thread_id];
4725 struct instruction *ip = t->ip;
4728 __instr_regwr_rii_exec(p, t, ip);
4735 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4737 struct thread *t = &p->threads[p->thread_id];
4738 struct instruction *ip = t->ip;
4741 __instr_regadd_rhh_exec(p, t, ip);
4748 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4750 struct thread *t = &p->threads[p->thread_id];
4751 struct instruction *ip = t->ip;
4754 __instr_regadd_rhm_exec(p, t, ip);
4761 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4763 struct thread *t = &p->threads[p->thread_id];
4764 struct instruction *ip = t->ip;
4767 __instr_regadd_rmh_exec(p, t, ip);
4774 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4776 struct thread *t = &p->threads[p->thread_id];
4777 struct instruction *ip = t->ip;
4780 __instr_regadd_rmm_exec(p, t, ip);
4787 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4789 struct thread *t = &p->threads[p->thread_id];
4790 struct instruction *ip = t->ip;
4793 __instr_regadd_rhi_exec(p, t, ip);
4800 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4802 struct thread *t = &p->threads[p->thread_id];
4803 struct instruction *ip = t->ip;
4806 __instr_regadd_rmi_exec(p, t, ip);
4813 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4815 struct thread *t = &p->threads[p->thread_id];
4816 struct instruction *ip = t->ip;
4819 __instr_regadd_rih_exec(p, t, ip);
4826 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4828 struct thread *t = &p->threads[p->thread_id];
4829 struct instruction *ip = t->ip;
4832 __instr_regadd_rim_exec(p, t, ip);
4839 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4841 struct thread *t = &p->threads[p->thread_id];
4842 struct instruction *ip = t->ip;
4845 __instr_regadd_rii_exec(p, t, ip);
4854 static struct metarray *
4855 metarray_find(struct rte_swx_pipeline *p, const char *name);
4858 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4859 struct action *action,
4862 struct instruction *instr,
4863 struct instruction_data *data __rte_unused)
4865 char *metarray = tokens[1], *idx = tokens[2];
4868 uint32_t idx_struct_id, idx_val;
4870 CHECK(n_tokens == 3, EINVAL);
4872 m = metarray_find(p, metarray);
4875 /* METPREFETCH_H, METPREFETCH_M. */
4876 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4878 CHECK(!fidx->var_size, EINVAL);
4880 instr->type = INSTR_METPREFETCH_M;
4882 instr->type = INSTR_METPREFETCH_H;
4884 instr->meter.metarray_id = m->id;
4885 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4886 instr->meter.idx.n_bits = fidx->n_bits;
4887 instr->meter.idx.offset = fidx->offset / 8;
4891 /* METPREFETCH_I. */
4892 idx_val = strtoul(idx, &idx, 0);
4893 CHECK(!idx[0], EINVAL);
4895 instr->type = INSTR_METPREFETCH_I;
4896 instr->meter.metarray_id = m->id;
4897 instr->meter.idx_val = idx_val;
4902 instr_meter_translate(struct rte_swx_pipeline *p,
4903 struct action *action,
4906 struct instruction *instr,
4907 struct instruction_data *data __rte_unused)
4909 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4910 char *color_in = tokens[4], *color_out = tokens[5];
4912 struct field *fidx, *flength, *fcin, *fcout;
4913 uint32_t idx_struct_id, length_struct_id;
4914 uint32_t color_in_struct_id, color_out_struct_id;
4916 CHECK(n_tokens == 6, EINVAL);
4918 m = metarray_find(p, metarray);
4921 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4923 flength = struct_field_parse(p, action, length, &length_struct_id);
4924 CHECK(flength, EINVAL);
4925 CHECK(!flength->var_size, EINVAL);
4927 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4929 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4930 CHECK(fcout, EINVAL);
4931 CHECK(!fcout->var_size, EINVAL);
4933 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4935 CHECK(!fidx->var_size, EINVAL);
4936 CHECK(!fcin->var_size, EINVAL);
4938 instr->type = INSTR_METER_MMM;
4939 if (idx[0] == 'h' && length[0] == 'h')
4940 instr->type = INSTR_METER_HHM;
4941 if (idx[0] == 'h' && length[0] != 'h')
4942 instr->type = INSTR_METER_HMM;
4943 if (idx[0] != 'h' && length[0] == 'h')
4944 instr->type = INSTR_METER_MHM;
4946 instr->meter.metarray_id = m->id;
4948 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4949 instr->meter.idx.n_bits = fidx->n_bits;
4950 instr->meter.idx.offset = fidx->offset / 8;
4952 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4953 instr->meter.length.n_bits = flength->n_bits;
4954 instr->meter.length.offset = flength->offset / 8;
4956 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4957 instr->meter.color_in.n_bits = fcin->n_bits;
4958 instr->meter.color_in.offset = fcin->offset / 8;
4960 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4961 instr->meter.color_out.n_bits = fcout->n_bits;
4962 instr->meter.color_out.offset = fcout->offset / 8;
4965 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4966 if (fidx && !fcin) {
4967 uint32_t color_in_val;
4969 CHECK(!fidx->var_size, EINVAL);
4971 color_in_val = strtoul(color_in, &color_in, 0);
4972 CHECK(!color_in[0], EINVAL);
4974 instr->type = INSTR_METER_MMI;
4975 if (idx[0] == 'h' && length[0] == 'h')
4976 instr->type = INSTR_METER_HHI;
4977 if (idx[0] == 'h' && length[0] != 'h')
4978 instr->type = INSTR_METER_HMI;
4979 if (idx[0] != 'h' && length[0] == 'h')
4980 instr->type = INSTR_METER_MHI;
4982 instr->meter.metarray_id = m->id;
4984 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4985 instr->meter.idx.n_bits = fidx->n_bits;
4986 instr->meter.idx.offset = fidx->offset / 8;
4988 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4989 instr->meter.length.n_bits = flength->n_bits;
4990 instr->meter.length.offset = flength->offset / 8;
4992 instr->meter.color_in_val = color_in_val;
4994 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4995 instr->meter.color_out.n_bits = fcout->n_bits;
4996 instr->meter.color_out.offset = fcout->offset / 8;
4999 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5000 if (!fidx && fcin) {
5003 idx_val = strtoul(idx, &idx, 0);
5004 CHECK(!idx[0], EINVAL);
5006 CHECK(!fcin->var_size, EINVAL);
5008 instr->type = INSTR_METER_IMM;
5009 if (length[0] == 'h')
5010 instr->type = INSTR_METER_IHM;
5012 instr->meter.metarray_id = m->id;
5014 instr->meter.idx_val = idx_val;
5016 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5017 instr->meter.length.n_bits = flength->n_bits;
5018 instr->meter.length.offset = flength->offset / 8;
5020 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5021 instr->meter.color_in.n_bits = fcin->n_bits;
5022 instr->meter.color_in.offset = fcin->offset / 8;
5024 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5025 instr->meter.color_out.n_bits = fcout->n_bits;
5026 instr->meter.color_out.offset = fcout->offset / 8;
5029 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5030 if (!fidx && !fcin) {
5031 uint32_t idx_val, color_in_val;
5033 idx_val = strtoul(idx, &idx, 0);
5034 CHECK(!idx[0], EINVAL);
5036 color_in_val = strtoul(color_in, &color_in, 0);
5037 CHECK(!color_in[0], EINVAL);
5039 instr->type = INSTR_METER_IMI;
5040 if (length[0] == 'h')
5041 instr->type = INSTR_METER_IHI;
5043 instr->meter.metarray_id = m->id;
5045 instr->meter.idx_val = idx_val;
5047 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5048 instr->meter.length.n_bits = flength->n_bits;
5049 instr->meter.length.offset = flength->offset / 8;
5051 instr->meter.color_in_val = color_in_val;
5053 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5054 instr->meter.color_out.n_bits = fcout->n_bits;
5055 instr->meter.color_out.offset = fcout->offset / 8;
5062 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
5064 struct thread *t = &p->threads[p->thread_id];
5065 struct instruction *ip = t->ip;
5068 __instr_metprefetch_h_exec(p, t, ip);
5075 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
5077 struct thread *t = &p->threads[p->thread_id];
5078 struct instruction *ip = t->ip;
5081 __instr_metprefetch_m_exec(p, t, ip);
5088 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
5090 struct thread *t = &p->threads[p->thread_id];
5091 struct instruction *ip = t->ip;
5094 __instr_metprefetch_i_exec(p, t, ip);
5101 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
5103 struct thread *t = &p->threads[p->thread_id];
5104 struct instruction *ip = t->ip;
5107 __instr_meter_hhm_exec(p, t, ip);
5114 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
5116 struct thread *t = &p->threads[p->thread_id];
5117 struct instruction *ip = t->ip;
5120 __instr_meter_hhi_exec(p, t, ip);
5127 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
5129 struct thread *t = &p->threads[p->thread_id];
5130 struct instruction *ip = t->ip;
5133 __instr_meter_hmm_exec(p, t, ip);
5140 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
5142 struct thread *t = &p->threads[p->thread_id];
5143 struct instruction *ip = t->ip;
5146 __instr_meter_hmi_exec(p, t, ip);
5153 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
5155 struct thread *t = &p->threads[p->thread_id];
5156 struct instruction *ip = t->ip;
5159 __instr_meter_mhm_exec(p, t, ip);
5166 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
5168 struct thread *t = &p->threads[p->thread_id];
5169 struct instruction *ip = t->ip;
5172 __instr_meter_mhi_exec(p, t, ip);
5179 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
5181 struct thread *t = &p->threads[p->thread_id];
5182 struct instruction *ip = t->ip;
5185 __instr_meter_mmm_exec(p, t, ip);
5192 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
5194 struct thread *t = &p->threads[p->thread_id];
5195 struct instruction *ip = t->ip;
5198 __instr_meter_mmi_exec(p, t, ip);
5205 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
5207 struct thread *t = &p->threads[p->thread_id];
5208 struct instruction *ip = t->ip;
5211 __instr_meter_ihm_exec(p, t, ip);
5218 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5220 struct thread *t = &p->threads[p->thread_id];
5221 struct instruction *ip = t->ip;
5224 __instr_meter_ihi_exec(p, t, ip);
5231 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5233 struct thread *t = &p->threads[p->thread_id];
5234 struct instruction *ip = t->ip;
5237 __instr_meter_imm_exec(p, t, ip);
5244 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5246 struct thread *t = &p->threads[p->thread_id];
5247 struct instruction *ip = t->ip;
5250 __instr_meter_imi_exec(p, t, ip);
5260 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5261 struct action *action __rte_unused,
5264 struct instruction *instr,
5265 struct instruction_data *data)
5267 CHECK(n_tokens == 2, EINVAL);
5269 strcpy(data->jmp_label, tokens[1]);
5271 instr->type = INSTR_JMP;
5272 instr->jmp.ip = NULL; /* Resolved later. */
5277 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5278 struct action *action __rte_unused,
5281 struct instruction *instr,
5282 struct instruction_data *data)
5286 CHECK(n_tokens == 3, EINVAL);
5288 strcpy(data->jmp_label, tokens[1]);
5290 h = header_parse(p, tokens[2]);
5293 instr->type = INSTR_JMP_VALID;
5294 instr->jmp.ip = NULL; /* Resolved later. */
5295 instr->jmp.header_id = h->id;
5300 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5301 struct action *action __rte_unused,
5304 struct instruction *instr,
5305 struct instruction_data *data)
5309 CHECK(n_tokens == 3, EINVAL);
5311 strcpy(data->jmp_label, tokens[1]);
5313 h = header_parse(p, tokens[2]);
5316 instr->type = INSTR_JMP_INVALID;
5317 instr->jmp.ip = NULL; /* Resolved later. */
5318 instr->jmp.header_id = h->id;
5323 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5324 struct action *action,
5327 struct instruction *instr,
5328 struct instruction_data *data)
5330 CHECK(!action, EINVAL);
5331 CHECK(n_tokens == 2, EINVAL);
5333 strcpy(data->jmp_label, tokens[1]);
5335 instr->type = INSTR_JMP_HIT;
5336 instr->jmp.ip = NULL; /* Resolved later. */
5341 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5342 struct action *action,
5345 struct instruction *instr,
5346 struct instruction_data *data)
5348 CHECK(!action, EINVAL);
5349 CHECK(n_tokens == 2, EINVAL);
5351 strcpy(data->jmp_label, tokens[1]);
5353 instr->type = INSTR_JMP_MISS;
5354 instr->jmp.ip = NULL; /* Resolved later. */
5359 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5360 struct action *action,
5363 struct instruction *instr,
5364 struct instruction_data *data)
5368 CHECK(!action, EINVAL);
5369 CHECK(n_tokens == 3, EINVAL);
5371 strcpy(data->jmp_label, tokens[1]);
5373 a = action_find(p, tokens[2]);
5376 instr->type = INSTR_JMP_ACTION_HIT;
5377 instr->jmp.ip = NULL; /* Resolved later. */
5378 instr->jmp.action_id = a->id;
5383 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5384 struct action *action,
5387 struct instruction *instr,
5388 struct instruction_data *data)
5392 CHECK(!action, EINVAL);
5393 CHECK(n_tokens == 3, EINVAL);
5395 strcpy(data->jmp_label, tokens[1]);
5397 a = action_find(p, tokens[2]);
5400 instr->type = INSTR_JMP_ACTION_MISS;
5401 instr->jmp.ip = NULL; /* Resolved later. */
5402 instr->jmp.action_id = a->id;
5407 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5408 struct action *action,
5411 struct instruction *instr,
5412 struct instruction_data *data)
5414 char *a = tokens[2], *b = tokens[3];
5415 struct field *fa, *fb;
5417 uint32_t a_struct_id, b_struct_id;
5419 CHECK(n_tokens == 4, EINVAL);
5421 strcpy(data->jmp_label, tokens[1]);
5423 fa = struct_field_parse(p, action, a, &a_struct_id);
5425 CHECK(!fa->var_size, EINVAL);
5427 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5428 fb = struct_field_parse(p, action, b, &b_struct_id);
5430 CHECK(!fb->var_size, EINVAL);
5432 instr->type = INSTR_JMP_EQ;
5433 if (a[0] != 'h' && b[0] == 'h')
5434 instr->type = INSTR_JMP_EQ_MH;
5435 if (a[0] == 'h' && b[0] != 'h')
5436 instr->type = INSTR_JMP_EQ_HM;
5437 if (a[0] == 'h' && b[0] == 'h')
5438 instr->type = INSTR_JMP_EQ_HH;
5439 instr->jmp.ip = NULL; /* Resolved later. */
5441 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5442 instr->jmp.a.n_bits = fa->n_bits;
5443 instr->jmp.a.offset = fa->offset / 8;
5444 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5445 instr->jmp.b.n_bits = fb->n_bits;
5446 instr->jmp.b.offset = fb->offset / 8;
5451 b_val = strtoull(b, &b, 0);
5452 CHECK(!b[0], EINVAL);
5455 b_val = hton64(b_val) >> (64 - fa->n_bits);
5457 instr->type = INSTR_JMP_EQ_I;
5458 instr->jmp.ip = NULL; /* Resolved later. */
5459 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5460 instr->jmp.a.n_bits = fa->n_bits;
5461 instr->jmp.a.offset = fa->offset / 8;
5462 instr->jmp.b_val = b_val;
5467 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5468 struct action *action,
5471 struct instruction *instr,
5472 struct instruction_data *data)
5474 char *a = tokens[2], *b = tokens[3];
5475 struct field *fa, *fb;
5477 uint32_t a_struct_id, b_struct_id;
5479 CHECK(n_tokens == 4, EINVAL);
5481 strcpy(data->jmp_label, tokens[1]);
5483 fa = struct_field_parse(p, action, a, &a_struct_id);
5485 CHECK(!fa->var_size, EINVAL);
5487 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5488 fb = struct_field_parse(p, action, b, &b_struct_id);
5490 CHECK(!fb->var_size, EINVAL);
5492 instr->type = INSTR_JMP_NEQ;
5493 if (a[0] != 'h' && b[0] == 'h')
5494 instr->type = INSTR_JMP_NEQ_MH;
5495 if (a[0] == 'h' && b[0] != 'h')
5496 instr->type = INSTR_JMP_NEQ_HM;
5497 if (a[0] == 'h' && b[0] == 'h')
5498 instr->type = INSTR_JMP_NEQ_HH;
5499 instr->jmp.ip = NULL; /* Resolved later. */
5501 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5502 instr->jmp.a.n_bits = fa->n_bits;
5503 instr->jmp.a.offset = fa->offset / 8;
5504 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5505 instr->jmp.b.n_bits = fb->n_bits;
5506 instr->jmp.b.offset = fb->offset / 8;
5511 b_val = strtoull(b, &b, 0);
5512 CHECK(!b[0], EINVAL);
5515 b_val = hton64(b_val) >> (64 - fa->n_bits);
5517 instr->type = INSTR_JMP_NEQ_I;
5518 instr->jmp.ip = NULL; /* Resolved later. */
5519 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5520 instr->jmp.a.n_bits = fa->n_bits;
5521 instr->jmp.a.offset = fa->offset / 8;
5522 instr->jmp.b_val = b_val;
5527 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5528 struct action *action,
5531 struct instruction *instr,
5532 struct instruction_data *data)
5534 char *a = tokens[2], *b = tokens[3];
5535 struct field *fa, *fb;
5537 uint32_t a_struct_id, b_struct_id;
5539 CHECK(n_tokens == 4, EINVAL);
5541 strcpy(data->jmp_label, tokens[1]);
5543 fa = struct_field_parse(p, action, a, &a_struct_id);
5545 CHECK(!fa->var_size, EINVAL);
5547 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5548 fb = struct_field_parse(p, action, b, &b_struct_id);
5550 CHECK(!fb->var_size, EINVAL);
5552 instr->type = INSTR_JMP_LT;
5553 if (a[0] == 'h' && b[0] != 'h')
5554 instr->type = INSTR_JMP_LT_HM;
5555 if (a[0] != 'h' && b[0] == 'h')
5556 instr->type = INSTR_JMP_LT_MH;
5557 if (a[0] == 'h' && b[0] == 'h')
5558 instr->type = INSTR_JMP_LT_HH;
5559 instr->jmp.ip = NULL; /* Resolved later. */
5561 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5562 instr->jmp.a.n_bits = fa->n_bits;
5563 instr->jmp.a.offset = fa->offset / 8;
5564 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5565 instr->jmp.b.n_bits = fb->n_bits;
5566 instr->jmp.b.offset = fb->offset / 8;
5570 /* JMP_LT_MI, JMP_LT_HI. */
5571 b_val = strtoull(b, &b, 0);
5572 CHECK(!b[0], EINVAL);
5574 instr->type = INSTR_JMP_LT_MI;
5576 instr->type = INSTR_JMP_LT_HI;
5577 instr->jmp.ip = NULL; /* Resolved later. */
5579 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5580 instr->jmp.a.n_bits = fa->n_bits;
5581 instr->jmp.a.offset = fa->offset / 8;
5582 instr->jmp.b_val = b_val;
5587 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5588 struct action *action,
5591 struct instruction *instr,
5592 struct instruction_data *data)
5594 char *a = tokens[2], *b = tokens[3];
5595 struct field *fa, *fb;
5597 uint32_t a_struct_id, b_struct_id;
5599 CHECK(n_tokens == 4, EINVAL);
5601 strcpy(data->jmp_label, tokens[1]);
5603 fa = struct_field_parse(p, action, a, &a_struct_id);
5605 CHECK(!fa->var_size, EINVAL);
5607 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5608 fb = struct_field_parse(p, action, b, &b_struct_id);
5610 CHECK(!fb->var_size, EINVAL);
5612 instr->type = INSTR_JMP_GT;
5613 if (a[0] == 'h' && b[0] != 'h')
5614 instr->type = INSTR_JMP_GT_HM;
5615 if (a[0] != 'h' && b[0] == 'h')
5616 instr->type = INSTR_JMP_GT_MH;
5617 if (a[0] == 'h' && b[0] == 'h')
5618 instr->type = INSTR_JMP_GT_HH;
5619 instr->jmp.ip = NULL; /* Resolved later. */
5621 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5622 instr->jmp.a.n_bits = fa->n_bits;
5623 instr->jmp.a.offset = fa->offset / 8;
5624 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5625 instr->jmp.b.n_bits = fb->n_bits;
5626 instr->jmp.b.offset = fb->offset / 8;
5630 /* JMP_GT_MI, JMP_GT_HI. */
5631 b_val = strtoull(b, &b, 0);
5632 CHECK(!b[0], EINVAL);
5634 instr->type = INSTR_JMP_GT_MI;
5636 instr->type = INSTR_JMP_GT_HI;
5637 instr->jmp.ip = NULL; /* Resolved later. */
5639 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5640 instr->jmp.a.n_bits = fa->n_bits;
5641 instr->jmp.a.offset = fa->offset / 8;
5642 instr->jmp.b_val = b_val;
5647 instr_jmp_exec(struct rte_swx_pipeline *p)
5649 struct thread *t = &p->threads[p->thread_id];
5650 struct instruction *ip = t->ip;
5652 TRACE("[Thread %2u] jmp\n", p->thread_id);
5654 thread_ip_set(t, ip->jmp.ip);
5658 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5660 struct thread *t = &p->threads[p->thread_id];
5661 struct instruction *ip = t->ip;
5662 uint32_t header_id = ip->jmp.header_id;
5664 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5666 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5670 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5672 struct thread *t = &p->threads[p->thread_id];
5673 struct instruction *ip = t->ip;
5674 uint32_t header_id = ip->jmp.header_id;
5676 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5678 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5682 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5684 struct thread *t = &p->threads[p->thread_id];
5685 struct instruction *ip = t->ip;
5686 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5688 TRACE("[Thread %2u] jmph\n", p->thread_id);
5690 t->ip = ip_next[t->hit];
5694 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5696 struct thread *t = &p->threads[p->thread_id];
5697 struct instruction *ip = t->ip;
5698 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5700 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5702 t->ip = ip_next[t->hit];
5706 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5708 struct thread *t = &p->threads[p->thread_id];
5709 struct instruction *ip = t->ip;
5711 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5713 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5717 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5719 struct thread *t = &p->threads[p->thread_id];
5720 struct instruction *ip = t->ip;
5722 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5724 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5728 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5730 struct thread *t = &p->threads[p->thread_id];
5731 struct instruction *ip = t->ip;
5733 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5739 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5741 struct thread *t = &p->threads[p->thread_id];
5742 struct instruction *ip = t->ip;
5744 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5746 JMP_CMP_MH(t, ip, ==);
5750 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5752 struct thread *t = &p->threads[p->thread_id];
5753 struct instruction *ip = t->ip;
5755 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5757 JMP_CMP_HM(t, ip, ==);
5761 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5763 struct thread *t = &p->threads[p->thread_id];
5764 struct instruction *ip = t->ip;
5766 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5768 JMP_CMP_HH_FAST(t, ip, ==);
5772 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5774 struct thread *t = &p->threads[p->thread_id];
5775 struct instruction *ip = t->ip;
5777 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5779 JMP_CMP_I(t, ip, ==);
5783 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5785 struct thread *t = &p->threads[p->thread_id];
5786 struct instruction *ip = t->ip;
5788 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5794 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5796 struct thread *t = &p->threads[p->thread_id];
5797 struct instruction *ip = t->ip;
5799 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5801 JMP_CMP_MH(t, ip, !=);
5805 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5807 struct thread *t = &p->threads[p->thread_id];
5808 struct instruction *ip = t->ip;
5810 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5812 JMP_CMP_HM(t, ip, !=);
5816 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5818 struct thread *t = &p->threads[p->thread_id];
5819 struct instruction *ip = t->ip;
5821 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5823 JMP_CMP_HH_FAST(t, ip, !=);
5827 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5829 struct thread *t = &p->threads[p->thread_id];
5830 struct instruction *ip = t->ip;
5832 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5834 JMP_CMP_I(t, ip, !=);
5838 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5840 struct thread *t = &p->threads[p->thread_id];
5841 struct instruction *ip = t->ip;
5843 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5849 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5851 struct thread *t = &p->threads[p->thread_id];
5852 struct instruction *ip = t->ip;
5854 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5856 JMP_CMP_MH(t, ip, <);
5860 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5862 struct thread *t = &p->threads[p->thread_id];
5863 struct instruction *ip = t->ip;
5865 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5867 JMP_CMP_HM(t, ip, <);
5871 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5873 struct thread *t = &p->threads[p->thread_id];
5874 struct instruction *ip = t->ip;
5876 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5878 JMP_CMP_HH(t, ip, <);
5882 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5884 struct thread *t = &p->threads[p->thread_id];
5885 struct instruction *ip = t->ip;
5887 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5889 JMP_CMP_MI(t, ip, <);
5893 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5895 struct thread *t = &p->threads[p->thread_id];
5896 struct instruction *ip = t->ip;
5898 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5900 JMP_CMP_HI(t, ip, <);
5904 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5906 struct thread *t = &p->threads[p->thread_id];
5907 struct instruction *ip = t->ip;
5909 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5915 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5917 struct thread *t = &p->threads[p->thread_id];
5918 struct instruction *ip = t->ip;
5920 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5922 JMP_CMP_MH(t, ip, >);
5926 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5928 struct thread *t = &p->threads[p->thread_id];
5929 struct instruction *ip = t->ip;
5931 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5933 JMP_CMP_HM(t, ip, >);
5937 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5939 struct thread *t = &p->threads[p->thread_id];
5940 struct instruction *ip = t->ip;
5942 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5944 JMP_CMP_HH(t, ip, >);
5948 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5950 struct thread *t = &p->threads[p->thread_id];
5951 struct instruction *ip = t->ip;
5953 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5955 JMP_CMP_MI(t, ip, >);
5959 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5961 struct thread *t = &p->threads[p->thread_id];
5962 struct instruction *ip = t->ip;
5964 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5966 JMP_CMP_HI(t, ip, >);
5973 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5974 struct action *action,
5975 char **tokens __rte_unused,
5977 struct instruction *instr,
5978 struct instruction_data *data __rte_unused)
5980 CHECK(action, EINVAL);
5981 CHECK(n_tokens == 1, EINVAL);
5983 instr->type = INSTR_RETURN;
5988 instr_return_exec(struct rte_swx_pipeline *p)
5990 struct thread *t = &p->threads[p->thread_id];
5992 TRACE("[Thread %2u] return\n", p->thread_id);
5998 instr_translate(struct rte_swx_pipeline *p,
5999 struct action *action,
6001 struct instruction *instr,
6002 struct instruction_data *data)
6004 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
6005 int n_tokens = 0, tpos = 0;
6007 /* Parse the instruction string into tokens. */
6011 token = strtok_r(string, " \t\v", &string);
6015 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
6016 CHECK_NAME(token, EINVAL);
6018 tokens[n_tokens] = token;
6022 CHECK(n_tokens, EINVAL);
6024 /* Handle the optional instruction label. */
6025 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
6026 strcpy(data->label, tokens[0]);
6029 CHECK(n_tokens - tpos, EINVAL);
6032 /* Identify the instruction type. */
6033 if (!strcmp(tokens[tpos], "rx"))
6034 return instr_rx_translate(p,
6041 if (!strcmp(tokens[tpos], "tx"))
6042 return instr_tx_translate(p,
6049 if (!strcmp(tokens[tpos], "drop"))
6050 return instr_drop_translate(p,
6057 if (!strcmp(tokens[tpos], "mirror"))
6058 return instr_mirror_translate(p,
6065 if (!strcmp(tokens[tpos], "recirculate"))
6066 return instr_recirculate_translate(p,
6073 if (!strcmp(tokens[tpos], "recircid"))
6074 return instr_recircid_translate(p,
6081 if (!strcmp(tokens[tpos], "extract"))
6082 return instr_hdr_extract_translate(p,
6089 if (!strcmp(tokens[tpos], "lookahead"))
6090 return instr_hdr_lookahead_translate(p,
6097 if (!strcmp(tokens[tpos], "emit"))
6098 return instr_hdr_emit_translate(p,
6105 if (!strcmp(tokens[tpos], "validate"))
6106 return instr_hdr_validate_translate(p,
6113 if (!strcmp(tokens[tpos], "invalidate"))
6114 return instr_hdr_invalidate_translate(p,
6121 if (!strcmp(tokens[tpos], "mov"))
6122 return instr_mov_translate(p,
6129 if (!strcmp(tokens[tpos], "add"))
6130 return instr_alu_add_translate(p,
6137 if (!strcmp(tokens[tpos], "sub"))
6138 return instr_alu_sub_translate(p,
6145 if (!strcmp(tokens[tpos], "ckadd"))
6146 return instr_alu_ckadd_translate(p,
6153 if (!strcmp(tokens[tpos], "cksub"))
6154 return instr_alu_cksub_translate(p,
6161 if (!strcmp(tokens[tpos], "and"))
6162 return instr_alu_and_translate(p,
6169 if (!strcmp(tokens[tpos], "or"))
6170 return instr_alu_or_translate(p,
6177 if (!strcmp(tokens[tpos], "xor"))
6178 return instr_alu_xor_translate(p,
6185 if (!strcmp(tokens[tpos], "shl"))
6186 return instr_alu_shl_translate(p,
6193 if (!strcmp(tokens[tpos], "shr"))
6194 return instr_alu_shr_translate(p,
6201 if (!strcmp(tokens[tpos], "regprefetch"))
6202 return instr_regprefetch_translate(p,
6209 if (!strcmp(tokens[tpos], "regrd"))
6210 return instr_regrd_translate(p,
6217 if (!strcmp(tokens[tpos], "regwr"))
6218 return instr_regwr_translate(p,
6225 if (!strcmp(tokens[tpos], "regadd"))
6226 return instr_regadd_translate(p,
6233 if (!strcmp(tokens[tpos], "metprefetch"))
6234 return instr_metprefetch_translate(p,
6241 if (!strcmp(tokens[tpos], "meter"))
6242 return instr_meter_translate(p,
6249 if (!strcmp(tokens[tpos], "table"))
6250 return instr_table_translate(p,
6257 if (!strcmp(tokens[tpos], "learn"))
6258 return instr_learn_translate(p,
6264 if (!strcmp(tokens[tpos], "rearm"))
6265 return instr_rearm_translate(p,
6272 if (!strcmp(tokens[tpos], "forget"))
6273 return instr_forget_translate(p,
6280 if (!strcmp(tokens[tpos], "extern"))
6281 return instr_extern_translate(p,
6288 if (!strcmp(tokens[tpos], "hash"))
6289 return instr_hash_translate(p,
6296 if (!strcmp(tokens[tpos], "jmp"))
6297 return instr_jmp_translate(p,
6304 if (!strcmp(tokens[tpos], "jmpv"))
6305 return instr_jmp_valid_translate(p,
6312 if (!strcmp(tokens[tpos], "jmpnv"))
6313 return instr_jmp_invalid_translate(p,
6320 if (!strcmp(tokens[tpos], "jmph"))
6321 return instr_jmp_hit_translate(p,
6328 if (!strcmp(tokens[tpos], "jmpnh"))
6329 return instr_jmp_miss_translate(p,
6336 if (!strcmp(tokens[tpos], "jmpa"))
6337 return instr_jmp_action_hit_translate(p,
6344 if (!strcmp(tokens[tpos], "jmpna"))
6345 return instr_jmp_action_miss_translate(p,
6352 if (!strcmp(tokens[tpos], "jmpeq"))
6353 return instr_jmp_eq_translate(p,
6360 if (!strcmp(tokens[tpos], "jmpneq"))
6361 return instr_jmp_neq_translate(p,
6368 if (!strcmp(tokens[tpos], "jmplt"))
6369 return instr_jmp_lt_translate(p,
6376 if (!strcmp(tokens[tpos], "jmpgt"))
6377 return instr_jmp_gt_translate(p,
6384 if (!strcmp(tokens[tpos], "return"))
6385 return instr_return_translate(p,
6395 static struct instruction_data *
6396 label_find(struct instruction_data *data, uint32_t n, const char *label)
6400 for (i = 0; i < n; i++)
6401 if (!strcmp(label, data[i].label))
6408 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6410 uint32_t count = 0, i;
6415 for (i = 0; i < n; i++)
6416 if (!strcmp(label, data[i].jmp_label))
6423 instr_label_check(struct instruction_data *instruction_data,
6424 uint32_t n_instructions)
6428 /* Check that all instruction labels are unique. */
6429 for (i = 0; i < n_instructions; i++) {
6430 struct instruction_data *data = &instruction_data[i];
6431 char *label = data->label;
6437 for (j = i + 1; j < n_instructions; j++)
6438 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6441 /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6442 for (i = 0; i < n_instructions; i++) {
6443 struct instruction_data *data = &instruction_data[i];
6444 char *label = data->label;
6445 char *jmp_label = data->jmp_label;
6447 /* Continue if this instruction does not have a label or it is not a jump. */
6448 if (!label[0] || !jmp_label[0])
6451 CHECK(strcmp(label, jmp_label), EINVAL);
6454 /* Get users for each instruction label. */
6455 for (i = 0; i < n_instructions; i++) {
6456 struct instruction_data *data = &instruction_data[i];
6457 char *label = data->label;
6459 data->n_users = label_is_used(instruction_data,
6468 instr_jmp_resolve(struct instruction *instructions,
6469 struct instruction_data *instruction_data,
6470 uint32_t n_instructions)
6474 for (i = 0; i < n_instructions; i++) {
6475 struct instruction *instr = &instructions[i];
6476 struct instruction_data *data = &instruction_data[i];
6477 struct instruction_data *found;
6479 if (!instruction_is_jmp(instr))
6482 found = label_find(instruction_data,
6485 CHECK(found, EINVAL);
6487 instr->jmp.ip = &instructions[found - instruction_data];
6494 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6496 struct instruction *instr,
6497 struct instruction_data *data __rte_unused,
6498 uint32_t n_instructions)
6501 enum instruction_type type;
6504 /* Check that the first instruction is rx. */
6505 CHECK(instr[0].type == INSTR_RX, EINVAL);
6507 /* Check that there is at least one tx instruction. */
6508 for (i = 0; i < n_instructions; i++) {
6509 type = instr[i].type;
6511 if (instruction_is_tx(type))
6514 CHECK(i < n_instructions, EINVAL);
6516 /* Check that the last instruction is either tx or unconditional
6519 type = instr[n_instructions - 1].type;
6520 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6524 enum instruction_type type;
6527 /* Check that there is at least one return or tx instruction. */
6528 for (i = 0; i < n_instructions; i++) {
6529 type = instr[i].type;
6531 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6534 CHECK(i < n_instructions, EINVAL);
6541 instr_compact(struct instruction *instructions,
6542 struct instruction_data *instruction_data,
6543 uint32_t n_instructions)
6545 uint32_t i, pos = 0;
6547 /* Eliminate the invalid instructions that have been optimized out. */
6548 for (i = 0; i < n_instructions; i++) {
6549 struct instruction *instr = &instructions[i];
6550 struct instruction_data *data = &instruction_data[i];
6556 memcpy(&instructions[pos], instr, sizeof(*instr));
6557 memcpy(&instruction_data[pos], data, sizeof(*data));
6567 instr_pattern_extract_many_search(struct instruction *instr,
6568 struct instruction_data *data,
6570 uint32_t *n_pattern_instr)
6574 for (i = 0; i < n_instr; i++) {
6575 if (data[i].invalid)
6578 if (instr[i].type != INSTR_HDR_EXTRACT)
6581 if (i == RTE_DIM(instr->io.hdr.header_id))
6584 if (i && data[i].n_users)
6591 *n_pattern_instr = i;
6596 instr_pattern_extract_many_replace(struct instruction *instr,
6597 struct instruction_data *data,
6602 for (i = 1; i < n_instr; i++) {
6604 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6605 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6606 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6608 data[i].invalid = 1;
6613 instr_pattern_extract_many_optimize(struct instruction *instructions,
6614 struct instruction_data *instruction_data,
6615 uint32_t n_instructions)
6619 for (i = 0; i < n_instructions; ) {
6620 struct instruction *instr = &instructions[i];
6621 struct instruction_data *data = &instruction_data[i];
6622 uint32_t n_instr = 0;
6626 detected = instr_pattern_extract_many_search(instr,
6631 instr_pattern_extract_many_replace(instr,
6638 /* No pattern starting at the current instruction. */
6642 /* Eliminate the invalid instructions that have been optimized out. */
6643 n_instructions = instr_compact(instructions,
6647 return n_instructions;
6651 instr_pattern_emit_many_tx_search(struct instruction *instr,
6652 struct instruction_data *data,
6654 uint32_t *n_pattern_instr)
6658 for (i = 0; i < n_instr; i++) {
6659 if (data[i].invalid)
6662 if (instr[i].type != INSTR_HDR_EMIT)
6665 if (i == RTE_DIM(instr->io.hdr.header_id))
6668 if (i && data[i].n_users)
6675 if (instr[i].type != INSTR_TX)
6678 if (data[i].n_users)
6683 *n_pattern_instr = i;
6688 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6689 struct instruction_data *data,
6694 /* Any emit instruction in addition to the first one. */
6695 for (i = 1; i < n_instr - 1; i++) {
6697 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6698 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6699 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6701 data[i].invalid = 1;
6704 /* The TX instruction is the last one in the pattern. */
6706 instr[0].io.io.offset = instr[i].io.io.offset;
6707 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6708 data[i].invalid = 1;
6712 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6713 struct instruction_data *instruction_data,
6714 uint32_t n_instructions)
6718 for (i = 0; i < n_instructions; ) {
6719 struct instruction *instr = &instructions[i];
6720 struct instruction_data *data = &instruction_data[i];
6721 uint32_t n_instr = 0;
6724 /* Emit many + TX. */
6725 detected = instr_pattern_emit_many_tx_search(instr,
6730 instr_pattern_emit_many_tx_replace(instr,
6737 /* No pattern starting at the current instruction. */
6741 /* Eliminate the invalid instructions that have been optimized out. */
6742 n_instructions = instr_compact(instructions,
6746 return n_instructions;
6750 action_arg_src_mov_count(struct action *a,
6752 struct instruction *instructions,
6753 struct instruction_data *instruction_data,
6754 uint32_t n_instructions);
6757 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6759 struct instruction *instr,
6760 struct instruction_data *data,
6762 struct instruction *instructions,
6763 struct instruction_data *instruction_data,
6764 uint32_t n_instructions,
6765 uint32_t *n_pattern_instr)
6768 uint32_t src_field_id, i, j;
6770 /* Prerequisites. */
6774 /* First instruction: MOV_HM. */
6775 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6778 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6779 if (!h || h->st->var_size)
6782 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6783 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6786 if (src_field_id == a->st->n_fields)
6789 if (instr[0].mov.dst.offset ||
6790 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6791 instr[0].mov.src.struct_id ||
6792 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6793 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6796 if ((n_instr < h->st->n_fields + 1) ||
6797 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6800 /* Subsequent instructions: MOV_HM. */
6801 for (i = 1; i < h->st->n_fields; i++)
6802 if (data[i].invalid ||
6804 (instr[i].type != INSTR_MOV_HM) ||
6805 (instr[i].mov.dst.struct_id != h->struct_id) ||
6806 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6807 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6808 instr[i].mov.src.struct_id ||
6809 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6810 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6811 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6814 /* Last instruction: HDR_VALIDATE. */
6815 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6816 (instr[i].valid.header_id != h->id))
6819 /* Check that none of the action args that are used as source for this
6820 * DMA transfer are not used as source in any other mov instruction.
6822 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6825 n_users = action_arg_src_mov_count(a,
6834 *n_pattern_instr = 1 + i;
6839 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6841 struct instruction *instr,
6842 struct instruction_data *data,
6846 uint32_t src_field_id, src_offset, i;
6848 /* Read from the instructions before they are modified. */
6849 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6853 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6854 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6857 if (src_field_id == a->st->n_fields)
6860 src_offset = instr[0].mov.src.offset;
6862 /* Modify the instructions. */
6863 instr[0].type = INSTR_DMA_HT;
6864 instr[0].dma.dst.header_id[0] = h->id;
6865 instr[0].dma.dst.struct_id[0] = h->struct_id;
6866 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6867 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6869 for (i = 1; i < n_instr; i++)
6870 data[i].invalid = 1;
6872 /* Update the endianness of the action arguments to header endianness. */
6873 for (i = 0; i < h->st->n_fields; i++)
6874 a->args_endianness[src_field_id + i] = 1;
6878 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6880 struct instruction *instructions,
6881 struct instruction_data *instruction_data,
6882 uint32_t n_instructions)
6887 return n_instructions;
6889 for (i = 0; i < n_instructions; ) {
6890 struct instruction *instr = &instructions[i];
6891 struct instruction_data *data = &instruction_data[i];
6892 uint32_t n_instr = 0;
6895 /* Mov all + validate. */
6896 detected = instr_pattern_mov_all_validate_search(p,
6906 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6911 /* No pattern starting at the current instruction. */
6915 /* Eliminate the invalid instructions that have been optimized out. */
6916 n_instructions = instr_compact(instructions,
6920 return n_instructions;
6924 instr_pattern_dma_many_search(struct instruction *instr,
6925 struct instruction_data *data,
6927 uint32_t *n_pattern_instr)
6931 for (i = 0; i < n_instr; i++) {
6932 if (data[i].invalid)
6935 if (instr[i].type != INSTR_DMA_HT)
6938 if (i == RTE_DIM(instr->dma.dst.header_id))
6941 if (i && data[i].n_users)
6948 *n_pattern_instr = i;
6953 instr_pattern_dma_many_replace(struct instruction *instr,
6954 struct instruction_data *data,
6959 for (i = 1; i < n_instr; i++) {
6961 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6962 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6963 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6964 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6966 data[i].invalid = 1;
6971 instr_pattern_dma_many_optimize(struct instruction *instructions,
6972 struct instruction_data *instruction_data,
6973 uint32_t n_instructions)
6977 for (i = 0; i < n_instructions; ) {
6978 struct instruction *instr = &instructions[i];
6979 struct instruction_data *data = &instruction_data[i];
6980 uint32_t n_instr = 0;
6984 detected = instr_pattern_dma_many_search(instr,
6989 instr_pattern_dma_many_replace(instr, data, n_instr);
6994 /* No pattern starting at the current instruction. */
6998 /* Eliminate the invalid instructions that have been optimized out. */
6999 n_instructions = instr_compact(instructions,
7003 return n_instructions;
7007 instr_optimize(struct rte_swx_pipeline *p,
7009 struct instruction *instructions,
7010 struct instruction_data *instruction_data,
7011 uint32_t n_instructions)
7014 n_instructions = instr_pattern_extract_many_optimize(instructions,
7018 /* Emit many + TX. */
7019 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
7023 /* Mov all + validate. */
7024 n_instructions = instr_pattern_mov_all_validate_optimize(p,
7031 n_instructions = instr_pattern_dma_many_optimize(instructions,
7035 return n_instructions;
7039 instruction_config(struct rte_swx_pipeline *p,
7041 const char **instructions,
7042 uint32_t n_instructions)
7044 struct instruction *instr = NULL;
7045 struct instruction_data *data = NULL;
7049 CHECK(n_instructions, EINVAL);
7050 CHECK(instructions, EINVAL);
7051 for (i = 0; i < n_instructions; i++)
7052 CHECK_INSTRUCTION(instructions[i], EINVAL);
7054 /* Memory allocation. */
7055 instr = calloc(n_instructions, sizeof(struct instruction));
7061 data = calloc(n_instructions, sizeof(struct instruction_data));
7067 for (i = 0; i < n_instructions; i++) {
7068 char *string = strdup(instructions[i]);
7074 err = instr_translate(p, a, string, &instr[i], &data[i]);
7083 err = instr_label_check(data, n_instructions);
7087 err = instr_verify(p, a, instr, data, n_instructions);
7091 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
7093 err = instr_jmp_resolve(instr, data, n_instructions);
7098 a->instructions = instr;
7099 a->instruction_data = data;
7100 a->n_instructions = n_instructions;
7102 p->instructions = instr;
7103 p->instruction_data = data;
7104 p->n_instructions = n_instructions;
7115 static instr_exec_t instruction_table[] = {
7116 [INSTR_RX] = instr_rx_exec,
7117 [INSTR_TX] = instr_tx_exec,
7118 [INSTR_TX_I] = instr_tx_i_exec,
7119 [INSTR_DROP] = instr_drop_exec,
7120 [INSTR_MIRROR] = instr_mirror_exec,
7121 [INSTR_RECIRCULATE] = instr_recirculate_exec,
7122 [INSTR_RECIRCID] = instr_recircid_exec,
7124 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
7125 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
7126 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
7127 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
7128 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
7129 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
7130 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
7131 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
7132 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
7133 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
7135 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
7136 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
7137 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
7138 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
7139 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
7140 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
7141 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
7142 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
7143 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
7145 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
7146 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
7148 [INSTR_MOV] = instr_mov_exec,
7149 [INSTR_MOV_MH] = instr_mov_mh_exec,
7150 [INSTR_MOV_HM] = instr_mov_hm_exec,
7151 [INSTR_MOV_HH] = instr_mov_hh_exec,
7152 [INSTR_MOV_I] = instr_mov_i_exec,
7154 [INSTR_DMA_HT] = instr_dma_ht_exec,
7155 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
7156 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
7157 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
7158 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
7159 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
7160 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
7161 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
7163 [INSTR_ALU_ADD] = instr_alu_add_exec,
7164 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
7165 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
7166 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
7167 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
7168 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
7170 [INSTR_ALU_SUB] = instr_alu_sub_exec,
7171 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
7172 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
7173 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
7174 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
7175 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
7177 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
7178 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
7179 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
7180 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
7182 [INSTR_ALU_AND] = instr_alu_and_exec,
7183 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
7184 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
7185 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
7186 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
7188 [INSTR_ALU_OR] = instr_alu_or_exec,
7189 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
7190 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
7191 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
7192 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
7194 [INSTR_ALU_XOR] = instr_alu_xor_exec,
7195 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
7196 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
7197 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
7198 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
7200 [INSTR_ALU_SHL] = instr_alu_shl_exec,
7201 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
7202 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
7203 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
7204 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
7205 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
7207 [INSTR_ALU_SHR] = instr_alu_shr_exec,
7208 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
7209 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
7210 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
7211 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
7212 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
7214 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
7215 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
7216 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
7218 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
7219 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
7220 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
7221 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
7222 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
7223 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7225 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7226 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7227 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7228 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7229 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7230 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7231 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7232 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7233 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7235 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7236 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7237 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7238 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7239 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7240 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7241 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7242 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7243 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7245 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7246 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7247 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7249 [INSTR_METER_HHM] = instr_meter_hhm_exec,
7250 [INSTR_METER_HHI] = instr_meter_hhi_exec,
7251 [INSTR_METER_HMM] = instr_meter_hmm_exec,
7252 [INSTR_METER_HMI] = instr_meter_hmi_exec,
7253 [INSTR_METER_MHM] = instr_meter_mhm_exec,
7254 [INSTR_METER_MHI] = instr_meter_mhi_exec,
7255 [INSTR_METER_MMM] = instr_meter_mmm_exec,
7256 [INSTR_METER_MMI] = instr_meter_mmi_exec,
7257 [INSTR_METER_IHM] = instr_meter_ihm_exec,
7258 [INSTR_METER_IHI] = instr_meter_ihi_exec,
7259 [INSTR_METER_IMM] = instr_meter_imm_exec,
7260 [INSTR_METER_IMI] = instr_meter_imi_exec,
7262 [INSTR_TABLE] = instr_table_exec,
7263 [INSTR_TABLE_AF] = instr_table_af_exec,
7264 [INSTR_SELECTOR] = instr_selector_exec,
7265 [INSTR_LEARNER] = instr_learner_exec,
7266 [INSTR_LEARNER_AF] = instr_learner_af_exec,
7267 [INSTR_LEARNER_LEARN] = instr_learn_exec,
7268 [INSTR_LEARNER_REARM] = instr_rearm_exec,
7269 [INSTR_LEARNER_REARM_NEW] = instr_rearm_new_exec,
7270 [INSTR_LEARNER_FORGET] = instr_forget_exec,
7271 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7272 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7273 [INSTR_HASH_FUNC] = instr_hash_func_exec,
7275 [INSTR_JMP] = instr_jmp_exec,
7276 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7277 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7278 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7279 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7280 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7281 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7283 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7284 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7285 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7286 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7287 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7289 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7290 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7291 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7292 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7293 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7295 [INSTR_JMP_LT] = instr_jmp_lt_exec,
7296 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7297 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7298 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7299 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7300 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7302 [INSTR_JMP_GT] = instr_jmp_gt_exec,
7303 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7304 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7305 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7306 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7307 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7309 [INSTR_RETURN] = instr_return_exec,
7313 instruction_table_build(struct rte_swx_pipeline *p)
7315 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7316 sizeof(struct instr_exec_t *));
7317 if (!p->instruction_table)
7320 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7326 instruction_table_build_free(struct rte_swx_pipeline *p)
7328 if (!p->instruction_table)
7331 free(p->instruction_table);
7332 p->instruction_table = NULL;
7336 instruction_table_free(struct rte_swx_pipeline *p)
7338 instruction_table_build_free(p);
7342 instr_exec(struct rte_swx_pipeline *p)
7344 struct thread *t = &p->threads[p->thread_id];
7345 struct instruction *ip = t->ip;
7346 instr_exec_t instr = p->instruction_table[ip->type];
7354 static struct action *
7355 action_find(struct rte_swx_pipeline *p, const char *name)
7357 struct action *elem;
7362 TAILQ_FOREACH(elem, &p->actions, node)
7363 if (strcmp(elem->name, name) == 0)
7369 static struct action *
7370 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7372 struct action *action = NULL;
7374 TAILQ_FOREACH(action, &p->actions, node)
7375 if (action->id == id)
7381 static struct field *
7382 action_field_find(struct action *a, const char *name)
7384 return a->st ? struct_type_field_find(a->st, name) : NULL;
7387 static struct field *
7388 action_field_parse(struct action *action, const char *name)
7390 if (name[0] != 't' || name[1] != '.')
7393 return action_field_find(action, &name[2]);
7397 action_has_nbo_args(struct action *a)
7401 /* Return if the action does not have any args. */
7403 return 0; /* FALSE */
7405 for (i = 0; i < a->st->n_fields; i++)
7406 if (a->args_endianness[i])
7407 return 1; /* TRUE */
7409 return 0; /* FALSE */
7413 action_does_learning(struct action *a)
7417 for (i = 0; i < a->n_instructions; i++)
7418 switch (a->instructions[i].type) {
7419 case INSTR_LEARNER_LEARN:
7420 return 1; /* TRUE */
7422 case INSTR_LEARNER_FORGET:
7423 return 1; /* TRUE */
7429 return 0; /* FALSE */
7433 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7435 const char *args_struct_type_name,
7436 const char **instructions,
7437 uint32_t n_instructions)
7439 struct struct_type *args_struct_type = NULL;
7440 struct action *a = NULL;
7445 CHECK_NAME(name, EINVAL);
7446 CHECK(!action_find(p, name), EEXIST);
7448 if (args_struct_type_name) {
7449 CHECK_NAME(args_struct_type_name, EINVAL);
7450 args_struct_type = struct_type_find(p, args_struct_type_name);
7451 CHECK(args_struct_type, EINVAL);
7452 CHECK(!args_struct_type->var_size, EINVAL);
7455 /* Node allocation. */
7456 a = calloc(1, sizeof(struct action));
7462 if (args_struct_type) {
7463 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7464 if (!a->args_endianness) {
7470 /* Node initialization. */
7471 strcpy(a->name, name);
7472 a->st = args_struct_type;
7473 a->id = p->n_actions;
7475 /* Instruction translation. */
7476 status = instruction_config(p, a, instructions, n_instructions);
7480 /* Node add to tailq. */
7481 TAILQ_INSERT_TAIL(&p->actions, a, node);
7490 free(a->args_endianness);
7491 free(a->instructions);
7492 free(a->instruction_data);
7499 action_build(struct rte_swx_pipeline *p)
7501 struct action *action;
7503 /* p->action_instructions. */
7504 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7505 CHECK(p->action_instructions, ENOMEM);
7507 TAILQ_FOREACH(action, &p->actions, node)
7508 p->action_instructions[action->id] = action->instructions;
7510 /* p->action_funcs. */
7511 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7512 CHECK(p->action_funcs, ENOMEM);
7518 action_build_free(struct rte_swx_pipeline *p)
7520 free(p->action_funcs);
7521 p->action_funcs = NULL;
7523 free(p->action_instructions);
7524 p->action_instructions = NULL;
7528 action_free(struct rte_swx_pipeline *p)
7530 action_build_free(p);
7533 struct action *action;
7535 action = TAILQ_FIRST(&p->actions);
7539 TAILQ_REMOVE(&p->actions, action, node);
7540 free(action->args_endianness);
7541 free(action->instructions);
7542 free(action->instruction_data);
7548 action_arg_src_mov_count(struct action *a,
7550 struct instruction *instructions,
7551 struct instruction_data *instruction_data,
7552 uint32_t n_instructions)
7554 uint32_t offset, n_users = 0, i;
7557 (arg_id >= a->st->n_fields) ||
7559 !instruction_data ||
7563 offset = a->st->fields[arg_id].offset / 8;
7565 for (i = 0; i < n_instructions; i++) {
7566 struct instruction *instr = &instructions[i];
7567 struct instruction_data *data = &instruction_data[i];
7569 if (data->invalid ||
7570 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7571 instr->mov.src.struct_id ||
7572 (instr->mov.src.offset != offset))
7581 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7582 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7583 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7585 #define field_ntoh(val, n_bits) (val)
7586 #define field_hton(val, n_bits) (val)
7589 #define ACTION_ARGS_TOKENS_MAX 256
7592 action_args_parse(struct action *a, const char *args, uint8_t *data)
7594 char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7595 uint32_t n_tokens = 0, offset = 0, i;
7599 if (!a->st || !args || !args[0]) {
7604 /* Memory allocation. */
7611 /* Parse the string into tokens. */
7615 token = strtok_r(s, " \f\n\r\t\v", &s);
7619 if (n_tokens >= RTE_DIM(tokens)) {
7624 tokens[n_tokens] = token;
7629 if (n_tokens != a->st->n_fields * 2) {
7634 /* Process the action arguments. */
7635 for (i = 0; i < a->st->n_fields; i++) {
7636 struct field *f = &a->st->fields[i];
7637 char *arg_name = tokens[i * 2];
7638 char *arg_val = tokens[i * 2 + 1];
7641 if (strcmp(arg_name, f->name)) {
7646 val = strtoull(arg_val, &arg_val, 0);
7652 /* Endianness conversion. */
7653 if (a->args_endianness[i])
7654 val = field_hton(val, f->n_bits);
7656 /* Copy to entry. */
7657 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7658 offset += f->n_bits / 8;
7669 static struct table_type *
7670 table_type_find(struct rte_swx_pipeline *p, const char *name)
7672 struct table_type *elem;
7674 TAILQ_FOREACH(elem, &p->table_types, node)
7675 if (strcmp(elem->name, name) == 0)
7681 static struct table_type *
7682 table_type_resolve(struct rte_swx_pipeline *p,
7683 const char *recommended_type_name,
7684 enum rte_swx_table_match_type match_type)
7686 struct table_type *elem;
7688 /* Only consider the recommended type if the match type is correct. */
7689 if (recommended_type_name)
7690 TAILQ_FOREACH(elem, &p->table_types, node)
7691 if (!strcmp(elem->name, recommended_type_name) &&
7692 (elem->match_type == match_type))
7695 /* Ignore the recommended type and get the first element with this match
7698 TAILQ_FOREACH(elem, &p->table_types, node)
7699 if (elem->match_type == match_type)
7705 static struct table *
7706 table_find(struct rte_swx_pipeline *p, const char *name)
7710 TAILQ_FOREACH(elem, &p->tables, node)
7711 if (strcmp(elem->name, name) == 0)
7717 static struct table *
7718 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7720 struct table *table = NULL;
7722 TAILQ_FOREACH(table, &p->tables, node)
7723 if (table->id == id)
7730 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7732 enum rte_swx_table_match_type match_type,
7733 struct rte_swx_table_ops *ops)
7735 struct table_type *elem;
7739 CHECK_NAME(name, EINVAL);
7740 CHECK(!table_type_find(p, name), EEXIST);
7743 CHECK(ops->create, EINVAL);
7744 CHECK(ops->lkp, EINVAL);
7745 CHECK(ops->free, EINVAL);
7747 /* Node allocation. */
7748 elem = calloc(1, sizeof(struct table_type));
7749 CHECK(elem, ENOMEM);
7751 /* Node initialization. */
7752 strcpy(elem->name, name);
7753 elem->match_type = match_type;
7754 memcpy(&elem->ops, ops, sizeof(*ops));
7756 /* Node add to tailq. */
7757 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7763 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7765 enum rte_swx_table_match_type *match_type)
7767 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7769 for (i = 0; i < n_fields; i++) {
7770 struct rte_swx_match_field_params *f = &fields[i];
7772 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7775 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7779 if ((n_fields_lpm > 1) ||
7780 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7783 *match_type = (n_fields_em == n_fields) ?
7784 RTE_SWX_TABLE_MATCH_EXACT :
7785 RTE_SWX_TABLE_MATCH_WILDCARD;
7791 table_match_fields_check(struct rte_swx_pipeline *p,
7792 struct rte_swx_pipeline_table_params *params,
7793 struct header **header)
7795 struct header *h0 = NULL;
7796 struct field *hf, *mf;
7797 uint32_t *offset = NULL, i;
7800 /* Return if no match fields. */
7801 if (!params->n_fields) {
7802 if (params->fields) {
7813 /* Memory allocation. */
7814 offset = calloc(params->n_fields, sizeof(uint32_t));
7820 /* Check that all the match fields belong to either the same header or
7823 hf = header_field_parse(p, params->fields[0].name, &h0);
7824 mf = metadata_field_parse(p, params->fields[0].name);
7825 if ((!hf && !mf) || (hf && hf->var_size)) {
7830 offset[0] = h0 ? hf->offset : mf->offset;
7832 for (i = 1; i < params->n_fields; i++)
7836 hf = header_field_parse(p, params->fields[i].name, &h);
7837 if (!hf || (h->id != h0->id) || hf->var_size) {
7842 offset[i] = hf->offset;
7844 mf = metadata_field_parse(p, params->fields[i].name);
7850 offset[i] = mf->offset;
7853 /* Check that there are no duplicated match fields. */
7854 for (i = 0; i < params->n_fields; i++) {
7857 for (j = 0; j < i; j++)
7858 if (offset[j] == offset[i]) {
7874 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7876 struct rte_swx_pipeline_table_params *params,
7877 const char *recommended_table_type_name,
7881 struct table_type *type;
7882 struct table *t = NULL;
7883 struct action *default_action;
7884 struct header *header = NULL;
7885 uint32_t action_data_size_max = 0, i;
7890 CHECK_NAME(name, EINVAL);
7891 CHECK(!table_find(p, name), EEXIST);
7892 CHECK(!selector_find(p, name), EEXIST);
7893 CHECK(!learner_find(p, name), EEXIST);
7895 CHECK(params, EINVAL);
7898 status = table_match_fields_check(p, params, &header);
7902 /* Action checks. */
7903 CHECK(params->n_actions, EINVAL);
7904 CHECK(params->action_names, EINVAL);
7905 for (i = 0; i < params->n_actions; i++) {
7906 const char *action_name = params->action_names[i];
7908 uint32_t action_data_size;
7909 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7911 CHECK_NAME(action_name, EINVAL);
7913 a = action_find(p, action_name);
7915 CHECK(!action_does_learning(a), EINVAL);
7917 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7918 if (action_data_size > action_data_size_max)
7919 action_data_size_max = action_data_size;
7921 if (params->action_is_for_table_entries)
7922 action_is_for_table_entries = params->action_is_for_table_entries[i];
7923 if (params->action_is_for_default_entry)
7924 action_is_for_default_entry = params->action_is_for_default_entry[i];
7925 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7928 CHECK_NAME(params->default_action_name, EINVAL);
7929 for (i = 0; i < p->n_actions; i++)
7930 if (!strcmp(params->action_names[i],
7931 params->default_action_name))
7933 CHECK(i < params->n_actions, EINVAL);
7934 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7937 default_action = action_find(p, params->default_action_name);
7938 CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7941 /* Table type checks. */
7942 if (recommended_table_type_name)
7943 CHECK_NAME(recommended_table_type_name, EINVAL);
7945 if (params->n_fields) {
7946 enum rte_swx_table_match_type match_type;
7948 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7952 type = table_type_resolve(p, recommended_table_type_name, match_type);
7953 CHECK(type, EINVAL);
7958 /* Memory allocation. */
7959 t = calloc(1, sizeof(struct table));
7965 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7971 t->actions = calloc(params->n_actions, sizeof(struct action *));
7977 if (action_data_size_max) {
7978 t->default_action_data = calloc(1, action_data_size_max);
7979 if (!t->default_action_data) {
7985 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7986 if (!t->action_is_for_table_entries) {
7991 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7992 if (!t->action_is_for_default_entry) {
7997 /* Node initialization. */
7998 strcpy(t->name, name);
7999 if (args && args[0])
8000 strcpy(t->args, args);
8003 for (i = 0; i < params->n_fields; i++) {
8004 struct rte_swx_match_field_params *field = ¶ms->fields[i];
8005 struct match_field *f = &t->fields[i];
8007 f->match_type = field->match_type;
8009 header_field_parse(p, field->name, NULL) :
8010 metadata_field_parse(p, field->name);
8012 t->n_fields = params->n_fields;
8015 for (i = 0; i < params->n_actions; i++) {
8016 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8018 if (params->action_is_for_table_entries)
8019 action_is_for_table_entries = params->action_is_for_table_entries[i];
8020 if (params->action_is_for_default_entry)
8021 action_is_for_default_entry = params->action_is_for_default_entry[i];
8023 t->actions[i] = action_find(p, params->action_names[i]);
8024 t->action_is_for_table_entries[i] = action_is_for_table_entries;
8025 t->action_is_for_default_entry[i] = action_is_for_default_entry;
8027 t->default_action = default_action;
8028 if (default_action->st) {
8029 status = action_args_parse(default_action,
8030 params->default_action_args,
8031 t->default_action_data);
8036 t->n_actions = params->n_actions;
8037 t->default_action_is_const = params->default_action_is_const;
8038 t->action_data_size_max = action_data_size_max;
8041 t->id = p->n_tables;
8043 /* Node add to tailq. */
8044 TAILQ_INSERT_TAIL(&p->tables, t, node);
8053 free(t->action_is_for_default_entry);
8054 free(t->action_is_for_table_entries);
8055 free(t->default_action_data);
8063 static struct rte_swx_table_params *
8064 table_params_get(struct table *table)
8066 struct rte_swx_table_params *params;
8067 struct field *first, *last;
8069 uint32_t key_size, key_offset, action_data_size, i;
8071 /* Memory allocation. */
8072 params = calloc(1, sizeof(struct rte_swx_table_params));
8076 /* Find first (smallest offset) and last (biggest offset) match fields. */
8077 first = table->fields[0].field;
8078 last = table->fields[0].field;
8080 for (i = 0; i < table->n_fields; i++) {
8081 struct field *f = table->fields[i].field;
8083 if (f->offset < first->offset)
8086 if (f->offset > last->offset)
8090 /* Key offset and size. */
8091 key_offset = first->offset / 8;
8092 key_size = (last->offset + last->n_bits - first->offset) / 8;
8094 /* Memory allocation. */
8095 key_mask = calloc(1, key_size);
8102 for (i = 0; i < table->n_fields; i++) {
8103 struct field *f = table->fields[i].field;
8104 uint32_t start = (f->offset - first->offset) / 8;
8105 size_t size = f->n_bits / 8;
8107 memset(&key_mask[start], 0xFF, size);
8110 /* Action data size. */
8111 action_data_size = 0;
8112 for (i = 0; i < table->n_actions; i++) {
8113 struct action *action = table->actions[i];
8114 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8116 if (ads > action_data_size)
8117 action_data_size = ads;
8121 params->match_type = table->type->match_type;
8122 params->key_size = key_size;
8123 params->key_offset = key_offset;
8124 params->key_mask0 = key_mask;
8125 params->action_data_size = action_data_size;
8126 params->n_keys_max = table->size;
8132 table_params_free(struct rte_swx_table_params *params)
8137 free(params->key_mask0);
8142 table_stub_lkp(void *table __rte_unused,
8143 void *mailbox __rte_unused,
8144 uint8_t **key __rte_unused,
8145 uint64_t *action_id __rte_unused,
8146 uint8_t **action_data __rte_unused,
8150 return 1; /* DONE. */
8154 table_build(struct rte_swx_pipeline *p)
8158 /* Per pipeline: table statistics. */
8159 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8160 CHECK(p->table_stats, ENOMEM);
8162 for (i = 0; i < p->n_tables; i++) {
8163 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8164 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8167 /* Per thread: table runt-time. */
8168 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8169 struct thread *t = &p->threads[i];
8170 struct table *table;
8172 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8173 CHECK(t->tables, ENOMEM);
8175 TAILQ_FOREACH(table, &p->tables, node) {
8176 struct table_runtime *r = &t->tables[table->id];
8181 size = table->type->ops.mailbox_size_get();
8184 r->func = table->type->ops.lkp;
8188 r->mailbox = calloc(1, size);
8189 CHECK(r->mailbox, ENOMEM);
8193 r->key = table->header ?
8194 &t->structs[table->header->struct_id] :
8195 &t->structs[p->metadata_struct_id];
8197 r->func = table_stub_lkp;
8206 table_build_free(struct rte_swx_pipeline *p)
8210 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8211 struct thread *t = &p->threads[i];
8217 for (j = 0; j < p->n_tables; j++) {
8218 struct table_runtime *r = &t->tables[j];
8227 if (p->table_stats) {
8228 for (i = 0; i < p->n_tables; i++)
8229 free(p->table_stats[i].n_pkts_action);
8231 free(p->table_stats);
8236 table_free(struct rte_swx_pipeline *p)
8238 table_build_free(p);
8244 elem = TAILQ_FIRST(&p->tables);
8248 TAILQ_REMOVE(&p->tables, elem, node);
8250 free(elem->actions);
8251 free(elem->default_action_data);
8257 struct table_type *elem;
8259 elem = TAILQ_FIRST(&p->table_types);
8263 TAILQ_REMOVE(&p->table_types, elem, node);
8271 static struct selector *
8272 selector_find(struct rte_swx_pipeline *p, const char *name)
8276 TAILQ_FOREACH(s, &p->selectors, node)
8277 if (strcmp(s->name, name) == 0)
8283 static struct selector *
8284 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8286 struct selector *s = NULL;
8288 TAILQ_FOREACH(s, &p->selectors, node)
8296 selector_fields_check(struct rte_swx_pipeline *p,
8297 struct rte_swx_pipeline_selector_params *params,
8298 struct header **header)
8300 struct header *h0 = NULL;
8301 struct field *hf, *mf;
8304 /* Return if no selector fields. */
8305 if (!params->n_selector_fields || !params->selector_field_names)
8308 /* Check that all the selector fields either belong to the same header
8309 * or are all meta-data fields.
8311 hf = header_field_parse(p, params->selector_field_names[0], &h0);
8312 mf = metadata_field_parse(p, params->selector_field_names[0]);
8316 for (i = 1; i < params->n_selector_fields; i++)
8320 hf = header_field_parse(p, params->selector_field_names[i], &h);
8321 if (!hf || (h->id != h0->id))
8324 mf = metadata_field_parse(p, params->selector_field_names[i]);
8329 /* Check that there are no duplicated match fields. */
8330 for (i = 0; i < params->n_selector_fields; i++) {
8331 const char *field_name = params->selector_field_names[i];
8334 for (j = i + 1; j < params->n_selector_fields; j++)
8335 if (!strcmp(params->selector_field_names[j], field_name))
8347 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8349 struct rte_swx_pipeline_selector_params *params)
8352 struct header *selector_header = NULL;
8353 struct field *group_id_field, *member_id_field;
8359 CHECK_NAME(name, EINVAL);
8360 CHECK(!table_find(p, name), EEXIST);
8361 CHECK(!selector_find(p, name), EEXIST);
8362 CHECK(!learner_find(p, name), EEXIST);
8364 CHECK(params, EINVAL);
8366 CHECK_NAME(params->group_id_field_name, EINVAL);
8367 group_id_field = metadata_field_parse(p, params->group_id_field_name);
8368 CHECK(group_id_field, EINVAL);
8370 for (i = 0; i < params->n_selector_fields; i++) {
8371 const char *field_name = params->selector_field_names[i];
8373 CHECK_NAME(field_name, EINVAL);
8375 status = selector_fields_check(p, params, &selector_header);
8379 CHECK_NAME(params->member_id_field_name, EINVAL);
8380 member_id_field = metadata_field_parse(p, params->member_id_field_name);
8381 CHECK(member_id_field, EINVAL);
8383 CHECK(params->n_groups_max, EINVAL);
8385 CHECK(params->n_members_per_group_max, EINVAL);
8387 /* Memory allocation. */
8388 s = calloc(1, sizeof(struct selector));
8394 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8395 if (!s->selector_fields) {
8400 /* Node initialization. */
8401 strcpy(s->name, name);
8403 s->group_id_field = group_id_field;
8405 for (i = 0; i < params->n_selector_fields; i++) {
8406 const char *field_name = params->selector_field_names[i];
8408 s->selector_fields[i] = selector_header ?
8409 header_field_parse(p, field_name, NULL) :
8410 metadata_field_parse(p, field_name);
8413 s->n_selector_fields = params->n_selector_fields;
8415 s->selector_header = selector_header;
8417 s->member_id_field = member_id_field;
8419 s->n_groups_max = params->n_groups_max;
8421 s->n_members_per_group_max = params->n_members_per_group_max;
8423 s->id = p->n_selectors;
8425 /* Node add to tailq. */
8426 TAILQ_INSERT_TAIL(&p->selectors, s, node);
8435 free(s->selector_fields);
8443 selector_params_free(struct rte_swx_table_selector_params *params)
8448 free(params->selector_mask);
8453 static struct rte_swx_table_selector_params *
8454 selector_table_params_get(struct selector *s)
8456 struct rte_swx_table_selector_params *params = NULL;
8457 struct field *first, *last;
8460 /* Memory allocation. */
8461 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8466 params->group_id_offset = s->group_id_field->offset / 8;
8468 /* Find first (smallest offset) and last (biggest offset) selector fields. */
8469 first = s->selector_fields[0];
8470 last = s->selector_fields[0];
8472 for (i = 0; i < s->n_selector_fields; i++) {
8473 struct field *f = s->selector_fields[i];
8475 if (f->offset < first->offset)
8478 if (f->offset > last->offset)
8482 /* Selector offset and size. */
8483 params->selector_offset = first->offset / 8;
8484 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8486 /* Memory allocation. */
8487 params->selector_mask = calloc(1, params->selector_size);
8488 if (!params->selector_mask)
8491 /* Selector mask. */
8492 for (i = 0; i < s->n_selector_fields; i++) {
8493 struct field *f = s->selector_fields[i];
8494 uint32_t start = (f->offset - first->offset) / 8;
8495 size_t size = f->n_bits / 8;
8497 memset(¶ms->selector_mask[start], 0xFF, size);
8501 params->member_id_offset = s->member_id_field->offset / 8;
8503 /* Maximum number of groups. */
8504 params->n_groups_max = s->n_groups_max;
8506 /* Maximum number of members per group. */
8507 params->n_members_per_group_max = s->n_members_per_group_max;
8512 selector_params_free(params);
8517 selector_build_free(struct rte_swx_pipeline *p)
8521 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8522 struct thread *t = &p->threads[i];
8528 for (j = 0; j < p->n_selectors; j++) {
8529 struct selector_runtime *r = &t->selectors[j];
8535 t->selectors = NULL;
8538 free(p->selector_stats);
8539 p->selector_stats = NULL;
8543 selector_build(struct rte_swx_pipeline *p)
8548 /* Per pipeline: selector statistics. */
8549 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8550 if (!p->selector_stats) {
8555 /* Per thread: selector run-time. */
8556 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8557 struct thread *t = &p->threads[i];
8560 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8561 if (!t->selectors) {
8566 TAILQ_FOREACH(s, &p->selectors, node) {
8567 struct selector_runtime *r = &t->selectors[s->id];
8571 size = rte_swx_table_selector_mailbox_size_get();
8573 r->mailbox = calloc(1, size);
8580 /* r->group_id_buffer. */
8581 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8583 /* r->selector_buffer. */
8584 r->selector_buffer = s->selector_header ?
8585 &t->structs[s->selector_header->struct_id] :
8586 &t->structs[p->metadata_struct_id];
8588 /* r->member_id_buffer. */
8589 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8596 selector_build_free(p);
8601 selector_free(struct rte_swx_pipeline *p)
8603 selector_build_free(p);
8605 /* Selector tables. */
8607 struct selector *elem;
8609 elem = TAILQ_FIRST(&p->selectors);
8613 TAILQ_REMOVE(&p->selectors, elem, node);
8614 free(elem->selector_fields);
8622 static struct learner *
8623 learner_find(struct rte_swx_pipeline *p, const char *name)
8627 TAILQ_FOREACH(l, &p->learners, node)
8628 if (!strcmp(l->name, name))
8634 static struct learner *
8635 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8637 struct learner *l = NULL;
8639 TAILQ_FOREACH(l, &p->learners, node)
8647 learner_match_fields_check(struct rte_swx_pipeline *p,
8648 struct rte_swx_pipeline_learner_params *params,
8649 struct header **header)
8651 struct header *h0 = NULL;
8652 struct field *hf, *mf;
8655 /* Return if no match fields. */
8656 if (!params->n_fields || !params->field_names)
8659 /* Check that all the match fields either belong to the same header
8660 * or are all meta-data fields.
8662 hf = header_field_parse(p, params->field_names[0], &h0);
8663 mf = metadata_field_parse(p, params->field_names[0]);
8667 for (i = 1; i < params->n_fields; i++)
8671 hf = header_field_parse(p, params->field_names[i], &h);
8672 if (!hf || (h->id != h0->id))
8675 mf = metadata_field_parse(p, params->field_names[i]);
8680 /* Check that there are no duplicated match fields. */
8681 for (i = 0; i < params->n_fields; i++) {
8682 const char *field_name = params->field_names[i];
8685 for (j = i + 1; j < params->n_fields; j++)
8686 if (!strcmp(params->field_names[j], field_name))
8698 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8700 struct struct_type *mst = p->metadata_st, *ast = a->st;
8701 struct field *mf, *af;
8711 /* Check that mf_name is the name of a valid meta-data field. */
8712 CHECK_NAME(mf_name, EINVAL);
8713 mf = metadata_field_parse(p, mf_name);
8716 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8717 * all the action arguments.
8719 mf_pos = mf - mst->fields;
8720 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8722 /* Check that the size of each of the identified meta-data fields matches exactly the size
8723 * of the corresponding action argument.
8725 for (i = 0; i < ast->n_fields; i++) {
8726 mf = &mst->fields[mf_pos + i];
8727 af = &ast->fields[i];
8729 CHECK(mf->n_bits == af->n_bits, EINVAL);
8736 learner_action_learning_check(struct rte_swx_pipeline *p,
8737 struct action *action,
8738 const char **action_names,
8743 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8744 * the action passed as argument to the "learn" instruction) is also enabled for the
8745 * current learner table.
8747 for (i = 0; i < action->n_instructions; i++) {
8748 struct instruction *instr = &action->instructions[i];
8749 uint32_t found = 0, j;
8751 if (instr->type != INSTR_LEARNER_LEARN)
8754 for (j = 0; j < n_actions; j++) {
8757 a = action_find(p, action_names[j]);
8761 if (a->id == instr->learn.action_id)
8773 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8775 struct rte_swx_pipeline_learner_params *params,
8778 uint32_t n_timeouts)
8780 struct learner *l = NULL;
8781 struct action *default_action;
8782 struct header *header = NULL;
8783 uint32_t action_data_size_max = 0, i;
8788 CHECK_NAME(name, EINVAL);
8789 CHECK(!table_find(p, name), EEXIST);
8790 CHECK(!selector_find(p, name), EEXIST);
8791 CHECK(!learner_find(p, name), EEXIST);
8793 CHECK(params, EINVAL);
8796 status = learner_match_fields_check(p, params, &header);
8800 /* Action checks. */
8801 CHECK(params->n_actions, EINVAL);
8802 CHECK(params->action_names, EINVAL);
8803 for (i = 0; i < params->n_actions; i++) {
8804 const char *action_name = params->action_names[i];
8806 uint32_t action_data_size;
8807 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8809 CHECK_NAME(action_name, EINVAL);
8811 a = action_find(p, action_name);
8814 status = learner_action_learning_check(p,
8816 params->action_names,
8821 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8822 if (action_data_size > action_data_size_max)
8823 action_data_size_max = action_data_size;
8825 if (params->action_is_for_table_entries)
8826 action_is_for_table_entries = params->action_is_for_table_entries[i];
8827 if (params->action_is_for_default_entry)
8828 action_is_for_default_entry = params->action_is_for_default_entry[i];
8829 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8832 CHECK_NAME(params->default_action_name, EINVAL);
8833 for (i = 0; i < p->n_actions; i++)
8834 if (!strcmp(params->action_names[i],
8835 params->default_action_name))
8837 CHECK(i < params->n_actions, EINVAL);
8838 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8841 default_action = action_find(p, params->default_action_name);
8842 CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8845 /* Any other checks. */
8846 CHECK(size, EINVAL);
8847 CHECK(timeout, EINVAL);
8848 CHECK(n_timeouts && (n_timeouts <= RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX), EINVAL);
8850 /* Memory allocation. */
8851 l = calloc(1, sizeof(struct learner));
8857 l->fields = calloc(params->n_fields, sizeof(struct field *));
8863 l->actions = calloc(params->n_actions, sizeof(struct action *));
8869 if (action_data_size_max) {
8870 l->default_action_data = calloc(1, action_data_size_max);
8871 if (!l->default_action_data) {
8877 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8878 if (!l->action_is_for_table_entries) {
8883 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8884 if (!l->action_is_for_default_entry) {
8889 /* Node initialization. */
8890 strcpy(l->name, name);
8892 for (i = 0; i < params->n_fields; i++) {
8893 const char *field_name = params->field_names[i];
8895 l->fields[i] = header ?
8896 header_field_parse(p, field_name, NULL) :
8897 metadata_field_parse(p, field_name);
8900 l->n_fields = params->n_fields;
8904 for (i = 0; i < params->n_actions; i++) {
8905 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8907 if (params->action_is_for_table_entries)
8908 action_is_for_table_entries = params->action_is_for_table_entries[i];
8909 if (params->action_is_for_default_entry)
8910 action_is_for_default_entry = params->action_is_for_default_entry[i];
8912 l->actions[i] = action_find(p, params->action_names[i]);
8913 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8914 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8917 l->default_action = default_action;
8919 if (default_action->st) {
8920 status = action_args_parse(default_action,
8921 params->default_action_args,
8922 l->default_action_data);
8927 l->n_actions = params->n_actions;
8929 l->default_action_is_const = params->default_action_is_const;
8931 l->action_data_size_max = action_data_size_max;
8935 for (i = 0; i < n_timeouts; i++)
8936 l->timeout[i] = timeout[i];
8938 l->n_timeouts = n_timeouts;
8940 l->id = p->n_learners;
8942 /* Node add to tailq. */
8943 TAILQ_INSERT_TAIL(&p->learners, l, node);
8952 free(l->action_is_for_default_entry);
8953 free(l->action_is_for_table_entries);
8954 free(l->default_action_data);
8963 learner_params_free(struct rte_swx_table_learner_params *params)
8968 free(params->key_mask0);
8970 free(params->key_timeout);
8975 static struct rte_swx_table_learner_params *
8976 learner_params_get(struct learner *l)
8978 struct rte_swx_table_learner_params *params = NULL;
8979 struct field *first, *last;
8982 /* Memory allocation. */
8983 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8987 /* Find first (smallest offset) and last (biggest offset) match fields. */
8988 first = l->fields[0];
8989 last = l->fields[0];
8991 for (i = 0; i < l->n_fields; i++) {
8992 struct field *f = l->fields[i];
8994 if (f->offset < first->offset)
8997 if (f->offset > last->offset)
9001 /* Key offset and size. */
9002 params->key_offset = first->offset / 8;
9003 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
9005 /* Memory allocation. */
9006 params->key_mask0 = calloc(1, params->key_size);
9007 if (!params->key_mask0)
9011 for (i = 0; i < l->n_fields; i++) {
9012 struct field *f = l->fields[i];
9013 uint32_t start = (f->offset - first->offset) / 8;
9014 size_t size = f->n_bits / 8;
9016 memset(¶ms->key_mask0[start], 0xFF, size);
9019 /* Action data size. */
9020 params->action_data_size = l->action_data_size_max;
9022 /* Maximum number of keys. */
9023 params->n_keys_max = l->size;
9025 /* Memory allocation. */
9026 params->key_timeout = calloc(l->n_timeouts, sizeof(uint32_t));
9027 if (!params->key_timeout)
9031 for (i = 0; i < l->n_timeouts; i++)
9032 params->key_timeout[i] = l->timeout[i];
9034 params->n_key_timeouts = l->n_timeouts;
9039 learner_params_free(params);
9044 learner_build_free(struct rte_swx_pipeline *p)
9048 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9049 struct thread *t = &p->threads[i];
9055 for (j = 0; j < p->n_learners; j++) {
9056 struct learner_runtime *r = &t->learners[j];
9065 if (p->learner_stats) {
9066 for (i = 0; i < p->n_learners; i++)
9067 free(p->learner_stats[i].n_pkts_action);
9069 free(p->learner_stats);
9074 learner_build(struct rte_swx_pipeline *p)
9079 /* Per pipeline: learner statistics. */
9080 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
9081 CHECK(p->learner_stats, ENOMEM);
9083 for (i = 0; i < p->n_learners; i++) {
9084 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9085 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
9088 /* Per thread: learner run-time. */
9089 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9090 struct thread *t = &p->threads[i];
9093 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
9099 TAILQ_FOREACH(l, &p->learners, node) {
9100 struct learner_runtime *r = &t->learners[l->id];
9104 size = rte_swx_table_learner_mailbox_size_get();
9106 r->mailbox = calloc(1, size);
9114 r->key = l->header ?
9115 &t->structs[l->header->struct_id] :
9116 &t->structs[p->metadata_struct_id];
9123 learner_build_free(p);
9128 learner_free(struct rte_swx_pipeline *p)
9130 learner_build_free(p);
9132 /* Learner tables. */
9136 l = TAILQ_FIRST(&p->learners);
9140 TAILQ_REMOVE(&p->learners, l, node);
9143 free(l->default_action_data);
9152 table_state_build(struct rte_swx_pipeline *p)
9154 struct table *table;
9158 p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
9159 sizeof(struct rte_swx_table_state));
9160 CHECK(p->table_state, ENOMEM);
9162 TAILQ_FOREACH(table, &p->tables, node) {
9163 struct rte_swx_table_state *ts = &p->table_state[table->id];
9166 struct rte_swx_table_params *params;
9169 params = table_params_get(table);
9170 CHECK(params, ENOMEM);
9172 ts->obj = table->type->ops.create(params,
9177 table_params_free(params);
9178 CHECK(ts->obj, ENODEV);
9181 /* ts->default_action_data. */
9182 if (table->action_data_size_max) {
9183 ts->default_action_data =
9184 malloc(table->action_data_size_max);
9185 CHECK(ts->default_action_data, ENOMEM);
9187 memcpy(ts->default_action_data,
9188 table->default_action_data,
9189 table->action_data_size_max);
9192 /* ts->default_action_id. */
9193 ts->default_action_id = table->default_action->id;
9196 TAILQ_FOREACH(s, &p->selectors, node) {
9197 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
9198 struct rte_swx_table_selector_params *params;
9201 params = selector_table_params_get(s);
9202 CHECK(params, ENOMEM);
9204 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
9206 selector_params_free(params);
9207 CHECK(ts->obj, ENODEV);
9210 TAILQ_FOREACH(l, &p->learners, node) {
9211 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
9212 p->n_selectors + l->id];
9213 struct rte_swx_table_learner_params *params;
9216 params = learner_params_get(l);
9217 CHECK(params, ENOMEM);
9219 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
9220 learner_params_free(params);
9221 CHECK(ts->obj, ENODEV);
9223 /* ts->default_action_data. */
9224 if (l->action_data_size_max) {
9225 ts->default_action_data = malloc(l->action_data_size_max);
9226 CHECK(ts->default_action_data, ENOMEM);
9228 memcpy(ts->default_action_data,
9229 l->default_action_data,
9230 l->action_data_size_max);
9233 /* ts->default_action_id. */
9234 ts->default_action_id = l->default_action->id;
9241 table_state_build_free(struct rte_swx_pipeline *p)
9245 if (!p->table_state)
9248 for (i = 0; i < p->n_tables; i++) {
9249 struct rte_swx_table_state *ts = &p->table_state[i];
9250 struct table *table = table_find_by_id(p, i);
9253 if (table->type && ts->obj)
9254 table->type->ops.free(ts->obj);
9256 /* ts->default_action_data. */
9257 free(ts->default_action_data);
9260 for (i = 0; i < p->n_selectors; i++) {
9261 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9265 rte_swx_table_selector_free(ts->obj);
9268 for (i = 0; i < p->n_learners; i++) {
9269 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9273 rte_swx_table_learner_free(ts->obj);
9275 /* ts->default_action_data. */
9276 free(ts->default_action_data);
9279 free(p->table_state);
9280 p->table_state = NULL;
9284 table_state_free(struct rte_swx_pipeline *p)
9286 table_state_build_free(p);
9292 static struct regarray *
9293 regarray_find(struct rte_swx_pipeline *p, const char *name)
9295 struct regarray *elem;
9297 TAILQ_FOREACH(elem, &p->regarrays, node)
9298 if (!strcmp(elem->name, name))
9304 static struct regarray *
9305 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9307 struct regarray *elem = NULL;
9309 TAILQ_FOREACH(elem, &p->regarrays, node)
9317 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9326 CHECK_NAME(name, EINVAL);
9327 CHECK(!regarray_find(p, name), EEXIST);
9329 CHECK(size, EINVAL);
9330 size = rte_align32pow2(size);
9332 /* Memory allocation. */
9333 r = calloc(1, sizeof(struct regarray));
9336 /* Node initialization. */
9337 strcpy(r->name, name);
9338 r->init_val = init_val;
9340 r->id = p->n_regarrays;
9342 /* Node add to tailq. */
9343 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9350 regarray_build(struct rte_swx_pipeline *p)
9352 struct regarray *regarray;
9354 if (!p->n_regarrays)
9357 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9358 CHECK(p->regarray_runtime, ENOMEM);
9360 TAILQ_FOREACH(regarray, &p->regarrays, node) {
9361 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9364 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9365 RTE_CACHE_LINE_SIZE,
9367 CHECK(r->regarray, ENOMEM);
9369 if (regarray->init_val)
9370 for (i = 0; i < regarray->size; i++)
9371 r->regarray[i] = regarray->init_val;
9373 r->size_mask = regarray->size - 1;
9380 regarray_build_free(struct rte_swx_pipeline *p)
9384 if (!p->regarray_runtime)
9387 for (i = 0; i < p->n_regarrays; i++) {
9388 struct regarray *regarray = regarray_find_by_id(p, i);
9389 struct regarray_runtime *r = &p->regarray_runtime[i];
9391 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9394 free(p->regarray_runtime);
9395 p->regarray_runtime = NULL;
9399 regarray_free(struct rte_swx_pipeline *p)
9401 regarray_build_free(p);
9404 struct regarray *elem;
9406 elem = TAILQ_FIRST(&p->regarrays);
9410 TAILQ_REMOVE(&p->regarrays, elem, node);
9418 static struct meter_profile *
9419 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9421 struct meter_profile *elem;
9423 TAILQ_FOREACH(elem, &p->meter_profiles, node)
9424 if (!strcmp(elem->name, name))
9430 static struct metarray *
9431 metarray_find(struct rte_swx_pipeline *p, const char *name)
9433 struct metarray *elem;
9435 TAILQ_FOREACH(elem, &p->metarrays, node)
9436 if (!strcmp(elem->name, name))
9442 static struct metarray *
9443 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9445 struct metarray *elem = NULL;
9447 TAILQ_FOREACH(elem, &p->metarrays, node)
9455 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9463 CHECK_NAME(name, EINVAL);
9464 CHECK(!metarray_find(p, name), EEXIST);
9466 CHECK(size, EINVAL);
9467 size = rte_align32pow2(size);
9469 /* Memory allocation. */
9470 m = calloc(1, sizeof(struct metarray));
9473 /* Node initialization. */
9474 strcpy(m->name, name);
9476 m->id = p->n_metarrays;
9478 /* Node add to tailq. */
9479 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9485 struct meter_profile meter_profile_default = {
9494 .cir_bytes_per_period = 1,
9496 .pir_bytes_per_period = 1,
9503 meter_init(struct meter *m)
9505 memset(m, 0, sizeof(struct meter));
9506 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9507 m->profile = &meter_profile_default;
9508 m->color_mask = RTE_COLOR_GREEN;
9510 meter_profile_default.n_users++;
9514 metarray_build(struct rte_swx_pipeline *p)
9518 if (!p->n_metarrays)
9521 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9522 CHECK(p->metarray_runtime, ENOMEM);
9524 TAILQ_FOREACH(m, &p->metarrays, node) {
9525 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9528 r->metarray = env_malloc(m->size * sizeof(struct meter),
9529 RTE_CACHE_LINE_SIZE,
9531 CHECK(r->metarray, ENOMEM);
9533 for (i = 0; i < m->size; i++)
9534 meter_init(&r->metarray[i]);
9536 r->size_mask = m->size - 1;
9543 metarray_build_free(struct rte_swx_pipeline *p)
9547 if (!p->metarray_runtime)
9550 for (i = 0; i < p->n_metarrays; i++) {
9551 struct metarray *m = metarray_find_by_id(p, i);
9552 struct metarray_runtime *r = &p->metarray_runtime[i];
9554 env_free(r->metarray, m->size * sizeof(struct meter));
9557 free(p->metarray_runtime);
9558 p->metarray_runtime = NULL;
9562 metarray_free(struct rte_swx_pipeline *p)
9564 metarray_build_free(p);
9568 struct metarray *elem;
9570 elem = TAILQ_FIRST(&p->metarrays);
9574 TAILQ_REMOVE(&p->metarrays, elem, node);
9578 /* Meter profiles. */
9580 struct meter_profile *elem;
9582 elem = TAILQ_FIRST(&p->meter_profiles);
9586 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9595 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9604 free(p->instruction_data);
9605 free(p->instructions);
9609 table_state_free(p);
9614 instruction_table_free(p);
9618 extern_func_free(p);
9632 port_in_types_register(struct rte_swx_pipeline *p)
9636 status = rte_swx_pipeline_port_in_type_register(p,
9638 &rte_swx_port_ethdev_reader_ops);
9642 status = rte_swx_pipeline_port_in_type_register(p,
9644 &rte_swx_port_ring_reader_ops);
9648 #ifdef RTE_PORT_PCAP
9649 status = rte_swx_pipeline_port_in_type_register(p,
9651 &rte_swx_port_source_ops);
9656 status = rte_swx_pipeline_port_in_type_register(p,
9658 &rte_swx_port_fd_reader_ops);
9666 port_out_types_register(struct rte_swx_pipeline *p)
9670 status = rte_swx_pipeline_port_out_type_register(p,
9672 &rte_swx_port_ethdev_writer_ops);
9676 status = rte_swx_pipeline_port_out_type_register(p,
9678 &rte_swx_port_ring_writer_ops);
9682 status = rte_swx_pipeline_port_out_type_register(p,
9684 &rte_swx_port_sink_ops);
9688 status = rte_swx_pipeline_port_out_type_register(p,
9690 &rte_swx_port_fd_writer_ops);
9698 table_types_register(struct rte_swx_pipeline *p)
9702 status = rte_swx_pipeline_table_type_register(p,
9704 RTE_SWX_TABLE_MATCH_EXACT,
9705 &rte_swx_table_exact_match_ops);
9709 status = rte_swx_pipeline_table_type_register(p,
9711 RTE_SWX_TABLE_MATCH_WILDCARD,
9712 &rte_swx_table_wildcard_match_ops);
9720 hash_funcs_register(struct rte_swx_pipeline *p)
9724 status = rte_swx_pipeline_hash_func_register(p, "jhash", rte_jhash);
9728 status = rte_swx_pipeline_hash_func_register(p, "crc32", rte_hash_crc);
9736 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9738 struct rte_swx_pipeline *pipeline = NULL;
9741 /* Check input parameters. */
9744 /* Memory allocation. */
9745 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9751 /* Initialization. */
9752 TAILQ_INIT(&pipeline->struct_types);
9753 TAILQ_INIT(&pipeline->port_in_types);
9754 TAILQ_INIT(&pipeline->ports_in);
9755 TAILQ_INIT(&pipeline->port_out_types);
9756 TAILQ_INIT(&pipeline->ports_out);
9757 TAILQ_INIT(&pipeline->extern_types);
9758 TAILQ_INIT(&pipeline->extern_objs);
9759 TAILQ_INIT(&pipeline->extern_funcs);
9760 TAILQ_INIT(&pipeline->hash_funcs);
9761 TAILQ_INIT(&pipeline->headers);
9762 TAILQ_INIT(&pipeline->actions);
9763 TAILQ_INIT(&pipeline->table_types);
9764 TAILQ_INIT(&pipeline->tables);
9765 TAILQ_INIT(&pipeline->selectors);
9766 TAILQ_INIT(&pipeline->learners);
9767 TAILQ_INIT(&pipeline->regarrays);
9768 TAILQ_INIT(&pipeline->meter_profiles);
9769 TAILQ_INIT(&pipeline->metarrays);
9771 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9772 pipeline->n_mirroring_slots = RTE_SWX_PACKET_MIRRORING_SLOTS_DEFAULT;
9773 pipeline->n_mirroring_sessions = RTE_SWX_PACKET_MIRRORING_SESSIONS_DEFAULT;
9774 pipeline->numa_node = numa_node;
9776 status = port_in_types_register(pipeline);
9780 status = port_out_types_register(pipeline);
9784 status = table_types_register(pipeline);
9788 status = hash_funcs_register(pipeline);
9796 rte_swx_pipeline_free(pipeline);
9801 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9802 const char **instructions,
9803 uint32_t n_instructions)
9808 err = instruction_config(p, NULL, instructions, n_instructions);
9812 /* Thread instruction pointer reset. */
9813 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9814 struct thread *t = &p->threads[i];
9816 thread_ip_reset(p, t);
9823 pipeline_compile(struct rte_swx_pipeline *p);
9826 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9828 struct rte_swx_port_sink_params drop_port_params = {
9834 CHECK(p->build_done == 0, EEXIST);
9836 status = port_in_build(p);
9841 status = rte_swx_pipeline_port_out_config(p,
9848 status = port_out_build(p);
9852 status = mirroring_build(p);
9856 status = struct_build(p);
9860 status = extern_obj_build(p);
9864 status = extern_func_build(p);
9868 status = hash_func_build(p);
9872 status = header_build(p);
9876 status = metadata_build(p);
9880 status = instruction_table_build(p);
9884 status = action_build(p);
9888 status = table_build(p);
9892 status = selector_build(p);
9896 status = learner_build(p);
9900 status = table_state_build(p);
9904 status = regarray_build(p);
9908 status = metarray_build(p);
9914 pipeline_compile(p);
9919 metarray_build_free(p);
9920 regarray_build_free(p);
9921 table_state_build_free(p);
9922 learner_build_free(p);
9923 selector_build_free(p);
9924 table_build_free(p);
9925 action_build_free(p);
9926 instruction_table_build_free(p);
9927 metadata_build_free(p);
9928 header_build_free(p);
9929 hash_func_build_free(p);
9930 extern_func_build_free(p);
9931 extern_obj_build_free(p);
9932 mirroring_build_free(p);
9933 port_out_build_free(p);
9934 port_in_build_free(p);
9935 struct_build_free(p);
9941 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9945 for (i = 0; i < n_instructions; i++)
9950 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9954 for (i = 0; i < p->n_ports_out; i++) {
9955 struct port_out_runtime *port = &p->out[i];
9958 port->flush(port->obj);
9966 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9967 struct rte_swx_ctl_pipeline_info *pipeline)
9969 struct action *action;
9970 struct table *table;
9971 uint32_t n_actions = 0, n_tables = 0;
9973 if (!p || !pipeline)
9976 TAILQ_FOREACH(action, &p->actions, node)
9979 TAILQ_FOREACH(table, &p->tables, node)
9982 pipeline->n_ports_in = p->n_ports_in;
9983 pipeline->n_ports_out = p->n_ports_out;
9984 pipeline->n_mirroring_slots = p->n_mirroring_slots;
9985 pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9986 pipeline->n_actions = n_actions;
9987 pipeline->n_tables = n_tables;
9988 pipeline->n_selectors = p->n_selectors;
9989 pipeline->n_learners = p->n_learners;
9990 pipeline->n_regarrays = p->n_regarrays;
9991 pipeline->n_metarrays = p->n_metarrays;
9997 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9999 if (!p || !numa_node)
10002 *numa_node = p->numa_node;
10007 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10008 uint32_t action_id,
10009 struct rte_swx_ctl_action_info *action)
10011 struct action *a = NULL;
10013 if (!p || (action_id >= p->n_actions) || !action)
10016 a = action_find_by_id(p, action_id);
10020 strcpy(action->name, a->name);
10021 action->n_args = a->st ? a->st->n_fields : 0;
10026 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10027 uint32_t action_id,
10028 uint32_t action_arg_id,
10029 struct rte_swx_ctl_action_arg_info *action_arg)
10031 struct action *a = NULL;
10032 struct field *arg = NULL;
10034 if (!p || (action_id >= p->n_actions) || !action_arg)
10037 a = action_find_by_id(p, action_id);
10038 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10041 arg = &a->st->fields[action_arg_id];
10042 strcpy(action_arg->name, arg->name);
10043 action_arg->n_bits = arg->n_bits;
10044 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10050 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10052 struct rte_swx_ctl_table_info *table)
10054 struct table *t = NULL;
10059 t = table_find_by_id(p, table_id);
10063 strcpy(table->name, t->name);
10064 strcpy(table->args, t->args);
10065 table->n_match_fields = t->n_fields;
10066 table->n_actions = t->n_actions;
10067 table->default_action_is_const = t->default_action_is_const;
10068 table->size = t->size;
10073 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10075 uint32_t match_field_id,
10076 struct rte_swx_ctl_table_match_field_info *match_field)
10079 struct match_field *f;
10081 if (!p || (table_id >= p->n_tables) || !match_field)
10084 t = table_find_by_id(p, table_id);
10085 if (!t || (match_field_id >= t->n_fields))
10088 f = &t->fields[match_field_id];
10089 match_field->match_type = f->match_type;
10090 match_field->is_header = t->header ? 1 : 0;
10091 match_field->n_bits = f->field->n_bits;
10092 match_field->offset = f->field->offset;
10098 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10100 uint32_t table_action_id,
10101 struct rte_swx_ctl_table_action_info *table_action)
10105 if (!p || (table_id >= p->n_tables) || !table_action)
10108 t = table_find_by_id(p, table_id);
10109 if (!t || (table_action_id >= t->n_actions))
10112 table_action->action_id = t->actions[table_action_id]->id;
10114 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
10115 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
10121 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10123 struct rte_swx_table_ops *table_ops,
10128 if (!p || (table_id >= p->n_tables))
10131 t = table_find_by_id(p, table_id);
10137 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10147 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10148 uint32_t selector_id,
10149 struct rte_swx_ctl_selector_info *selector)
10151 struct selector *s = NULL;
10153 if (!p || !selector)
10156 s = selector_find_by_id(p, selector_id);
10160 strcpy(selector->name, s->name);
10162 selector->n_selector_fields = s->n_selector_fields;
10163 selector->n_groups_max = s->n_groups_max;
10164 selector->n_members_per_group_max = s->n_members_per_group_max;
10170 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10171 uint32_t selector_id,
10172 struct rte_swx_ctl_table_match_field_info *field)
10174 struct selector *s;
10176 if (!p || (selector_id >= p->n_selectors) || !field)
10179 s = selector_find_by_id(p, selector_id);
10183 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10184 field->is_header = 0;
10185 field->n_bits = s->group_id_field->n_bits;
10186 field->offset = s->group_id_field->offset;
10192 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10193 uint32_t selector_id,
10194 uint32_t selector_field_id,
10195 struct rte_swx_ctl_table_match_field_info *field)
10197 struct selector *s;
10200 if (!p || (selector_id >= p->n_selectors) || !field)
10203 s = selector_find_by_id(p, selector_id);
10204 if (!s || (selector_field_id >= s->n_selector_fields))
10207 f = s->selector_fields[selector_field_id];
10208 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10209 field->is_header = s->selector_header ? 1 : 0;
10210 field->n_bits = f->n_bits;
10211 field->offset = f->offset;
10217 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10218 uint32_t selector_id,
10219 struct rte_swx_ctl_table_match_field_info *field)
10221 struct selector *s;
10223 if (!p || (selector_id >= p->n_selectors) || !field)
10226 s = selector_find_by_id(p, selector_id);
10230 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10231 field->is_header = 0;
10232 field->n_bits = s->member_id_field->n_bits;
10233 field->offset = s->member_id_field->offset;
10239 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
10240 uint32_t learner_id,
10241 struct rte_swx_ctl_learner_info *learner)
10243 struct learner *l = NULL;
10245 if (!p || !learner)
10248 l = learner_find_by_id(p, learner_id);
10252 strcpy(learner->name, l->name);
10254 learner->n_match_fields = l->n_fields;
10255 learner->n_actions = l->n_actions;
10256 learner->default_action_is_const = l->default_action_is_const;
10257 learner->size = l->size;
10258 learner->n_key_timeouts = l->n_timeouts;
10264 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
10265 uint32_t learner_id,
10266 uint32_t match_field_id,
10267 struct rte_swx_ctl_table_match_field_info *match_field)
10272 if (!p || (learner_id >= p->n_learners) || !match_field)
10275 l = learner_find_by_id(p, learner_id);
10276 if (!l || (match_field_id >= l->n_fields))
10279 f = l->fields[match_field_id];
10280 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10281 match_field->is_header = l->header ? 1 : 0;
10282 match_field->n_bits = f->n_bits;
10283 match_field->offset = f->offset;
10289 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10290 uint32_t learner_id,
10291 uint32_t learner_action_id,
10292 struct rte_swx_ctl_table_action_info *learner_action)
10296 if (!p || (learner_id >= p->n_learners) || !learner_action)
10299 l = learner_find_by_id(p, learner_id);
10300 if (!l || (learner_action_id >= l->n_actions))
10303 learner_action->action_id = l->actions[learner_action_id]->id;
10305 learner_action->action_is_for_table_entries =
10306 l->action_is_for_table_entries[learner_action_id];
10308 learner_action->action_is_for_default_entry =
10309 l->action_is_for_default_entry[learner_action_id];
10315 rte_swx_ctl_pipeline_learner_timeout_get(struct rte_swx_pipeline *p,
10316 uint32_t learner_id,
10317 uint32_t timeout_id,
10322 if (!p || (learner_id >= p->n_learners) || !timeout)
10325 l = learner_find_by_id(p, learner_id);
10326 if (!l || (timeout_id >= l->n_timeouts))
10329 *timeout = l->timeout[timeout_id];
10334 rte_swx_ctl_pipeline_learner_timeout_set(struct rte_swx_pipeline *p,
10335 uint32_t learner_id,
10336 uint32_t timeout_id,
10340 struct rte_swx_table_state *ts;
10343 if (!p || (learner_id >= p->n_learners) || !timeout)
10346 l = learner_find_by_id(p, learner_id);
10347 if (!l || (timeout_id >= l->n_timeouts))
10350 if (!p->build_done)
10353 ts = &p->table_state[p->n_tables + p->n_selectors + l->id];
10355 status = rte_swx_table_learner_timeout_update(ts->obj, timeout_id, timeout);
10359 l->timeout[timeout_id] = timeout;
10365 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10366 struct rte_swx_table_state **table_state)
10368 if (!p || !table_state || !p->build_done)
10371 *table_state = p->table_state;
10376 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10377 struct rte_swx_table_state *table_state)
10379 if (!p || !table_state || !p->build_done)
10382 p->table_state = table_state;
10387 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10389 struct rte_swx_port_in_stats *stats)
10391 struct port_in *port;
10396 port = port_in_find(p, port_id);
10400 port->type->ops.stats_read(port->obj, stats);
10405 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10407 struct rte_swx_port_out_stats *stats)
10409 struct port_out *port;
10414 port = port_out_find(p, port_id);
10418 port->type->ops.stats_read(port->obj, stats);
10423 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10424 const char *table_name,
10425 struct rte_swx_table_stats *stats)
10427 struct table *table;
10428 struct table_statistics *table_stats;
10430 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10433 table = table_find(p, table_name);
10437 table_stats = &p->table_stats[table->id];
10439 memcpy(stats->n_pkts_action,
10440 table_stats->n_pkts_action,
10441 p->n_actions * sizeof(uint64_t));
10443 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10444 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10450 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10451 const char *selector_name,
10452 struct rte_swx_pipeline_selector_stats *stats)
10454 struct selector *s;
10456 if (!p || !selector_name || !selector_name[0] || !stats)
10459 s = selector_find(p, selector_name);
10463 stats->n_pkts = p->selector_stats[s->id].n_pkts;
10469 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10470 const char *learner_name,
10471 struct rte_swx_learner_stats *stats)
10474 struct learner_statistics *learner_stats;
10476 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10479 l = learner_find(p, learner_name);
10483 learner_stats = &p->learner_stats[l->id];
10485 memcpy(stats->n_pkts_action,
10486 learner_stats->n_pkts_action,
10487 p->n_actions * sizeof(uint64_t));
10489 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10490 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10492 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10493 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10495 stats->n_pkts_rearm = learner_stats->n_pkts_rearm;
10496 stats->n_pkts_forget = learner_stats->n_pkts_forget;
10502 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10503 uint32_t regarray_id,
10504 struct rte_swx_ctl_regarray_info *regarray)
10506 struct regarray *r;
10508 if (!p || !regarray)
10511 r = regarray_find_by_id(p, regarray_id);
10515 strcpy(regarray->name, r->name);
10516 regarray->size = r->size;
10521 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10522 const char *regarray_name,
10523 uint32_t regarray_index,
10526 struct regarray *regarray;
10527 struct regarray_runtime *r;
10529 if (!p || !regarray_name || !value)
10532 regarray = regarray_find(p, regarray_name);
10533 if (!regarray || (regarray_index >= regarray->size))
10536 r = &p->regarray_runtime[regarray->id];
10537 *value = r->regarray[regarray_index];
10542 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10543 const char *regarray_name,
10544 uint32_t regarray_index,
10547 struct regarray *regarray;
10548 struct regarray_runtime *r;
10550 if (!p || !regarray_name)
10553 regarray = regarray_find(p, regarray_name);
10554 if (!regarray || (regarray_index >= regarray->size))
10557 r = &p->regarray_runtime[regarray->id];
10558 r->regarray[regarray_index] = value;
10563 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10564 uint32_t metarray_id,
10565 struct rte_swx_ctl_metarray_info *metarray)
10567 struct metarray *m;
10569 if (!p || !metarray)
10572 m = metarray_find_by_id(p, metarray_id);
10576 strcpy(metarray->name, m->name);
10577 metarray->size = m->size;
10582 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10584 struct rte_meter_trtcm_params *params)
10586 struct meter_profile *mp;
10590 CHECK_NAME(name, EINVAL);
10591 CHECK(params, EINVAL);
10592 CHECK(!meter_profile_find(p, name), EEXIST);
10594 /* Node allocation. */
10595 mp = calloc(1, sizeof(struct meter_profile));
10598 /* Node initialization. */
10599 strcpy(mp->name, name);
10600 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10601 status = rte_meter_trtcm_profile_config(&mp->profile, params);
10607 /* Node add to tailq. */
10608 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10614 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10617 struct meter_profile *mp;
10620 CHECK_NAME(name, EINVAL);
10622 mp = meter_profile_find(p, name);
10624 CHECK(!mp->n_users, EBUSY);
10626 /* Remove node from tailq. */
10627 TAILQ_REMOVE(&p->meter_profiles, mp, node);
10634 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10635 const char *metarray_name,
10636 uint32_t metarray_index)
10638 struct meter_profile *mp_old;
10639 struct metarray *metarray;
10640 struct metarray_runtime *metarray_runtime;
10644 CHECK_NAME(metarray_name, EINVAL);
10646 metarray = metarray_find(p, metarray_name);
10647 CHECK(metarray, EINVAL);
10648 CHECK(metarray_index < metarray->size, EINVAL);
10650 metarray_runtime = &p->metarray_runtime[metarray->id];
10651 m = &metarray_runtime->metarray[metarray_index];
10652 mp_old = m->profile;
10662 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10663 const char *metarray_name,
10664 uint32_t metarray_index,
10665 const char *profile_name)
10667 struct meter_profile *mp, *mp_old;
10668 struct metarray *metarray;
10669 struct metarray_runtime *metarray_runtime;
10673 CHECK_NAME(metarray_name, EINVAL);
10675 metarray = metarray_find(p, metarray_name);
10676 CHECK(metarray, EINVAL);
10677 CHECK(metarray_index < metarray->size, EINVAL);
10679 mp = meter_profile_find(p, profile_name);
10682 metarray_runtime = &p->metarray_runtime[metarray->id];
10683 m = &metarray_runtime->metarray[metarray_index];
10684 mp_old = m->profile;
10686 memset(m, 0, sizeof(struct meter));
10687 rte_meter_trtcm_config(&m->m, &mp->profile);
10689 m->color_mask = RTE_COLORS;
10698 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10699 const char *metarray_name,
10700 uint32_t metarray_index,
10701 struct rte_swx_ctl_meter_stats *stats)
10703 struct metarray *metarray;
10704 struct metarray_runtime *metarray_runtime;
10708 CHECK_NAME(metarray_name, EINVAL);
10710 metarray = metarray_find(p, metarray_name);
10711 CHECK(metarray, EINVAL);
10712 CHECK(metarray_index < metarray->size, EINVAL);
10714 CHECK(stats, EINVAL);
10716 metarray_runtime = &p->metarray_runtime[metarray->id];
10717 m = &metarray_runtime->metarray[metarray_index];
10719 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10720 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10726 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10727 uint32_t session_id,
10728 struct rte_swx_pipeline_mirroring_session_params *params)
10730 struct mirroring_session *s;
10733 CHECK(p->build_done, EEXIST);
10734 CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10735 CHECK(params, EINVAL);
10736 CHECK(params->port_id < p->n_ports_out, EINVAL);
10738 s = &p->mirroring_sessions[session_id];
10739 s->port_id = params->port_id;
10740 s->fast_clone = params->fast_clone;
10741 s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10747 * Pipeline compilation.
10749 static const char *
10750 instr_type_to_name(struct instruction *instr)
10752 switch (instr->type) {
10753 case INSTR_RX: return "INSTR_RX";
10755 case INSTR_TX: return "INSTR_TX";
10756 case INSTR_TX_I: return "INSTR_TX_I";
10757 case INSTR_DROP: return "INSTR_DROP";
10758 case INSTR_MIRROR: return "INSTR_MIRROR";
10759 case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10760 case INSTR_RECIRCID: return "INSTR_RECIRCID";
10762 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10763 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10764 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10765 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10766 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10767 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10768 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10769 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10771 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10773 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10775 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10776 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10777 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10778 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10779 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10780 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10781 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10782 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10783 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10785 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10786 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10788 case INSTR_MOV: return "INSTR_MOV";
10789 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10790 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10791 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10792 case INSTR_MOV_I: return "INSTR_MOV_I";
10794 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10795 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10796 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10797 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10798 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10799 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10800 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10801 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10803 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10804 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10805 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10806 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10807 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10808 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10810 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10811 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10812 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10813 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10814 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10815 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10817 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10818 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10819 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10820 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10822 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10823 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10824 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10825 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10826 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10828 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10829 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10830 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10831 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10832 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10834 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10835 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10836 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10837 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10838 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10840 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10841 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10842 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10843 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10844 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10845 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10847 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10848 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10849 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10850 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10851 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10852 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10854 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10855 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10856 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10858 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10859 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10860 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10861 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10862 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10863 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10865 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10866 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10867 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10868 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10869 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10870 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10871 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10872 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10873 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10875 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10876 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10877 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10878 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10879 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10880 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10881 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10882 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10883 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10885 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10886 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10887 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10889 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10890 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10891 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10892 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10893 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10894 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10895 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10896 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10897 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10898 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10899 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10900 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10902 case INSTR_TABLE: return "INSTR_TABLE";
10903 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10904 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10905 case INSTR_LEARNER: return "INSTR_LEARNER";
10906 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10908 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10909 case INSTR_LEARNER_REARM: return "INSTR_LEARNER_REARM";
10910 case INSTR_LEARNER_REARM_NEW: return "INSTR_LEARNER_REARM_NEW";
10911 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10913 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10914 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10915 case INSTR_HASH_FUNC: return "INSTR_HASH_FUNC";
10917 case INSTR_JMP: return "INSTR_JMP";
10918 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10919 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10920 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10921 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10922 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10923 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10924 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10925 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10926 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10927 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10928 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10929 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10930 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10931 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10932 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10933 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10934 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10935 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10936 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10937 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10938 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10939 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10940 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10941 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10942 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10943 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10944 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10945 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10947 case INSTR_RETURN: return "INSTR_RETURN";
10949 default: return "INSTR_UNKNOWN";
10954 (*instruction_export_t)(struct instruction *, FILE *);
10957 instr_io_export(struct instruction *instr, FILE *f)
10959 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10961 /* n_io, n_io_imm, n_hdrs. */
10962 if (instr->type == INSTR_RX ||
10963 instr->type == INSTR_TX ||
10964 instr->type == INSTR_HDR_EXTRACT_M ||
10965 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10968 if (instr->type == INSTR_TX_I)
10971 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10972 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10974 if (instr->type == INSTR_HDR_EXTRACT_M ||
10975 instr->type == INSTR_HDR_LOOKAHEAD ||
10976 instr->type == INSTR_HDR_EMIT)
10979 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10980 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10985 "\t\t.type = %s,\n",
10986 instr_type_to_name(instr));
10989 if (n_io || n_io_imm || n_hdrs)
10997 "\t\t\t\t.offset = %u,\n"
10998 "\t\t\t\t.n_bits = %u,\n"
11000 instr->io.io.offset,
11001 instr->io.io.n_bits);
11006 "\t\t\t\t.val = %u,\n"
11010 /* instr.io.hdr. */
11015 /* instr.io.hdr.header_id. */
11017 "\t\t\t.header_id = {");
11019 for (i = 0; i < n_hdrs; i++)
11022 instr->io.hdr.header_id[i]);
11027 /* instr.io.hdr.struct_id. */
11029 "\t\t\t.struct_id = {");
11031 for (i = 0; i < n_hdrs; i++)
11034 instr->io.hdr.struct_id[i]);
11039 /* instr.io.hdr.n_bytes. */
11041 "\t\t\t.n_bytes = {");
11043 for (i = 0; i < n_hdrs; i++)
11046 instr->io.hdr.n_bytes[i]);
11051 /* instr.io.hdr - closing curly brace. */
11056 /* instr.io - closing curly brace. */
11057 if (n_io || n_io_imm || n_hdrs)
11061 /* instr - closing curly brace. */
11067 instr_mirror_export(struct instruction *instr, FILE *f)
11071 "\t\t.type = %s,\n"
11072 "\t\t.mirror = {\n"
11074 "\t\t\t\t.struct_id = %u,\n"
11075 "\t\t\t\t.n_bits = %u,\n"
11076 "\t\t\t\t.offset = %u,\n"
11079 "\t\t\t\t.struct_id = %u,\n"
11080 "\t\t\t\t.n_bits = %u,\n"
11081 "\t\t\t\t.offset = %u,\n"
11085 instr_type_to_name(instr),
11086 instr->mirror.dst.struct_id,
11087 instr->mirror.dst.n_bits,
11088 instr->mirror.dst.offset,
11089 instr->mirror.src.struct_id,
11090 instr->mirror.src.n_bits,
11091 instr->mirror.src.offset);
11095 instr_recirculate_export(struct instruction *instr, FILE *f)
11099 "\t\t.type = %s,\n"
11101 instr_type_to_name(instr));
11105 instr_recircid_export(struct instruction *instr, FILE *f)
11109 "\t\t.type = %s,\n"
11111 "\t\t\t.offset = %u,\n"
11112 "\t\t\t.n_bits = %u,\n"
11115 instr_type_to_name(instr),
11116 instr->io.io.offset,
11117 instr->io.io.n_bits);
11121 instr_hdr_validate_export(struct instruction *instr, FILE *f)
11125 "\t\t.type = %s,\n"
11127 "\t\t\t.header_id = %u,\n"
11130 instr_type_to_name(instr),
11131 instr->valid.header_id);
11135 instr_mov_export(struct instruction *instr, FILE *f)
11137 if (instr->type != INSTR_MOV_I)
11140 "\t\t.type = %s,\n"
11143 "\t\t\t\t.struct_id = %u,\n"
11144 "\t\t\t\t.n_bits = %u,\n"
11145 "\t\t\t\t.offset = %u,\n"
11148 "\t\t\t\t.struct_id = %u,\n"
11149 "\t\t\t\t.n_bits = %u,\n"
11150 "\t\t\t\t.offset = %u,\n"
11154 instr_type_to_name(instr),
11155 instr->mov.dst.struct_id,
11156 instr->mov.dst.n_bits,
11157 instr->mov.dst.offset,
11158 instr->mov.src.struct_id,
11159 instr->mov.src.n_bits,
11160 instr->mov.src.offset);
11164 "\t\t.type = %s,\n"
11167 "\t\t\t\t.struct_id = %u,\n"
11168 "\t\t\t\t.n_bits = %u,\n"
11169 "\t\t\t\t.offset = %u,\n"
11171 "\t\t\t.src_val = %" PRIu64 ",\n"
11174 instr_type_to_name(instr),
11175 instr->mov.dst.struct_id,
11176 instr->mov.dst.n_bits,
11177 instr->mov.dst.offset,
11178 instr->mov.src_val);
11182 instr_dma_ht_export(struct instruction *instr, FILE *f)
11184 uint32_t n_dma = 0, i;
11187 n_dma = 1 + (instr->type - INSTR_DMA_HT);
11192 "\t\t.type = %s,\n",
11193 instr_type_to_name(instr));
11199 /* instr.dma.dst. */
11201 "\t\t\t.dst = {\n");
11203 /* instr.dma.dst.header_id. */
11205 "\t\t\t\t.header_id = {");
11207 for (i = 0; i < n_dma; i++)
11210 instr->dma.dst.header_id[i]);
11215 /* instr.dma.dst.struct_id. */
11217 "\t\t\t\t.struct_id = {");
11219 for (i = 0; i < n_dma; i++)
11222 instr->dma.dst.struct_id[i]);
11227 /* instr.dma.dst - closing curly brace. */
11231 /* instr.dma.src. */
11233 "\t\t\t.src = {\n");
11235 /* instr.dma.src.offset. */
11237 "\t\t\t\t.offset = {");
11239 for (i = 0; i < n_dma; i++)
11242 instr->dma.src.offset[i]);
11247 /* instr.dma.src - closing curly brace. */
11251 /* instr.dma.n_bytes. */
11253 "\t\t\t.n_bytes = {");
11255 for (i = 0; i < n_dma; i++)
11258 instr->dma.n_bytes[i]);
11263 /* instr.dma - closing curly brace. */
11267 /* instr - closing curly brace. */
11273 instr_alu_export(struct instruction *instr, FILE *f)
11277 if (instr->type == INSTR_ALU_ADD_MI ||
11278 instr->type == INSTR_ALU_ADD_HI ||
11279 instr->type == INSTR_ALU_SUB_MI ||
11280 instr->type == INSTR_ALU_SUB_HI ||
11281 instr->type == INSTR_ALU_SHL_MI ||
11282 instr->type == INSTR_ALU_SHL_HI ||
11283 instr->type == INSTR_ALU_SHR_MI ||
11284 instr->type == INSTR_ALU_SHR_HI ||
11285 instr->type == INSTR_ALU_AND_I ||
11286 instr->type == INSTR_ALU_OR_I ||
11287 instr->type == INSTR_ALU_XOR_I)
11293 "\t\t.type = %s,\n"
11296 "\t\t\t\t.struct_id = %u,\n"
11297 "\t\t\t\t.n_bits = %u,\n"
11298 "\t\t\t\t.offset = %u,\n"
11301 "\t\t\t\t.struct_id = %u,\n"
11302 "\t\t\t\t.n_bits = %u,\n"
11303 "\t\t\t\t.offset = %u,\n"
11307 instr_type_to_name(instr),
11308 instr->alu.dst.struct_id,
11309 instr->alu.dst.n_bits,
11310 instr->alu.dst.offset,
11311 instr->alu.src.struct_id,
11312 instr->alu.src.n_bits,
11313 instr->alu.src.offset);
11317 "\t\t.type = %s,\n"
11320 "\t\t\t\t.struct_id = %u,\n"
11321 "\t\t\t\t.n_bits = %u,\n"
11322 "\t\t\t\t.offset = %u,\n"
11324 "\t\t\t.src_val = %" PRIu64 ",\n"
11327 instr_type_to_name(instr),
11328 instr->alu.dst.struct_id,
11329 instr->alu.dst.n_bits,
11330 instr->alu.dst.offset,
11331 instr->alu.src_val);
11335 instr_hash_export(struct instruction *instr, FILE *f)
11339 "\t\t.type = %s,\n"
11340 "\t\t.hash_func = {\n"
11341 "\t\t\t.hash_func_id = %u,\n"
11343 "\t\t\t\t.offset = %u,\n"
11344 "\t\t\t\t.n_bits = %u,\n"
11347 "\t\t\t\t.struct_id = %u,\n"
11348 "\t\t\t\t.offset = %u,\n"
11349 "\t\t\t\t.n_bytes = %u,\n"
11353 instr_type_to_name(instr),
11354 instr->hash_func.hash_func_id,
11355 instr->hash_func.dst.offset,
11356 instr->hash_func.dst.n_bits,
11357 instr->hash_func.src.struct_id,
11358 instr->hash_func.src.offset,
11359 instr->hash_func.src.n_bytes);
11363 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11365 int prefetch = 0, idx_imm = 0, src_imm = 0;
11367 if (instr->type == INSTR_REGPREFETCH_RH ||
11368 instr->type == INSTR_REGPREFETCH_RM ||
11369 instr->type == INSTR_REGPREFETCH_RI)
11372 /* index is the 3rd operand for the regrd instruction and the 2nd
11373 * operand for the regwr and regadd instructions.
11375 if (instr->type == INSTR_REGPREFETCH_RI ||
11376 instr->type == INSTR_REGRD_HRI ||
11377 instr->type == INSTR_REGRD_MRI ||
11378 instr->type == INSTR_REGWR_RIH ||
11379 instr->type == INSTR_REGWR_RIM ||
11380 instr->type == INSTR_REGWR_RII ||
11381 instr->type == INSTR_REGADD_RIH ||
11382 instr->type == INSTR_REGADD_RIM ||
11383 instr->type == INSTR_REGADD_RII)
11386 /* src is the 3rd operand for the regwr and regadd instructions. */
11387 if (instr->type == INSTR_REGWR_RHI ||
11388 instr->type == INSTR_REGWR_RMI ||
11389 instr->type == INSTR_REGWR_RII ||
11390 instr->type == INSTR_REGADD_RHI ||
11391 instr->type == INSTR_REGADD_RMI ||
11392 instr->type == INSTR_REGADD_RII)
11395 /* instr.regarray.regarray_id. */
11398 "\t\t.type = %s,\n"
11399 "\t\t.regarray = {\n"
11400 "\t\t\t.regarray_id = %u,\n",
11401 instr_type_to_name(instr),
11402 instr->regarray.regarray_id);
11404 /* instr.regarray.idx / instr.regarray.idx_val. */
11407 "\t\t\t\t.idx = {\n"
11408 "\t\t\t\t\t.struct_id = %u,\n"
11409 "\t\t\t\t\t.n_bits = %u,\n"
11410 "\t\t\t\t\t.offset = %u,\n"
11412 instr->regarray.idx.struct_id,
11413 instr->regarray.idx.n_bits,
11414 instr->regarray.idx.offset);
11417 "\t\t\t\t.idx_val = %u,\n",
11418 instr->regarray.idx_val);
11420 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11424 "\t\t\t\t.dstsrc = {\n"
11425 "\t\t\t\t\t.struct_id = %u,\n"
11426 "\t\t\t\t\t.n_bits = %u,\n"
11427 "\t\t\t\t\t.offset = %u,\n"
11429 instr->regarray.dstsrc.struct_id,
11430 instr->regarray.dstsrc.n_bits,
11431 instr->regarray.dstsrc.offset);
11434 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11435 instr->regarray.dstsrc_val);
11438 /* instr.regarray and instr - closing curly braces. */
11445 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11447 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
11449 if (instr->type == INSTR_METPREFETCH_H ||
11450 instr->type == INSTR_METPREFETCH_M ||
11451 instr->type == INSTR_METPREFETCH_I)
11455 if (instr->type == INSTR_METPREFETCH_I ||
11456 instr->type == INSTR_METER_IHM ||
11457 instr->type == INSTR_METER_IHI ||
11458 instr->type == INSTR_METER_IMM ||
11459 instr->type == INSTR_METER_IMI)
11462 /* color_in_imm. */
11463 if (instr->type == INSTR_METER_HHI ||
11464 instr->type == INSTR_METER_HMI ||
11465 instr->type == INSTR_METER_MHI ||
11466 instr->type == INSTR_METER_MMI ||
11467 instr->type == INSTR_METER_IHI ||
11468 instr->type == INSTR_METER_IMI)
11471 /* instr.meter.metarray_id. */
11474 "\t\t.type = %s,\n"
11476 "\t\t\t.metarray_id = %u,\n",
11477 instr_type_to_name(instr),
11478 instr->meter.metarray_id);
11480 /* instr.meter.idx / instr.meter.idx_val. */
11484 "\t\t\t\t.struct_id = %u,\n"
11485 "\t\t\t\t.n_bits = %u,\n"
11486 "\t\t\t\t.offset = %u,\n"
11488 instr->meter.idx.struct_id,
11489 instr->meter.idx.n_bits,
11490 instr->meter.idx.offset);
11493 "\t\t\t.idx_val = %u,\n",
11494 instr->meter.idx_val);
11497 /* instr.meter.length. */
11499 "\t\t\t.length = {\n"
11500 "\t\t\t\t.struct_id = %u,\n"
11501 "\t\t\t\t.n_bits = %u,\n"
11502 "\t\t\t\t.offset = %u,\n"
11504 instr->meter.length.struct_id,
11505 instr->meter.length.n_bits,
11506 instr->meter.length.offset);
11508 /* instr.meter.color_in / instr.meter.color_in_val. */
11511 "\t\t\t.color_in = {\n"
11512 "\t\t\t\t.struct_id = %u,\n"
11513 "\t\t\t\t.n_bits = %u,\n"
11514 "\t\t\t\t.offset = %u,\n"
11516 instr->meter.color_in.struct_id,
11517 instr->meter.color_in.n_bits,
11518 instr->meter.color_in.offset);
11521 "\t\t\t.color_in_val = %u,\n",
11522 (uint32_t)instr->meter.color_in_val);
11524 /* instr.meter.color_out. */
11526 "\t\t\t.color_out = {\n"
11527 "\t\t\t\t.struct_id = %u,\n"
11528 "\t\t\t\t.n_bits = %u,\n"
11529 "\t\t\t\t.offset = %u,\n"
11531 instr->meter.color_out.struct_id,
11532 instr->meter.color_out.n_bits,
11533 instr->meter.color_out.offset);
11536 /* instr.meter and instr - closing curly braces. */
11543 instr_table_export(struct instruction *instr,
11548 "\t\t.type = %s,\n"
11550 "\t\t\t.table_id = %u,\n"
11553 instr_type_to_name(instr),
11554 instr->table.table_id);
11558 instr_learn_export(struct instruction *instr, FILE *f)
11562 "\t\t.type = %s,\n"
11564 "\t\t\t.action_id = %u,\n"
11565 "\t\t\t.mf_first_arg_offset = %u,\n"
11566 "\t\t\t.mf_timeout_id_offset = %u,\n"
11567 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11570 instr_type_to_name(instr),
11571 instr->learn.action_id,
11572 instr->learn.mf_first_arg_offset,
11573 instr->learn.mf_timeout_id_offset,
11574 instr->learn.mf_timeout_id_n_bits);
11578 instr_rearm_export(struct instruction *instr, FILE *f)
11580 if (instr->type == INSTR_LEARNER_REARM)
11583 "\t\t.type = %s,\n"
11585 instr_type_to_name(instr));
11589 "\t\t.type = %s,\n"
11591 "\t\t\t.mf_timeout_id_offset = %u,\n"
11592 "\t\t\t.mf_timeout_id_n_bits = %u,\n"
11595 instr_type_to_name(instr),
11596 instr->learn.mf_timeout_id_offset,
11597 instr->learn.mf_timeout_id_n_bits);
11601 instr_forget_export(struct instruction *instr, FILE *f)
11605 "\t\t.type = %s,\n"
11607 instr_type_to_name(instr));
11611 instr_extern_export(struct instruction *instr, FILE *f)
11613 if (instr->type == INSTR_EXTERN_OBJ)
11616 "\t\t.type = %s,\n"
11617 "\t\t.ext_obj = {\n"
11618 "\t\t\t.ext_obj_id = %u,\n"
11619 "\t\t\t.func_id = %u,\n"
11622 instr_type_to_name(instr),
11623 instr->ext_obj.ext_obj_id,
11624 instr->ext_obj.func_id);
11628 "\t\t.type = %s,\n"
11629 "\t\t.ext_func = {\n"
11630 "\t\t\t.ext_func_id = %u,\n"
11633 instr_type_to_name(instr),
11634 instr->ext_func.ext_func_id);
11638 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11642 "\t\t.type = %s,\n"
11644 "\t\t\t.ip = NULL,\n",
11645 instr_type_to_name(instr));
11647 switch (instr->type) {
11648 case INSTR_JMP_VALID:
11649 case INSTR_JMP_INVALID:
11651 "\t\t\t.header_id = %u,\n",
11652 instr->jmp.header_id);
11655 case INSTR_JMP_ACTION_HIT:
11656 case INSTR_JMP_ACTION_MISS:
11658 "\t\t\t.action_id = %u,\n",
11659 instr->jmp.action_id);
11663 case INSTR_JMP_EQ_MH:
11664 case INSTR_JMP_EQ_HM:
11665 case INSTR_JMP_EQ_HH:
11666 case INSTR_JMP_NEQ:
11667 case INSTR_JMP_NEQ_MH:
11668 case INSTR_JMP_NEQ_HM:
11669 case INSTR_JMP_NEQ_HH:
11671 case INSTR_JMP_LT_MH:
11672 case INSTR_JMP_LT_HM:
11673 case INSTR_JMP_LT_HH:
11675 case INSTR_JMP_GT_MH:
11676 case INSTR_JMP_GT_HM:
11677 case INSTR_JMP_GT_HH:
11680 "\t\t\t\t.struct_id = %u,\n"
11681 "\t\t\t\t.n_bits = %u,\n"
11682 "\t\t\t\t.offset = %u,\n"
11685 "\t\t\t\t.struct_id = %u,\n"
11686 "\t\t\t\t.n_bits = %u,\n"
11687 "\t\t\t\t.offset = %u,\n"
11689 instr->jmp.a.struct_id,
11690 instr->jmp.a.n_bits,
11691 instr->jmp.a.offset,
11692 instr->jmp.b.struct_id,
11693 instr->jmp.b.n_bits,
11694 instr->jmp.b.offset);
11697 case INSTR_JMP_EQ_I:
11698 case INSTR_JMP_NEQ_I:
11699 case INSTR_JMP_LT_MI:
11700 case INSTR_JMP_LT_HI:
11701 case INSTR_JMP_GT_MI:
11702 case INSTR_JMP_GT_HI:
11705 "\t\t\t\t.struct_id = %u,\n"
11706 "\t\t\t\t.n_bits = %u,\n"
11707 "\t\t\t\t.offset = %u,\n"
11709 "\t\t\t.b_val = %" PRIu64 ",\n",
11710 instr->jmp.a.struct_id,
11711 instr->jmp.a.n_bits,
11712 instr->jmp.a.offset,
11726 instr_return_export(struct instruction *instr,
11731 "\t\t.type = %s,\n",
11732 instr_type_to_name(instr));
11738 static instruction_export_t export_table[] = {
11739 [INSTR_RX] = instr_io_export,
11741 [INSTR_TX] = instr_io_export,
11742 [INSTR_TX_I] = instr_io_export,
11743 [INSTR_DROP] = instr_io_export,
11744 [INSTR_MIRROR] = instr_mirror_export,
11745 [INSTR_RECIRCULATE] = instr_recirculate_export,
11746 [INSTR_RECIRCID] = instr_recircid_export,
11748 [INSTR_HDR_EXTRACT] = instr_io_export,
11749 [INSTR_HDR_EXTRACT2] = instr_io_export,
11750 [INSTR_HDR_EXTRACT3] = instr_io_export,
11751 [INSTR_HDR_EXTRACT4] = instr_io_export,
11752 [INSTR_HDR_EXTRACT5] = instr_io_export,
11753 [INSTR_HDR_EXTRACT6] = instr_io_export,
11754 [INSTR_HDR_EXTRACT7] = instr_io_export,
11755 [INSTR_HDR_EXTRACT8] = instr_io_export,
11757 [INSTR_HDR_EXTRACT_M] = instr_io_export,
11759 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11761 [INSTR_HDR_EMIT] = instr_io_export,
11762 [INSTR_HDR_EMIT_TX] = instr_io_export,
11763 [INSTR_HDR_EMIT2_TX] = instr_io_export,
11764 [INSTR_HDR_EMIT3_TX] = instr_io_export,
11765 [INSTR_HDR_EMIT4_TX] = instr_io_export,
11766 [INSTR_HDR_EMIT5_TX] = instr_io_export,
11767 [INSTR_HDR_EMIT6_TX] = instr_io_export,
11768 [INSTR_HDR_EMIT7_TX] = instr_io_export,
11769 [INSTR_HDR_EMIT8_TX] = instr_io_export,
11771 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11772 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11774 [INSTR_MOV] = instr_mov_export,
11775 [INSTR_MOV_MH] = instr_mov_export,
11776 [INSTR_MOV_HM] = instr_mov_export,
11777 [INSTR_MOV_HH] = instr_mov_export,
11778 [INSTR_MOV_I] = instr_mov_export,
11780 [INSTR_DMA_HT] = instr_dma_ht_export,
11781 [INSTR_DMA_HT2] = instr_dma_ht_export,
11782 [INSTR_DMA_HT3] = instr_dma_ht_export,
11783 [INSTR_DMA_HT4] = instr_dma_ht_export,
11784 [INSTR_DMA_HT5] = instr_dma_ht_export,
11785 [INSTR_DMA_HT6] = instr_dma_ht_export,
11786 [INSTR_DMA_HT7] = instr_dma_ht_export,
11787 [INSTR_DMA_HT8] = instr_dma_ht_export,
11789 [INSTR_ALU_ADD] = instr_alu_export,
11790 [INSTR_ALU_ADD_MH] = instr_alu_export,
11791 [INSTR_ALU_ADD_HM] = instr_alu_export,
11792 [INSTR_ALU_ADD_HH] = instr_alu_export,
11793 [INSTR_ALU_ADD_MI] = instr_alu_export,
11794 [INSTR_ALU_ADD_HI] = instr_alu_export,
11796 [INSTR_ALU_SUB] = instr_alu_export,
11797 [INSTR_ALU_SUB_MH] = instr_alu_export,
11798 [INSTR_ALU_SUB_HM] = instr_alu_export,
11799 [INSTR_ALU_SUB_HH] = instr_alu_export,
11800 [INSTR_ALU_SUB_MI] = instr_alu_export,
11801 [INSTR_ALU_SUB_HI] = instr_alu_export,
11803 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11804 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11805 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11806 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11808 [INSTR_ALU_AND] = instr_alu_export,
11809 [INSTR_ALU_AND_MH] = instr_alu_export,
11810 [INSTR_ALU_AND_HM] = instr_alu_export,
11811 [INSTR_ALU_AND_HH] = instr_alu_export,
11812 [INSTR_ALU_AND_I] = instr_alu_export,
11814 [INSTR_ALU_OR] = instr_alu_export,
11815 [INSTR_ALU_OR_MH] = instr_alu_export,
11816 [INSTR_ALU_OR_HM] = instr_alu_export,
11817 [INSTR_ALU_OR_HH] = instr_alu_export,
11818 [INSTR_ALU_OR_I] = instr_alu_export,
11820 [INSTR_ALU_XOR] = instr_alu_export,
11821 [INSTR_ALU_XOR_MH] = instr_alu_export,
11822 [INSTR_ALU_XOR_HM] = instr_alu_export,
11823 [INSTR_ALU_XOR_HH] = instr_alu_export,
11824 [INSTR_ALU_XOR_I] = instr_alu_export,
11826 [INSTR_ALU_SHL] = instr_alu_export,
11827 [INSTR_ALU_SHL_MH] = instr_alu_export,
11828 [INSTR_ALU_SHL_HM] = instr_alu_export,
11829 [INSTR_ALU_SHL_HH] = instr_alu_export,
11830 [INSTR_ALU_SHL_MI] = instr_alu_export,
11831 [INSTR_ALU_SHL_HI] = instr_alu_export,
11833 [INSTR_ALU_SHR] = instr_alu_export,
11834 [INSTR_ALU_SHR_MH] = instr_alu_export,
11835 [INSTR_ALU_SHR_HM] = instr_alu_export,
11836 [INSTR_ALU_SHR_HH] = instr_alu_export,
11837 [INSTR_ALU_SHR_MI] = instr_alu_export,
11838 [INSTR_ALU_SHR_HI] = instr_alu_export,
11840 [INSTR_REGPREFETCH_RH] = instr_reg_export,
11841 [INSTR_REGPREFETCH_RM] = instr_reg_export,
11842 [INSTR_REGPREFETCH_RI] = instr_reg_export,
11844 [INSTR_REGRD_HRH] = instr_reg_export,
11845 [INSTR_REGRD_HRM] = instr_reg_export,
11846 [INSTR_REGRD_MRH] = instr_reg_export,
11847 [INSTR_REGRD_MRM] = instr_reg_export,
11848 [INSTR_REGRD_HRI] = instr_reg_export,
11849 [INSTR_REGRD_MRI] = instr_reg_export,
11851 [INSTR_REGWR_RHH] = instr_reg_export,
11852 [INSTR_REGWR_RHM] = instr_reg_export,
11853 [INSTR_REGWR_RMH] = instr_reg_export,
11854 [INSTR_REGWR_RMM] = instr_reg_export,
11855 [INSTR_REGWR_RHI] = instr_reg_export,
11856 [INSTR_REGWR_RMI] = instr_reg_export,
11857 [INSTR_REGWR_RIH] = instr_reg_export,
11858 [INSTR_REGWR_RIM] = instr_reg_export,
11859 [INSTR_REGWR_RII] = instr_reg_export,
11861 [INSTR_REGADD_RHH] = instr_reg_export,
11862 [INSTR_REGADD_RHM] = instr_reg_export,
11863 [INSTR_REGADD_RMH] = instr_reg_export,
11864 [INSTR_REGADD_RMM] = instr_reg_export,
11865 [INSTR_REGADD_RHI] = instr_reg_export,
11866 [INSTR_REGADD_RMI] = instr_reg_export,
11867 [INSTR_REGADD_RIH] = instr_reg_export,
11868 [INSTR_REGADD_RIM] = instr_reg_export,
11869 [INSTR_REGADD_RII] = instr_reg_export,
11871 [INSTR_METPREFETCH_H] = instr_meter_export,
11872 [INSTR_METPREFETCH_M] = instr_meter_export,
11873 [INSTR_METPREFETCH_I] = instr_meter_export,
11875 [INSTR_METER_HHM] = instr_meter_export,
11876 [INSTR_METER_HHI] = instr_meter_export,
11877 [INSTR_METER_HMM] = instr_meter_export,
11878 [INSTR_METER_HMI] = instr_meter_export,
11879 [INSTR_METER_MHM] = instr_meter_export,
11880 [INSTR_METER_MHI] = instr_meter_export,
11881 [INSTR_METER_MMM] = instr_meter_export,
11882 [INSTR_METER_MMI] = instr_meter_export,
11883 [INSTR_METER_IHM] = instr_meter_export,
11884 [INSTR_METER_IHI] = instr_meter_export,
11885 [INSTR_METER_IMM] = instr_meter_export,
11886 [INSTR_METER_IMI] = instr_meter_export,
11888 [INSTR_TABLE] = instr_table_export,
11889 [INSTR_TABLE_AF] = instr_table_export,
11890 [INSTR_SELECTOR] = instr_table_export,
11891 [INSTR_LEARNER] = instr_table_export,
11892 [INSTR_LEARNER_AF] = instr_table_export,
11894 [INSTR_LEARNER_LEARN] = instr_learn_export,
11895 [INSTR_LEARNER_REARM] = instr_rearm_export,
11896 [INSTR_LEARNER_REARM_NEW] = instr_rearm_export,
11897 [INSTR_LEARNER_FORGET] = instr_forget_export,
11899 [INSTR_EXTERN_OBJ] = instr_extern_export,
11900 [INSTR_EXTERN_FUNC] = instr_extern_export,
11901 [INSTR_HASH_FUNC] = instr_hash_export,
11903 [INSTR_JMP] = instr_jmp_export,
11904 [INSTR_JMP_VALID] = instr_jmp_export,
11905 [INSTR_JMP_INVALID] = instr_jmp_export,
11906 [INSTR_JMP_HIT] = instr_jmp_export,
11907 [INSTR_JMP_MISS] = instr_jmp_export,
11908 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11909 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11911 [INSTR_JMP_EQ] = instr_jmp_export,
11912 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11913 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11914 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11915 [INSTR_JMP_EQ_I] = instr_jmp_export,
11917 [INSTR_JMP_NEQ] = instr_jmp_export,
11918 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11919 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11920 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11921 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11923 [INSTR_JMP_LT] = instr_jmp_export,
11924 [INSTR_JMP_LT_MH] = instr_jmp_export,
11925 [INSTR_JMP_LT_HM] = instr_jmp_export,
11926 [INSTR_JMP_LT_HH] = instr_jmp_export,
11927 [INSTR_JMP_LT_MI] = instr_jmp_export,
11928 [INSTR_JMP_LT_HI] = instr_jmp_export,
11930 [INSTR_JMP_GT] = instr_jmp_export,
11931 [INSTR_JMP_GT_MH] = instr_jmp_export,
11932 [INSTR_JMP_GT_HM] = instr_jmp_export,
11933 [INSTR_JMP_GT_HH] = instr_jmp_export,
11934 [INSTR_JMP_GT_MI] = instr_jmp_export,
11935 [INSTR_JMP_GT_HI] = instr_jmp_export,
11937 [INSTR_RETURN] = instr_return_export,
11941 action_data_codegen(struct action *a, FILE *f)
11946 "static const struct instruction action_%s_instructions[] = {\n",
11949 for (i = 0; i < a->n_instructions; i++) {
11950 struct instruction *instr = &a->instructions[i];
11951 instruction_export_t func = export_table[instr->type];
11956 fprintf(f, "};\n");
11959 static const char *
11960 instr_type_to_func(struct instruction *instr)
11962 switch (instr->type) {
11963 case INSTR_RX: return NULL;
11965 case INSTR_TX: return "__instr_tx_exec";
11966 case INSTR_TX_I: return "__instr_tx_i_exec";
11967 case INSTR_DROP: return "__instr_drop_exec";
11968 case INSTR_MIRROR: return "__instr_mirror_exec";
11969 case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11970 case INSTR_RECIRCID: return "__instr_recircid_exec";
11972 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11973 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11974 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11975 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11976 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11977 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11978 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11979 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11981 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11983 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11985 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11986 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11987 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11988 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11989 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11990 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11991 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11992 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11993 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11995 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11996 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11998 case INSTR_MOV: return "__instr_mov_exec";
11999 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
12000 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
12001 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
12002 case INSTR_MOV_I: return "__instr_mov_i_exec";
12004 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
12005 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
12006 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
12007 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
12008 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
12009 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
12010 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
12011 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
12013 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
12014 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
12015 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
12016 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
12017 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
12018 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
12020 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
12021 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
12022 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
12023 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
12024 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
12025 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
12027 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
12028 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
12029 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
12030 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
12032 case INSTR_ALU_AND: return "__instr_alu_and_exec";
12033 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
12034 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
12035 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
12036 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
12038 case INSTR_ALU_OR: return "__instr_alu_or_exec";
12039 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
12040 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
12041 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
12042 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
12044 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
12045 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
12046 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
12047 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
12048 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
12050 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
12051 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
12052 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
12053 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
12054 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
12055 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
12057 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
12058 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
12059 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
12060 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
12061 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
12062 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
12064 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
12065 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
12066 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
12068 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
12069 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
12070 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
12071 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
12072 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
12073 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
12075 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
12076 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
12077 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
12078 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
12079 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
12080 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
12081 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
12082 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
12083 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
12085 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
12086 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
12087 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
12088 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
12089 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
12090 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
12091 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
12092 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
12093 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
12095 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
12096 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
12097 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
12099 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
12100 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
12101 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
12102 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
12103 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
12104 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
12105 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
12106 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
12107 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
12108 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
12109 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
12110 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
12112 case INSTR_TABLE: return NULL;
12113 case INSTR_TABLE_AF: return NULL;
12114 case INSTR_SELECTOR: return NULL;
12115 case INSTR_LEARNER: return NULL;
12116 case INSTR_LEARNER_AF: return NULL;
12118 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
12119 case INSTR_LEARNER_REARM: return "__instr_rearm_exec";
12120 case INSTR_LEARNER_REARM_NEW: return "__instr_rearm_new_exec";
12121 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
12123 case INSTR_EXTERN_OBJ: return NULL;
12124 case INSTR_EXTERN_FUNC: return NULL;
12125 case INSTR_HASH_FUNC: return "__instr_hash_func_exec";
12127 case INSTR_JMP: return NULL;
12128 case INSTR_JMP_VALID: return NULL;
12129 case INSTR_JMP_INVALID: return NULL;
12130 case INSTR_JMP_HIT: return NULL;
12131 case INSTR_JMP_MISS: return NULL;
12132 case INSTR_JMP_ACTION_HIT: return NULL;
12133 case INSTR_JMP_ACTION_MISS: return NULL;
12134 case INSTR_JMP_EQ: return NULL;
12135 case INSTR_JMP_EQ_MH: return NULL;
12136 case INSTR_JMP_EQ_HM: return NULL;
12137 case INSTR_JMP_EQ_HH: return NULL;
12138 case INSTR_JMP_EQ_I: return NULL;
12139 case INSTR_JMP_NEQ: return NULL;
12140 case INSTR_JMP_NEQ_MH: return NULL;
12141 case INSTR_JMP_NEQ_HM: return NULL;
12142 case INSTR_JMP_NEQ_HH: return NULL;
12143 case INSTR_JMP_NEQ_I: return NULL;
12144 case INSTR_JMP_LT: return NULL;
12145 case INSTR_JMP_LT_MH: return NULL;
12146 case INSTR_JMP_LT_HM: return NULL;
12147 case INSTR_JMP_LT_HH: return NULL;
12148 case INSTR_JMP_LT_MI: return NULL;
12149 case INSTR_JMP_LT_HI: return NULL;
12150 case INSTR_JMP_GT: return NULL;
12151 case INSTR_JMP_GT_MH: return NULL;
12152 case INSTR_JMP_GT_HM: return NULL;
12153 case INSTR_JMP_GT_HH: return NULL;
12154 case INSTR_JMP_GT_MI: return NULL;
12155 case INSTR_JMP_GT_HI: return NULL;
12157 case INSTR_RETURN: return NULL;
12159 default: return NULL;
12164 action_instr_does_tx_codegen(struct action *a,
12165 uint32_t instr_pos,
12166 struct instruction *instr,
12170 "%s(p, t, &action_%s_instructions[%u]);\n"
12171 "\tthread_ip_reset(p, t);\n"
12172 "\tinstr_rx_exec(p);\n"
12174 instr_type_to_func(instr),
12180 action_instr_extern_obj_codegen(struct action *a,
12181 uint32_t instr_pos,
12185 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
12191 action_instr_extern_func_codegen(struct action *a,
12192 uint32_t instr_pos,
12196 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
12202 action_instr_jmp_codegen(struct action *a,
12203 uint32_t instr_pos,
12204 struct instruction *instr,
12205 struct instruction_data *data,
12208 switch (instr->type) {
12215 case INSTR_JMP_VALID:
12217 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12224 case INSTR_JMP_INVALID:
12226 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
12233 case INSTR_JMP_HIT:
12240 case INSTR_JMP_MISS:
12247 case INSTR_JMP_ACTION_HIT:
12249 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
12256 case INSTR_JMP_ACTION_MISS:
12258 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
12267 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12268 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12277 case INSTR_JMP_EQ_MH:
12279 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12280 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12289 case INSTR_JMP_EQ_HM:
12291 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12292 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12301 case INSTR_JMP_EQ_HH:
12303 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
12304 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12313 case INSTR_JMP_EQ_I:
12315 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
12316 "action_%s_instructions[%u].jmp.b_val)\n"
12325 case INSTR_JMP_NEQ:
12327 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12328 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12337 case INSTR_JMP_NEQ_MH:
12339 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12340 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12349 case INSTR_JMP_NEQ_HM:
12351 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12352 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12361 case INSTR_JMP_NEQ_HH:
12363 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
12364 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12373 case INSTR_JMP_NEQ_I:
12375 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
12376 "action_%s_instructions[%u].jmp.b_val)\n"
12387 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12388 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12397 case INSTR_JMP_LT_MH:
12399 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12400 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12409 case INSTR_JMP_LT_HM:
12411 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12412 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12421 case INSTR_JMP_LT_HH:
12423 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12424 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12433 case INSTR_JMP_LT_MI:
12435 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12436 "action_%s_instructions[%u].jmp.b_val)\n"
12445 case INSTR_JMP_LT_HI:
12447 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12448 "action_%s_instructions[%u].jmp.b_val)\n"
12459 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12460 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12469 case INSTR_JMP_GT_MH:
12471 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12472 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12481 case INSTR_JMP_GT_HM:
12483 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12484 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12493 case INSTR_JMP_GT_HH:
12495 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12496 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12505 case INSTR_JMP_GT_MI:
12507 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12508 "action_%s_instructions[%u].jmp.b_val)\n"
12517 case INSTR_JMP_GT_HI:
12519 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12520 "action_%s_instructions[%u].jmp.b_val)\n"
12535 action_instr_return_codegen(FILE *f)
12542 action_instr_codegen(struct action *a, FILE *f)
12548 "action_%s_run(struct rte_swx_pipeline *p)\n"
12550 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12554 for (i = 0; i < a->n_instructions; i++) {
12555 struct instruction *instr = &a->instructions[i];
12556 struct instruction_data *data = &a->instruction_data[i];
12558 /* Label, if present. */
12559 if (data->label[0])
12560 fprintf(f, "\n%s : ", data->label);
12562 fprintf(f, "\n\t");
12564 /* TX instruction type. */
12565 if (instruction_does_tx(instr)) {
12566 action_instr_does_tx_codegen(a, i, instr, f);
12570 /* Extern object/function instruction type. */
12571 if (instr->type == INSTR_EXTERN_OBJ) {
12572 action_instr_extern_obj_codegen(a, i, f);
12576 if (instr->type == INSTR_EXTERN_FUNC) {
12577 action_instr_extern_func_codegen(a, i, f);
12581 /* Jump instruction type. */
12582 if (instruction_is_jmp(instr)) {
12583 action_instr_jmp_codegen(a, i, instr, data, f);
12587 /* Return instruction type. */
12588 if (instr->type == INSTR_RETURN) {
12589 action_instr_return_codegen(f);
12593 /* Any other instruction type. */
12595 "%s(p, t, &action_%s_instructions[%u]);\n",
12596 instr_type_to_func(instr),
12601 fprintf(f, "}\n\n");
12604 struct instruction_group {
12605 TAILQ_ENTRY(instruction_group) node;
12609 uint32_t first_instr_id;
12611 uint32_t last_instr_id;
12616 TAILQ_HEAD(instruction_group_list, instruction_group);
12618 static struct instruction_group *
12619 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12621 struct instruction_group *g;
12623 TAILQ_FOREACH(g, igl, node)
12624 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12631 instruction_group_list_free(struct instruction_group_list *igl)
12637 struct instruction_group *g;
12639 g = TAILQ_FIRST(igl);
12643 TAILQ_REMOVE(igl, g, node);
12650 static struct instruction_group_list *
12651 instruction_group_list_create(struct rte_swx_pipeline *p)
12653 struct instruction_group_list *igl = NULL;
12654 struct instruction_group *g = NULL;
12655 uint32_t n_groups = 0, i;
12657 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12661 igl = calloc(1, sizeof(struct instruction_group_list));
12667 /* Allocate the first group. */
12668 g = calloc(1, sizeof(struct instruction_group));
12672 /* Iteration 1: Separate the instructions into groups based on the thread yield
12673 * instructions. Do not worry about the jump instructions at this point.
12675 for (i = 0; i < p->n_instructions; i++) {
12676 struct instruction *instr = &p->instructions[i];
12678 /* Check for thread yield instructions. */
12679 if (!instruction_does_thread_yield(instr))
12682 /* If the current group contains at least one instruction, then finalize it (with
12683 * the previous instruction), add it to the list and allocate a new group (that
12684 * starts with the current instruction).
12686 if (i - g->first_instr_id) {
12687 /* Finalize the group. */
12688 g->last_instr_id = i - 1;
12690 /* Add the group to the list. Advance the number of groups. */
12691 TAILQ_INSERT_TAIL(igl, g, node);
12694 /* Allocate a new group. */
12695 g = calloc(1, sizeof(struct instruction_group));
12699 /* Initialize the new group. */
12700 g->group_id = n_groups;
12701 g->first_instr_id = i;
12704 /* Finalize the current group (with the current instruction, therefore this group
12705 * contains just the current thread yield instruction), add it to the list and
12706 * allocate a new group (that starts with the next instruction).
12709 /* Finalize the group. */
12710 g->last_instr_id = i;
12712 /* Add the group to the list. Advance the number of groups. */
12713 TAILQ_INSERT_TAIL(igl, g, node);
12716 /* Allocate a new group. */
12717 g = calloc(1, sizeof(struct instruction_group));
12721 /* Initialize the new group. */
12722 g->group_id = n_groups;
12723 g->first_instr_id = i + 1;
12726 /* Handle the last group. */
12727 if (i - g->first_instr_id) {
12728 /* Finalize the group. */
12729 g->last_instr_id = i - 1;
12731 /* Add the group to the list. Advance the number of groups. */
12732 TAILQ_INSERT_TAIL(igl, g, node);
12739 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12740 * the destination of a jump instruction located in a different group ("far jump"), then the
12741 * current group has to be split, so that the instruction representing the far jump
12742 * destination is at the start of its group.
12745 int is_modified = 0;
12747 for (i = 0; i < p->n_instructions; i++) {
12748 struct instruction_data *data = &p->instruction_data[i];
12749 struct instruction_group *g;
12752 /* Continue when the current instruction is not a jump destination. */
12753 if (!data->n_users)
12756 g = instruction_group_list_group_find(igl, i);
12760 /* Find out all the jump instructions with this destination. */
12761 for (j = 0; j < p->n_instructions; j++) {
12762 struct instruction *jmp_instr = &p->instructions[j];
12763 struct instruction_data *jmp_data = &p->instruction_data[j];
12764 struct instruction_group *jmp_g, *new_g;
12766 /* Continue when not a jump instruction. Even when jump instruction,
12767 * continue when the jump destination is not this instruction.
12769 if (!instruction_is_jmp(jmp_instr) ||
12770 strcmp(jmp_data->jmp_label, data->label))
12773 jmp_g = instruction_group_list_group_find(igl, j);
12777 /* Continue when both the jump instruction and the jump destination
12778 * instruction are in the same group. Even when in different groups,
12779 * still continue if the jump destination instruction is already the
12780 * first instruction of its group.
12782 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12785 /* Split the group of the current jump destination instruction to
12786 * make this instruction the first instruction of a new group.
12788 new_g = calloc(1, sizeof(struct instruction_group));
12792 new_g->group_id = n_groups;
12793 new_g->first_instr_id = i;
12794 new_g->last_instr_id = g->last_instr_id;
12796 g->last_instr_id = i - 1;
12798 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12802 /* The decision to split this group (to make the current instruction
12803 * the first instruction of a new group) is already taken and fully
12804 * implemented, so no need to search for more reasons to do it.
12810 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12811 * previously considered local (i.e. the jump destination is in the same group as
12812 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12813 * different group than the jump instruction). Wost case scenario: each instruction
12814 * that is a jump destination ends up as the first instruction of its group.
12820 /* Re-assign the group IDs to be in incremental order. */
12822 TAILQ_FOREACH(g, igl, node) {
12831 instruction_group_list_free(igl);
12839 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12840 uint32_t instr_pos,
12841 struct instruction *instr,
12845 "%s(p, t, &pipeline_instructions[%u]);\n"
12846 "\tthread_ip_reset(p, t);\n"
12847 "\tinstr_rx_exec(p);\n"
12849 instr_type_to_func(instr),
12854 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12855 struct instruction_group_list *igl,
12856 uint32_t jmp_instr_id,
12857 struct instruction *jmp_instr,
12858 struct instruction_data *jmp_data,
12861 struct instruction_group *jmp_g, *g;
12862 struct instruction_data *data;
12865 switch (jmp_instr->type) {
12869 case INSTR_JMP_VALID:
12871 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12875 case INSTR_JMP_INVALID:
12877 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12881 case INSTR_JMP_HIT:
12886 case INSTR_JMP_MISS:
12891 case INSTR_JMP_ACTION_HIT:
12893 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12897 case INSTR_JMP_ACTION_MISS:
12899 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12905 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12906 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12911 case INSTR_JMP_EQ_MH:
12913 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12914 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12919 case INSTR_JMP_EQ_HM:
12921 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12922 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12927 case INSTR_JMP_EQ_HH:
12929 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12930 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12935 case INSTR_JMP_EQ_I:
12937 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12938 "pipeline_instructions[%u].jmp.b_val)",
12943 case INSTR_JMP_NEQ:
12945 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12946 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12951 case INSTR_JMP_NEQ_MH:
12953 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12954 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12959 case INSTR_JMP_NEQ_HM:
12961 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12962 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12967 case INSTR_JMP_NEQ_HH:
12969 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12970 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12975 case INSTR_JMP_NEQ_I:
12977 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12978 "pipeline_instructions[%u].jmp.b_val)",
12985 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12986 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12991 case INSTR_JMP_LT_MH:
12993 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12994 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12999 case INSTR_JMP_LT_HM:
13001 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13002 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13007 case INSTR_JMP_LT_HH:
13009 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13010 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13015 case INSTR_JMP_LT_MI:
13017 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
13018 "pipeline_instructions[%u].jmp.b_val)",
13023 case INSTR_JMP_LT_HI:
13025 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
13026 "pipeline_instructions[%u].jmp.b_val)",
13033 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13034 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13039 case INSTR_JMP_GT_MH:
13041 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13042 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13047 case INSTR_JMP_GT_HM:
13049 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13050 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
13055 case INSTR_JMP_GT_HH:
13057 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13058 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
13063 case INSTR_JMP_GT_MI:
13065 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
13066 "pipeline_instructions[%u].jmp.b_val)",
13071 case INSTR_JMP_GT_HI:
13073 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
13074 "pipeline_instructions[%u].jmp.b_val)",
13083 /* Find the instruction group of the jump instruction. */
13084 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
13088 /* Find the instruction group of the jump destination instruction. */
13089 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
13093 instr_id = data - p->instruction_data;
13095 g = instruction_group_list_group_find(igl, instr_id);
13099 /* Code generation for "near" jump (same instruction group) or "far" jump (different
13100 * instruction group).
13102 if (g->group_id == jmp_g->group_id)
13104 "\n\t\tgoto %s;\n",
13105 jmp_data->jmp_label);
13109 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
13118 instruction_group_list_codegen(struct instruction_group_list *igl,
13119 struct rte_swx_pipeline *p,
13122 struct instruction_group *g;
13124 int is_required = 0;
13126 /* Check if code generation is required. */
13127 TAILQ_FOREACH(g, igl, node)
13128 if (g->first_instr_id < g->last_instr_id)
13134 /* Generate the code for the pipeline instruction array. */
13136 "static const struct instruction pipeline_instructions[] = {\n");
13138 for (i = 0; i < p->n_instructions; i++) {
13139 struct instruction *instr = &p->instructions[i];
13140 instruction_export_t func = export_table[instr->type];
13145 fprintf(f, "};\n\n");
13147 /* Generate the code for the pipeline functions: one function for each instruction group
13148 * that contains more than one instruction.
13150 TAILQ_FOREACH(g, igl, node) {
13151 struct instruction *last_instr;
13154 /* Skip if group contains a single instruction. */
13155 if (g->last_instr_id == g->first_instr_id)
13158 /* Generate new pipeline function. */
13161 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
13163 "\tstruct thread *t = &p->threads[p->thread_id];\n"
13167 /* Generate the code for each pipeline instruction. */
13168 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
13169 struct instruction *instr = &p->instructions[j];
13170 struct instruction_data *data = &p->instruction_data[j];
13172 /* Label, if present. */
13173 if (data->label[0])
13174 fprintf(f, "\n%s : ", data->label);
13176 fprintf(f, "\n\t");
13178 /* TX instruction type. */
13179 if (instruction_does_tx(instr)) {
13180 pipeline_instr_does_tx_codegen(p, j, instr, f);
13184 /* Jump instruction type. */
13185 if (instruction_is_jmp(instr)) {
13186 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
13190 /* Any other instruction type. */
13192 "%s(p, t, &pipeline_instructions[%u]);\n",
13193 instr_type_to_func(instr),
13197 /* Finalize the generated pipeline function. For some instructions such as TX,
13198 * emit-many-and-TX and unconditional jump, the next instruction has been already
13199 * decided unconditionally and the instruction pointer of the current thread set
13200 * accordingly; for all the other instructions, the instruction pointer must be
13203 last_instr = &p->instructions[g->last_instr_id];
13205 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
13207 "thread_ip_inc(p);\n");
13216 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
13218 struct instruction_group *g;
13219 uint32_t n_custom_instr = 0;
13221 /* Groups with a single instruction: no function is generated for this group, the group
13222 * keeps its current instruction. Groups with more than two instructions: one function and
13223 * the associated custom instruction get generated for each such group.
13225 TAILQ_FOREACH(g, igl, node) {
13226 if (g->first_instr_id == g->last_instr_id)
13232 return n_custom_instr;
13236 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13241 /* Create the .c file. */
13242 f = fopen("/tmp/pipeline.c", "w");
13246 /* Include the .h file. */
13247 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
13249 /* Add the code for each action. */
13250 TAILQ_FOREACH(a, &p->actions, node) {
13251 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
13253 action_data_codegen(a, f);
13257 action_instr_codegen(a, f);
13262 /* Add the pipeline code. */
13263 instruction_group_list_codegen(igl, p, f);
13265 /* Close the .c file. */
13271 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
13272 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
13276 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13279 struct instruction_group *g;
13280 char *dir_in, *buffer = NULL;
13281 const char *dir_out;
13284 /* Get the environment variables. */
13285 dir_in = getenv("RTE_INSTALL_DIR");
13293 /* Memory allocation for the command buffer. */
13294 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
13301 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13302 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
13303 "-I %s/lib/pipeline "
13304 "-I %s/lib/eal/include "
13305 "-I %s/lib/eal/x86/include "
13306 "-I %s/lib/eal/include/generic "
13310 "-I %s/lib/pipeline "
13313 "-I %s/lib/eal/linux/include "
13314 ">%s/pipeline.log 2>&1 "
13316 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
13317 ">>%s/pipeline.log 2>&1",
13336 /* Build the shared object library. */
13337 status = system(buffer);
13341 /* Open library. */
13343 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
13344 "%s/libpipeline.so",
13347 p->lib = dlopen(buffer, RTLD_LAZY);
13353 /* Get the action function symbols. */
13354 TAILQ_FOREACH(a, &p->actions, node) {
13355 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
13357 p->action_funcs[a->id] = dlsym(p->lib, buffer);
13358 if (!p->action_funcs[a->id]) {
13364 /* Get the pipeline function symbols. */
13365 TAILQ_FOREACH(g, igl, node) {
13366 if (g->first_instr_id == g->last_instr_id)
13369 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
13371 g->func = dlsym(p->lib, buffer);
13379 if (status && p->lib) {
13390 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13391 struct instruction_group_list *igl)
13393 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13395 /* Check that enough space is available within the pipeline instruction table to store all
13396 * the custom instructions.
13398 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13405 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13407 struct instruction_group *g;
13410 /* Pipeline table instructions. */
13411 for (i = 0; i < p->n_instructions; i++) {
13412 struct instruction *instr = &p->instructions[i];
13414 if (instr->type == INSTR_TABLE)
13415 instr->type = INSTR_TABLE_AF;
13417 if (instr->type == INSTR_LEARNER)
13418 instr->type = INSTR_LEARNER_AF;
13421 /* Pipeline custom instructions. */
13423 TAILQ_FOREACH(g, igl, node) {
13424 struct instruction *instr = &p->instructions[g->first_instr_id];
13427 if (g->first_instr_id == g->last_instr_id)
13430 /* Install a new custom instruction. */
13431 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13433 /* First instruction of the group: change its type to the new custom instruction. */
13434 instr->type = INSTR_CUSTOM_0 + i;
13436 /* All the subsequent instructions of the group: invalidate. */
13437 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13438 struct instruction_data *data = &p->instruction_data[j];
13446 /* Remove the invalidated instructions. */
13447 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13449 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13450 * instructions that are the only instruction within their group, so they were left
13453 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13457 pipeline_compile(struct rte_swx_pipeline *p)
13459 struct instruction_group_list *igl = NULL;
13462 igl = instruction_group_list_create(p);
13468 /* Code generation. */
13469 status = pipeline_codegen(p, igl);
13473 /* Build and load the shared object library. */
13474 status = pipeline_libload(p, igl);
13478 /* Adjust instructions. */
13479 status = pipeline_adjust_check(p, igl);
13483 pipeline_adjust(p, igl);
13486 instruction_group_list_free(igl);