1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
9 #include <rte_swx_port_ethdev.h>
10 #include <rte_swx_port_fd.h>
11 #include <rte_swx_port_ring.h>
12 #include "rte_swx_port_source_sink.h"
14 #include <rte_swx_table_em.h>
15 #include <rte_swx_table_wm.h>
17 #include "rte_swx_pipeline_internal.h"
19 #define CHECK(condition, err_code) \
25 #define CHECK_NAME(name, err_code) \
28 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
31 #define CHECK_INSTRUCTION(instr, err_code) \
34 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
35 RTE_SWX_INSTRUCTION_SIZE), \
41 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
43 #include <rte_malloc.h>
46 env_malloc(size_t size, size_t alignment, int numa_node)
48 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
52 env_free(void *start, size_t size __rte_unused)
62 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
66 if (numa_available() == -1)
69 start = numa_alloc_onnode(size, numa_node);
73 memset(start, 0, size);
78 env_free(void *start, size_t size)
80 if (numa_available() == -1)
83 numa_free(start, size);
91 static struct struct_type *
92 struct_type_find(struct rte_swx_pipeline *p, const char *name)
94 struct struct_type *elem;
96 TAILQ_FOREACH(elem, &p->struct_types, node)
97 if (strcmp(elem->name, name) == 0)
103 static struct field *
104 struct_type_field_find(struct struct_type *st, const char *name)
108 for (i = 0; i < st->n_fields; i++) {
109 struct field *f = &st->fields[i];
111 if (strcmp(f->name, name) == 0)
119 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
121 struct rte_swx_field_params *fields,
123 int last_field_has_variable_size)
125 struct struct_type *st;
129 CHECK_NAME(name, EINVAL);
130 CHECK(fields, EINVAL);
131 CHECK(n_fields, EINVAL);
133 for (i = 0; i < n_fields; i++) {
134 struct rte_swx_field_params *f = &fields[i];
135 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
138 CHECK_NAME(f->name, EINVAL);
139 CHECK(f->n_bits, EINVAL);
140 CHECK((f->n_bits <= 64) || var_size, EINVAL);
141 CHECK((f->n_bits & 7) == 0, EINVAL);
143 for (j = 0; j < i; j++) {
144 struct rte_swx_field_params *f_prev = &fields[j];
146 CHECK(strcmp(f->name, f_prev->name), EINVAL);
150 CHECK(!struct_type_find(p, name), EEXIST);
152 /* Node allocation. */
153 st = calloc(1, sizeof(struct struct_type));
156 st->fields = calloc(n_fields, sizeof(struct field));
162 /* Node initialization. */
163 strcpy(st->name, name);
164 for (i = 0; i < n_fields; i++) {
165 struct field *dst = &st->fields[i];
166 struct rte_swx_field_params *src = &fields[i];
167 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
169 strcpy(dst->name, src->name);
170 dst->n_bits = src->n_bits;
171 dst->offset = st->n_bits;
172 dst->var_size = var_size;
174 st->n_bits += src->n_bits;
175 st->n_bits_min += var_size ? 0 : src->n_bits;
177 st->n_fields = n_fields;
178 st->var_size = last_field_has_variable_size;
180 /* Node add to tailq. */
181 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
187 struct_build(struct rte_swx_pipeline *p)
191 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
192 struct thread *t = &p->threads[i];
194 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
195 CHECK(t->structs, ENOMEM);
202 struct_build_free(struct rte_swx_pipeline *p)
206 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
207 struct thread *t = &p->threads[i];
215 struct_free(struct rte_swx_pipeline *p)
217 struct_build_free(p);
221 struct struct_type *elem;
223 elem = TAILQ_FIRST(&p->struct_types);
227 TAILQ_REMOVE(&p->struct_types, elem, node);
236 static struct port_in_type *
237 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
239 struct port_in_type *elem;
244 TAILQ_FOREACH(elem, &p->port_in_types, node)
245 if (strcmp(elem->name, name) == 0)
252 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
254 struct rte_swx_port_in_ops *ops)
256 struct port_in_type *elem;
259 CHECK_NAME(name, EINVAL);
261 CHECK(ops->create, EINVAL);
262 CHECK(ops->free, EINVAL);
263 CHECK(ops->pkt_rx, EINVAL);
264 CHECK(ops->stats_read, EINVAL);
266 CHECK(!port_in_type_find(p, name), EEXIST);
268 /* Node allocation. */
269 elem = calloc(1, sizeof(struct port_in_type));
272 /* Node initialization. */
273 strcpy(elem->name, name);
274 memcpy(&elem->ops, ops, sizeof(*ops));
276 /* Node add to tailq. */
277 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
282 static struct port_in *
283 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
285 struct port_in *port;
287 TAILQ_FOREACH(port, &p->ports_in, node)
288 if (port->id == port_id)
295 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
297 const char *port_type_name,
300 struct port_in_type *type = NULL;
301 struct port_in *port = NULL;
306 CHECK(!port_in_find(p, port_id), EINVAL);
308 CHECK_NAME(port_type_name, EINVAL);
309 type = port_in_type_find(p, port_type_name);
312 obj = type->ops.create(args);
315 /* Node allocation. */
316 port = calloc(1, sizeof(struct port_in));
319 /* Node initialization. */
324 /* Node add to tailq. */
325 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
326 if (p->n_ports_in < port_id + 1)
327 p->n_ports_in = port_id + 1;
333 port_in_build(struct rte_swx_pipeline *p)
335 struct port_in *port;
338 CHECK(p->n_ports_in, EINVAL);
339 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
341 for (i = 0; i < p->n_ports_in; i++)
342 CHECK(port_in_find(p, i), EINVAL);
344 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
345 CHECK(p->in, ENOMEM);
347 TAILQ_FOREACH(port, &p->ports_in, node) {
348 struct port_in_runtime *in = &p->in[port->id];
350 in->pkt_rx = port->type->ops.pkt_rx;
358 port_in_build_free(struct rte_swx_pipeline *p)
365 port_in_free(struct rte_swx_pipeline *p)
367 port_in_build_free(p);
371 struct port_in *port;
373 port = TAILQ_FIRST(&p->ports_in);
377 TAILQ_REMOVE(&p->ports_in, port, node);
378 port->type->ops.free(port->obj);
382 /* Input port types. */
384 struct port_in_type *elem;
386 elem = TAILQ_FIRST(&p->port_in_types);
390 TAILQ_REMOVE(&p->port_in_types, elem, node);
398 static struct port_out_type *
399 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
401 struct port_out_type *elem;
406 TAILQ_FOREACH(elem, &p->port_out_types, node)
407 if (!strcmp(elem->name, name))
414 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
416 struct rte_swx_port_out_ops *ops)
418 struct port_out_type *elem;
421 CHECK_NAME(name, EINVAL);
423 CHECK(ops->create, EINVAL);
424 CHECK(ops->free, EINVAL);
425 CHECK(ops->pkt_tx, EINVAL);
426 CHECK(ops->pkt_fast_clone_tx, EINVAL);
427 CHECK(ops->pkt_clone_tx, EINVAL);
428 CHECK(ops->stats_read, EINVAL);
430 CHECK(!port_out_type_find(p, name), EEXIST);
432 /* Node allocation. */
433 elem = calloc(1, sizeof(struct port_out_type));
436 /* Node initialization. */
437 strcpy(elem->name, name);
438 memcpy(&elem->ops, ops, sizeof(*ops));
440 /* Node add to tailq. */
441 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
446 static struct port_out *
447 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
449 struct port_out *port;
451 TAILQ_FOREACH(port, &p->ports_out, node)
452 if (port->id == port_id)
459 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
461 const char *port_type_name,
464 struct port_out_type *type = NULL;
465 struct port_out *port = NULL;
470 CHECK(!port_out_find(p, port_id), EINVAL);
472 CHECK_NAME(port_type_name, EINVAL);
473 type = port_out_type_find(p, port_type_name);
476 obj = type->ops.create(args);
479 /* Node allocation. */
480 port = calloc(1, sizeof(struct port_out));
483 /* Node initialization. */
488 /* Node add to tailq. */
489 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
490 if (p->n_ports_out < port_id + 1)
491 p->n_ports_out = port_id + 1;
497 port_out_build(struct rte_swx_pipeline *p)
499 struct port_out *port;
502 CHECK(p->n_ports_out, EINVAL);
504 for (i = 0; i < p->n_ports_out; i++)
505 CHECK(port_out_find(p, i), EINVAL);
507 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
508 CHECK(p->out, ENOMEM);
510 TAILQ_FOREACH(port, &p->ports_out, node) {
511 struct port_out_runtime *out = &p->out[port->id];
513 out->pkt_tx = port->type->ops.pkt_tx;
514 out->pkt_fast_clone_tx = port->type->ops.pkt_fast_clone_tx;
515 out->pkt_clone_tx = port->type->ops.pkt_clone_tx;
516 out->flush = port->type->ops.flush;
517 out->obj = port->obj;
524 port_out_build_free(struct rte_swx_pipeline *p)
531 port_out_free(struct rte_swx_pipeline *p)
533 port_out_build_free(p);
537 struct port_out *port;
539 port = TAILQ_FIRST(&p->ports_out);
543 TAILQ_REMOVE(&p->ports_out, port, node);
544 port->type->ops.free(port->obj);
548 /* Output port types. */
550 struct port_out_type *elem;
552 elem = TAILQ_FIRST(&p->port_out_types);
556 TAILQ_REMOVE(&p->port_out_types, elem, node);
565 rte_swx_pipeline_mirroring_config(struct rte_swx_pipeline *p,
566 struct rte_swx_pipeline_mirroring_params *params)
569 CHECK(params, EINVAL);
570 CHECK(params->n_slots, EINVAL);
571 CHECK(params->n_sessions, EINVAL);
572 CHECK(!p->build_done, EEXIST);
574 p->n_mirroring_slots = rte_align32pow2(params->n_slots);
575 if (p->n_mirroring_slots > 64)
576 p->n_mirroring_slots = 64;
578 p->n_mirroring_sessions = rte_align32pow2(params->n_sessions);
584 mirroring_build_free(struct rte_swx_pipeline *p)
588 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
589 struct thread *t = &p->threads[i];
591 /* mirroring_slots. */
592 free(t->mirroring_slots);
593 t->mirroring_slots = NULL;
596 /* mirroring_sessions. */
597 free(p->mirroring_sessions);
598 p->mirroring_sessions = NULL;
602 mirroring_free(struct rte_swx_pipeline *p)
604 mirroring_build_free(p);
608 mirroring_build(struct rte_swx_pipeline *p)
612 if (!p->n_mirroring_slots || !p->n_mirroring_sessions)
615 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
616 struct thread *t = &p->threads[i];
618 /* mirroring_slots. */
619 t->mirroring_slots = calloc(p->n_mirroring_slots, sizeof(uint32_t));
620 if (!t->mirroring_slots)
624 /* mirroring_sessions. */
625 p->mirroring_sessions = calloc(p->n_mirroring_sessions, sizeof(struct mirroring_session));
626 if (!p->mirroring_sessions)
632 mirroring_build_free(p);
639 static struct extern_type *
640 extern_type_find(struct rte_swx_pipeline *p, const char *name)
642 struct extern_type *elem;
644 TAILQ_FOREACH(elem, &p->extern_types, node)
645 if (strcmp(elem->name, name) == 0)
651 static struct extern_type_member_func *
652 extern_type_member_func_find(struct extern_type *type, const char *name)
654 struct extern_type_member_func *elem;
656 TAILQ_FOREACH(elem, &type->funcs, node)
657 if (strcmp(elem->name, name) == 0)
663 static struct extern_obj *
664 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
666 struct extern_obj *elem;
668 TAILQ_FOREACH(elem, &p->extern_objs, node)
669 if (strcmp(elem->name, name) == 0)
675 static struct extern_type_member_func *
676 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
678 struct extern_obj **obj)
680 struct extern_obj *object;
681 struct extern_type_member_func *func;
682 char *object_name, *func_name;
684 if (name[0] != 'e' || name[1] != '.')
687 object_name = strdup(&name[2]);
691 func_name = strchr(object_name, '.');
700 object = extern_obj_find(p, object_name);
706 func = extern_type_member_func_find(object->type, func_name);
719 static struct field *
720 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
722 struct extern_obj **object)
724 struct extern_obj *obj;
726 char *obj_name, *field_name;
728 if ((name[0] != 'e') || (name[1] != '.'))
731 obj_name = strdup(&name[2]);
735 field_name = strchr(obj_name, '.');
744 obj = extern_obj_find(p, obj_name);
750 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
764 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
766 const char *mailbox_struct_type_name,
767 rte_swx_extern_type_constructor_t constructor,
768 rte_swx_extern_type_destructor_t destructor)
770 struct extern_type *elem;
771 struct struct_type *mailbox_struct_type;
775 CHECK_NAME(name, EINVAL);
776 CHECK(!extern_type_find(p, name), EEXIST);
778 CHECK_NAME(mailbox_struct_type_name, EINVAL);
779 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
780 CHECK(mailbox_struct_type, EINVAL);
781 CHECK(!mailbox_struct_type->var_size, EINVAL);
783 CHECK(constructor, EINVAL);
784 CHECK(destructor, EINVAL);
786 /* Node allocation. */
787 elem = calloc(1, sizeof(struct extern_type));
790 /* Node initialization. */
791 strcpy(elem->name, name);
792 elem->mailbox_struct_type = mailbox_struct_type;
793 elem->constructor = constructor;
794 elem->destructor = destructor;
795 TAILQ_INIT(&elem->funcs);
797 /* Node add to tailq. */
798 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
804 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
805 const char *extern_type_name,
807 rte_swx_extern_type_member_func_t member_func)
809 struct extern_type *type;
810 struct extern_type_member_func *type_member;
814 CHECK_NAME(extern_type_name, EINVAL);
815 type = extern_type_find(p, extern_type_name);
817 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
819 CHECK_NAME(name, EINVAL);
820 CHECK(!extern_type_member_func_find(type, name), EEXIST);
822 CHECK(member_func, EINVAL);
824 /* Node allocation. */
825 type_member = calloc(1, sizeof(struct extern_type_member_func));
826 CHECK(type_member, ENOMEM);
828 /* Node initialization. */
829 strcpy(type_member->name, name);
830 type_member->func = member_func;
831 type_member->id = type->n_funcs;
833 /* Node add to tailq. */
834 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
841 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
842 const char *extern_type_name,
846 struct extern_type *type;
847 struct extern_obj *obj;
852 CHECK_NAME(extern_type_name, EINVAL);
853 type = extern_type_find(p, extern_type_name);
856 CHECK_NAME(name, EINVAL);
857 CHECK(!extern_obj_find(p, name), EEXIST);
859 /* Node allocation. */
860 obj = calloc(1, sizeof(struct extern_obj));
863 /* Object construction. */
864 obj_handle = type->constructor(args);
870 /* Node initialization. */
871 strcpy(obj->name, name);
873 obj->obj = obj_handle;
874 obj->struct_id = p->n_structs;
875 obj->id = p->n_extern_objs;
877 /* Node add to tailq. */
878 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
886 extern_obj_build(struct rte_swx_pipeline *p)
890 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
891 struct thread *t = &p->threads[i];
892 struct extern_obj *obj;
894 t->extern_objs = calloc(p->n_extern_objs,
895 sizeof(struct extern_obj_runtime));
896 CHECK(t->extern_objs, ENOMEM);
898 TAILQ_FOREACH(obj, &p->extern_objs, node) {
899 struct extern_obj_runtime *r =
900 &t->extern_objs[obj->id];
901 struct extern_type_member_func *func;
902 uint32_t mailbox_size =
903 obj->type->mailbox_struct_type->n_bits / 8;
907 r->mailbox = calloc(1, mailbox_size);
908 CHECK(r->mailbox, ENOMEM);
910 TAILQ_FOREACH(func, &obj->type->funcs, node)
911 r->funcs[func->id] = func->func;
913 t->structs[obj->struct_id] = r->mailbox;
921 extern_obj_build_free(struct rte_swx_pipeline *p)
925 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
926 struct thread *t = &p->threads[i];
932 for (j = 0; j < p->n_extern_objs; j++) {
933 struct extern_obj_runtime *r = &t->extern_objs[j];
938 free(t->extern_objs);
939 t->extern_objs = NULL;
944 extern_obj_free(struct rte_swx_pipeline *p)
946 extern_obj_build_free(p);
948 /* Extern objects. */
950 struct extern_obj *elem;
952 elem = TAILQ_FIRST(&p->extern_objs);
956 TAILQ_REMOVE(&p->extern_objs, elem, node);
958 elem->type->destructor(elem->obj);
964 struct extern_type *elem;
966 elem = TAILQ_FIRST(&p->extern_types);
970 TAILQ_REMOVE(&p->extern_types, elem, node);
973 struct extern_type_member_func *func;
975 func = TAILQ_FIRST(&elem->funcs);
979 TAILQ_REMOVE(&elem->funcs, func, node);
990 static struct extern_func *
991 extern_func_find(struct rte_swx_pipeline *p, const char *name)
993 struct extern_func *elem;
995 TAILQ_FOREACH(elem, &p->extern_funcs, node)
996 if (strcmp(elem->name, name) == 0)
1002 static struct extern_func *
1003 extern_func_parse(struct rte_swx_pipeline *p,
1006 if (name[0] != 'f' || name[1] != '.')
1009 return extern_func_find(p, &name[2]);
1012 static struct field *
1013 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
1015 struct extern_func **function)
1017 struct extern_func *func;
1019 char *func_name, *field_name;
1021 if ((name[0] != 'f') || (name[1] != '.'))
1024 func_name = strdup(&name[2]);
1028 field_name = strchr(func_name, '.');
1037 func = extern_func_find(p, func_name);
1043 f = struct_type_field_find(func->mailbox_struct_type, field_name);
1057 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
1059 const char *mailbox_struct_type_name,
1060 rte_swx_extern_func_t func)
1062 struct extern_func *f;
1063 struct struct_type *mailbox_struct_type;
1067 CHECK_NAME(name, EINVAL);
1068 CHECK(!extern_func_find(p, name), EEXIST);
1070 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1071 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1072 CHECK(mailbox_struct_type, EINVAL);
1073 CHECK(!mailbox_struct_type->var_size, EINVAL);
1075 CHECK(func, EINVAL);
1077 /* Node allocation. */
1078 f = calloc(1, sizeof(struct extern_func));
1079 CHECK(func, ENOMEM);
1081 /* Node initialization. */
1082 strcpy(f->name, name);
1083 f->mailbox_struct_type = mailbox_struct_type;
1085 f->struct_id = p->n_structs;
1086 f->id = p->n_extern_funcs;
1088 /* Node add to tailq. */
1089 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1090 p->n_extern_funcs++;
1097 extern_func_build(struct rte_swx_pipeline *p)
1101 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1102 struct thread *t = &p->threads[i];
1103 struct extern_func *func;
1105 /* Memory allocation. */
1106 t->extern_funcs = calloc(p->n_extern_funcs,
1107 sizeof(struct extern_func_runtime));
1108 CHECK(t->extern_funcs, ENOMEM);
1110 /* Extern function. */
1111 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1112 struct extern_func_runtime *r =
1113 &t->extern_funcs[func->id];
1114 uint32_t mailbox_size =
1115 func->mailbox_struct_type->n_bits / 8;
1117 r->func = func->func;
1119 r->mailbox = calloc(1, mailbox_size);
1120 CHECK(r->mailbox, ENOMEM);
1122 t->structs[func->struct_id] = r->mailbox;
1130 extern_func_build_free(struct rte_swx_pipeline *p)
1134 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1135 struct thread *t = &p->threads[i];
1138 if (!t->extern_funcs)
1141 for (j = 0; j < p->n_extern_funcs; j++) {
1142 struct extern_func_runtime *r = &t->extern_funcs[j];
1147 free(t->extern_funcs);
1148 t->extern_funcs = NULL;
1153 extern_func_free(struct rte_swx_pipeline *p)
1155 extern_func_build_free(p);
1158 struct extern_func *elem;
1160 elem = TAILQ_FIRST(&p->extern_funcs);
1164 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1172 static struct header *
1173 header_find(struct rte_swx_pipeline *p, const char *name)
1175 struct header *elem;
1177 TAILQ_FOREACH(elem, &p->headers, node)
1178 if (strcmp(elem->name, name) == 0)
1184 static struct header *
1185 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1187 struct header *elem;
1189 TAILQ_FOREACH(elem, &p->headers, node)
1190 if (elem->struct_id == struct_id)
1196 static struct header *
1197 header_parse(struct rte_swx_pipeline *p,
1200 if (name[0] != 'h' || name[1] != '.')
1203 return header_find(p, &name[2]);
1206 static struct field *
1207 header_field_parse(struct rte_swx_pipeline *p,
1209 struct header **header)
1213 char *header_name, *field_name;
1215 if ((name[0] != 'h') || (name[1] != '.'))
1218 header_name = strdup(&name[2]);
1222 field_name = strchr(header_name, '.');
1231 h = header_find(p, header_name);
1237 f = struct_type_field_find(h->st, field_name);
1251 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1253 const char *struct_type_name)
1255 struct struct_type *st;
1257 size_t n_headers_max;
1260 CHECK_NAME(name, EINVAL);
1261 CHECK_NAME(struct_type_name, EINVAL);
1263 CHECK(!header_find(p, name), EEXIST);
1265 st = struct_type_find(p, struct_type_name);
1268 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1269 CHECK(p->n_headers < n_headers_max, ENOSPC);
1271 /* Node allocation. */
1272 h = calloc(1, sizeof(struct header));
1275 /* Node initialization. */
1276 strcpy(h->name, name);
1278 h->struct_id = p->n_structs;
1279 h->id = p->n_headers;
1281 /* Node add to tailq. */
1282 TAILQ_INSERT_TAIL(&p->headers, h, node);
1290 header_build(struct rte_swx_pipeline *p)
1293 uint32_t n_bytes = 0, i;
1295 TAILQ_FOREACH(h, &p->headers, node) {
1296 n_bytes += h->st->n_bits / 8;
1299 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1300 struct thread *t = &p->threads[i];
1301 uint32_t offset = 0;
1303 t->headers = calloc(p->n_headers,
1304 sizeof(struct header_runtime));
1305 CHECK(t->headers, ENOMEM);
1307 t->headers_out = calloc(p->n_headers,
1308 sizeof(struct header_out_runtime));
1309 CHECK(t->headers_out, ENOMEM);
1311 t->header_storage = calloc(1, n_bytes);
1312 CHECK(t->header_storage, ENOMEM);
1314 t->header_out_storage = calloc(1, n_bytes);
1315 CHECK(t->header_out_storage, ENOMEM);
1317 TAILQ_FOREACH(h, &p->headers, node) {
1318 uint8_t *header_storage;
1319 uint32_t n_bytes = h->st->n_bits / 8;
1321 header_storage = &t->header_storage[offset];
1324 t->headers[h->id].ptr0 = header_storage;
1325 t->headers[h->id].n_bytes = n_bytes;
1327 t->structs[h->struct_id] = header_storage;
1335 header_build_free(struct rte_swx_pipeline *p)
1339 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1340 struct thread *t = &p->threads[i];
1342 free(t->headers_out);
1343 t->headers_out = NULL;
1348 free(t->header_out_storage);
1349 t->header_out_storage = NULL;
1351 free(t->header_storage);
1352 t->header_storage = NULL;
1357 header_free(struct rte_swx_pipeline *p)
1359 header_build_free(p);
1362 struct header *elem;
1364 elem = TAILQ_FIRST(&p->headers);
1368 TAILQ_REMOVE(&p->headers, elem, node);
1376 static struct field *
1377 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1379 if (!p->metadata_st)
1382 if (name[0] != 'm' || name[1] != '.')
1385 return struct_type_field_find(p->metadata_st, &name[2]);
1389 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1390 const char *struct_type_name)
1392 struct struct_type *st = NULL;
1396 CHECK_NAME(struct_type_name, EINVAL);
1397 st = struct_type_find(p, struct_type_name);
1399 CHECK(!st->var_size, EINVAL);
1400 CHECK(!p->metadata_st, EINVAL);
1402 p->metadata_st = st;
1403 p->metadata_struct_id = p->n_structs;
1411 metadata_build(struct rte_swx_pipeline *p)
1413 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1416 /* Thread-level initialization. */
1417 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1418 struct thread *t = &p->threads[i];
1421 metadata = calloc(1, n_bytes);
1422 CHECK(metadata, ENOMEM);
1424 t->metadata = metadata;
1425 t->structs[p->metadata_struct_id] = metadata;
1432 metadata_build_free(struct rte_swx_pipeline *p)
1436 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1437 struct thread *t = &p->threads[i];
1445 metadata_free(struct rte_swx_pipeline *p)
1447 metadata_build_free(p);
1454 instruction_is_tx(enum instruction_type type)
1468 instruction_does_tx(struct instruction *instr)
1470 switch (instr->type) {
1474 case INSTR_HDR_EMIT_TX:
1475 case INSTR_HDR_EMIT2_TX:
1476 case INSTR_HDR_EMIT3_TX:
1477 case INSTR_HDR_EMIT4_TX:
1478 case INSTR_HDR_EMIT5_TX:
1479 case INSTR_HDR_EMIT6_TX:
1480 case INSTR_HDR_EMIT7_TX:
1481 case INSTR_HDR_EMIT8_TX:
1489 instruction_is_jmp(struct instruction *instr)
1491 switch (instr->type) {
1493 case INSTR_JMP_VALID:
1494 case INSTR_JMP_INVALID:
1496 case INSTR_JMP_MISS:
1497 case INSTR_JMP_ACTION_HIT:
1498 case INSTR_JMP_ACTION_MISS:
1500 case INSTR_JMP_EQ_MH:
1501 case INSTR_JMP_EQ_HM:
1502 case INSTR_JMP_EQ_HH:
1503 case INSTR_JMP_EQ_I:
1505 case INSTR_JMP_NEQ_MH:
1506 case INSTR_JMP_NEQ_HM:
1507 case INSTR_JMP_NEQ_HH:
1508 case INSTR_JMP_NEQ_I:
1510 case INSTR_JMP_LT_MH:
1511 case INSTR_JMP_LT_HM:
1512 case INSTR_JMP_LT_HH:
1513 case INSTR_JMP_LT_MI:
1514 case INSTR_JMP_LT_HI:
1516 case INSTR_JMP_GT_MH:
1517 case INSTR_JMP_GT_HM:
1518 case INSTR_JMP_GT_HH:
1519 case INSTR_JMP_GT_MI:
1520 case INSTR_JMP_GT_HI:
1529 instruction_does_thread_yield(struct instruction *instr)
1531 switch (instr->type) {
1534 case INSTR_TABLE_AF:
1535 case INSTR_SELECTOR:
1537 case INSTR_LEARNER_AF:
1538 case INSTR_EXTERN_OBJ:
1539 case INSTR_EXTERN_FUNC:
1546 static struct field *
1547 action_field_parse(struct action *action, const char *name);
1549 static struct field *
1550 struct_field_parse(struct rte_swx_pipeline *p,
1551 struct action *action,
1553 uint32_t *struct_id)
1560 struct header *header;
1562 f = header_field_parse(p, name, &header);
1566 *struct_id = header->struct_id;
1572 f = metadata_field_parse(p, name);
1576 *struct_id = p->metadata_struct_id;
1585 f = action_field_parse(action, name);
1595 struct extern_obj *obj;
1597 f = extern_obj_mailbox_field_parse(p, name, &obj);
1601 *struct_id = obj->struct_id;
1607 struct extern_func *func;
1609 f = extern_func_mailbox_field_parse(p, name, &func);
1613 *struct_id = func->struct_id;
1626 instr_rx_translate(struct rte_swx_pipeline *p,
1627 struct action *action,
1630 struct instruction *instr,
1631 struct instruction_data *data __rte_unused)
1635 CHECK(!action, EINVAL);
1636 CHECK(n_tokens == 2, EINVAL);
1638 f = metadata_field_parse(p, tokens[1]);
1641 instr->type = INSTR_RX;
1642 instr->io.io.offset = f->offset / 8;
1643 instr->io.io.n_bits = f->n_bits;
1651 instr_tx_translate(struct rte_swx_pipeline *p,
1652 struct action *action __rte_unused,
1655 struct instruction *instr,
1656 struct instruction_data *data __rte_unused)
1658 char *port = tokens[1];
1662 CHECK(n_tokens == 2, EINVAL);
1664 f = metadata_field_parse(p, port);
1666 instr->type = INSTR_TX;
1667 instr->io.io.offset = f->offset / 8;
1668 instr->io.io.n_bits = f->n_bits;
1673 port_val = strtoul(port, &port, 0);
1674 CHECK(!port[0], EINVAL);
1676 instr->type = INSTR_TX_I;
1677 instr->io.io.val = port_val;
1682 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1683 struct action *action __rte_unused,
1684 char **tokens __rte_unused,
1686 struct instruction *instr,
1687 struct instruction_data *data __rte_unused)
1689 CHECK(n_tokens == 1, EINVAL);
1692 instr->type = INSTR_DROP;
1697 instr_tx_exec(struct rte_swx_pipeline *p)
1699 struct thread *t = &p->threads[p->thread_id];
1700 struct instruction *ip = t->ip;
1702 __instr_tx_exec(p, t, ip);
1705 thread_ip_reset(p, t);
1710 instr_tx_i_exec(struct rte_swx_pipeline *p)
1712 struct thread *t = &p->threads[p->thread_id];
1713 struct instruction *ip = t->ip;
1715 __instr_tx_i_exec(p, t, ip);
1718 thread_ip_reset(p, t);
1723 instr_drop_exec(struct rte_swx_pipeline *p)
1725 struct thread *t = &p->threads[p->thread_id];
1726 struct instruction *ip = t->ip;
1728 __instr_drop_exec(p, t, ip);
1731 thread_ip_reset(p, t);
1739 instr_mirror_translate(struct rte_swx_pipeline *p,
1740 struct action *action,
1743 struct instruction *instr,
1744 struct instruction_data *data __rte_unused)
1746 char *dst = tokens[1], *src = tokens[2];
1747 struct field *fdst, *fsrc;
1748 uint32_t dst_struct_id = 0, src_struct_id = 0;
1750 CHECK(n_tokens == 3, EINVAL);
1752 fdst = struct_field_parse(p, action, dst, &dst_struct_id);
1753 CHECK(fdst, EINVAL);
1754 CHECK(dst[0] != 'h', EINVAL);
1755 CHECK(!fdst->var_size, EINVAL);
1757 fsrc = struct_field_parse(p, action, src, &src_struct_id);
1758 CHECK(fsrc, EINVAL);
1759 CHECK(src[0] != 'h', EINVAL);
1760 CHECK(!fsrc->var_size, EINVAL);
1762 instr->type = INSTR_MIRROR;
1763 instr->mirror.dst.struct_id = (uint8_t)dst_struct_id;
1764 instr->mirror.dst.n_bits = fdst->n_bits;
1765 instr->mirror.dst.offset = fdst->offset / 8;
1766 instr->mirror.src.struct_id = (uint8_t)src_struct_id;
1767 instr->mirror.src.n_bits = fsrc->n_bits;
1768 instr->mirror.src.offset = fsrc->offset / 8;
1774 instr_mirror_exec(struct rte_swx_pipeline *p)
1776 struct thread *t = &p->threads[p->thread_id];
1777 struct instruction *ip = t->ip;
1779 __instr_mirror_exec(p, t, ip);
1789 instr_recirculate_translate(struct rte_swx_pipeline *p __rte_unused,
1790 struct action *action __rte_unused,
1791 char **tokens __rte_unused,
1793 struct instruction *instr,
1794 struct instruction_data *data __rte_unused)
1796 CHECK(n_tokens == 1, EINVAL);
1798 instr->type = INSTR_RECIRCULATE;
1803 instr_recircid_translate(struct rte_swx_pipeline *p,
1804 struct action *action __rte_unused,
1807 struct instruction *instr,
1808 struct instruction_data *data __rte_unused)
1812 CHECK(n_tokens == 2, EINVAL);
1814 f = metadata_field_parse(p, tokens[1]);
1817 instr->type = INSTR_RECIRCID;
1818 instr->io.io.offset = f->offset / 8;
1819 instr->io.io.n_bits = f->n_bits;
1824 instr_recirculate_exec(struct rte_swx_pipeline *p)
1826 struct thread *t = &p->threads[p->thread_id];
1827 struct instruction *ip = t->ip;
1829 __instr_recirculate_exec(p, t, ip);
1836 instr_recircid_exec(struct rte_swx_pipeline *p)
1838 struct thread *t = &p->threads[p->thread_id];
1839 struct instruction *ip = t->ip;
1841 __instr_recircid_exec(p, t, ip);
1851 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1852 struct action *action,
1855 struct instruction *instr,
1856 struct instruction_data *data __rte_unused)
1860 CHECK(!action, EINVAL);
1861 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1863 h = header_parse(p, tokens[1]);
1866 if (n_tokens == 2) {
1867 CHECK(!h->st->var_size, EINVAL);
1869 instr->type = INSTR_HDR_EXTRACT;
1870 instr->io.hdr.header_id[0] = h->id;
1871 instr->io.hdr.struct_id[0] = h->struct_id;
1872 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1876 CHECK(h->st->var_size, EINVAL);
1878 mf = metadata_field_parse(p, tokens[2]);
1880 CHECK(!mf->var_size, EINVAL);
1882 instr->type = INSTR_HDR_EXTRACT_M;
1883 instr->io.io.offset = mf->offset / 8;
1884 instr->io.io.n_bits = mf->n_bits;
1885 instr->io.hdr.header_id[0] = h->id;
1886 instr->io.hdr.struct_id[0] = h->struct_id;
1887 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1894 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1895 struct action *action,
1898 struct instruction *instr,
1899 struct instruction_data *data __rte_unused)
1903 CHECK(!action, EINVAL);
1904 CHECK(n_tokens == 2, EINVAL);
1906 h = header_parse(p, tokens[1]);
1908 CHECK(!h->st->var_size, EINVAL);
1910 instr->type = INSTR_HDR_LOOKAHEAD;
1911 instr->io.hdr.header_id[0] = h->id;
1912 instr->io.hdr.struct_id[0] = h->struct_id;
1913 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1919 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1921 struct thread *t = &p->threads[p->thread_id];
1922 struct instruction *ip = t->ip;
1924 __instr_hdr_extract_exec(p, t, ip);
1931 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1933 struct thread *t = &p->threads[p->thread_id];
1934 struct instruction *ip = t->ip;
1936 __instr_hdr_extract2_exec(p, t, ip);
1943 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1945 struct thread *t = &p->threads[p->thread_id];
1946 struct instruction *ip = t->ip;
1948 __instr_hdr_extract3_exec(p, t, ip);
1955 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1957 struct thread *t = &p->threads[p->thread_id];
1958 struct instruction *ip = t->ip;
1960 __instr_hdr_extract4_exec(p, t, ip);
1967 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1969 struct thread *t = &p->threads[p->thread_id];
1970 struct instruction *ip = t->ip;
1972 __instr_hdr_extract5_exec(p, t, ip);
1979 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1981 struct thread *t = &p->threads[p->thread_id];
1982 struct instruction *ip = t->ip;
1984 __instr_hdr_extract6_exec(p, t, ip);
1991 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1993 struct thread *t = &p->threads[p->thread_id];
1994 struct instruction *ip = t->ip;
1996 __instr_hdr_extract7_exec(p, t, ip);
2003 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2005 struct thread *t = &p->threads[p->thread_id];
2006 struct instruction *ip = t->ip;
2008 __instr_hdr_extract8_exec(p, t, ip);
2015 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
2017 struct thread *t = &p->threads[p->thread_id];
2018 struct instruction *ip = t->ip;
2020 __instr_hdr_extract_m_exec(p, t, ip);
2027 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
2029 struct thread *t = &p->threads[p->thread_id];
2030 struct instruction *ip = t->ip;
2032 __instr_hdr_lookahead_exec(p, t, ip);
2042 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2043 struct action *action __rte_unused,
2046 struct instruction *instr,
2047 struct instruction_data *data __rte_unused)
2051 CHECK(n_tokens == 2, EINVAL);
2053 h = header_parse(p, tokens[1]);
2056 instr->type = INSTR_HDR_EMIT;
2057 instr->io.hdr.header_id[0] = h->id;
2058 instr->io.hdr.struct_id[0] = h->struct_id;
2059 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2064 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
2066 struct thread *t = &p->threads[p->thread_id];
2067 struct instruction *ip = t->ip;
2069 __instr_hdr_emit_exec(p, t, ip);
2076 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
2078 struct thread *t = &p->threads[p->thread_id];
2079 struct instruction *ip = t->ip;
2081 __instr_hdr_emit_tx_exec(p, t, ip);
2084 thread_ip_reset(p, t);
2089 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
2091 struct thread *t = &p->threads[p->thread_id];
2092 struct instruction *ip = t->ip;
2094 __instr_hdr_emit2_tx_exec(p, t, ip);
2097 thread_ip_reset(p, t);
2102 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
2104 struct thread *t = &p->threads[p->thread_id];
2105 struct instruction *ip = t->ip;
2107 __instr_hdr_emit3_tx_exec(p, t, ip);
2110 thread_ip_reset(p, t);
2115 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
2117 struct thread *t = &p->threads[p->thread_id];
2118 struct instruction *ip = t->ip;
2120 __instr_hdr_emit4_tx_exec(p, t, ip);
2123 thread_ip_reset(p, t);
2128 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
2130 struct thread *t = &p->threads[p->thread_id];
2131 struct instruction *ip = t->ip;
2133 __instr_hdr_emit5_tx_exec(p, t, ip);
2136 thread_ip_reset(p, t);
2141 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
2143 struct thread *t = &p->threads[p->thread_id];
2144 struct instruction *ip = t->ip;
2146 __instr_hdr_emit6_tx_exec(p, t, ip);
2149 thread_ip_reset(p, t);
2154 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
2156 struct thread *t = &p->threads[p->thread_id];
2157 struct instruction *ip = t->ip;
2159 __instr_hdr_emit7_tx_exec(p, t, ip);
2162 thread_ip_reset(p, t);
2167 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
2169 struct thread *t = &p->threads[p->thread_id];
2170 struct instruction *ip = t->ip;
2172 __instr_hdr_emit8_tx_exec(p, t, ip);
2175 thread_ip_reset(p, t);
2183 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
2184 struct action *action __rte_unused,
2187 struct instruction *instr,
2188 struct instruction_data *data __rte_unused)
2192 CHECK(n_tokens == 2, EINVAL);
2194 h = header_parse(p, tokens[1]);
2197 instr->type = INSTR_HDR_VALIDATE;
2198 instr->valid.header_id = h->id;
2203 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2205 struct thread *t = &p->threads[p->thread_id];
2206 struct instruction *ip = t->ip;
2208 __instr_hdr_validate_exec(p, t, ip);
2218 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2219 struct action *action __rte_unused,
2222 struct instruction *instr,
2223 struct instruction_data *data __rte_unused)
2227 CHECK(n_tokens == 2, EINVAL);
2229 h = header_parse(p, tokens[1]);
2232 instr->type = INSTR_HDR_INVALIDATE;
2233 instr->valid.header_id = h->id;
2238 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2240 struct thread *t = &p->threads[p->thread_id];
2241 struct instruction *ip = t->ip;
2243 __instr_hdr_invalidate_exec(p, t, ip);
2252 static struct table *
2253 table_find(struct rte_swx_pipeline *p, const char *name);
2255 static struct selector *
2256 selector_find(struct rte_swx_pipeline *p, const char *name);
2258 static struct learner *
2259 learner_find(struct rte_swx_pipeline *p, const char *name);
2262 instr_table_translate(struct rte_swx_pipeline *p,
2263 struct action *action,
2266 struct instruction *instr,
2267 struct instruction_data *data __rte_unused)
2273 CHECK(!action, EINVAL);
2274 CHECK(n_tokens == 2, EINVAL);
2276 t = table_find(p, tokens[1]);
2278 instr->type = INSTR_TABLE;
2279 instr->table.table_id = t->id;
2283 s = selector_find(p, tokens[1]);
2285 instr->type = INSTR_SELECTOR;
2286 instr->table.table_id = s->id;
2290 l = learner_find(p, tokens[1]);
2292 instr->type = INSTR_LEARNER;
2293 instr->table.table_id = l->id;
2301 instr_table_exec(struct rte_swx_pipeline *p)
2303 struct thread *t = &p->threads[p->thread_id];
2304 struct instruction *ip = t->ip;
2305 uint32_t table_id = ip->table.table_id;
2306 struct rte_swx_table_state *ts = &t->table_state[table_id];
2307 struct table_runtime *table = &t->tables[table_id];
2308 struct table_statistics *stats = &p->table_stats[table_id];
2309 uint64_t action_id, n_pkts_hit, n_pkts_action;
2310 uint8_t *action_data;
2314 done = table->func(ts->obj,
2322 TRACE("[Thread %2u] table %u (not finalized)\n",
2330 action_id = hit ? action_id : ts->default_action_id;
2331 action_data = hit ? action_data : ts->default_action_data;
2332 n_pkts_hit = stats->n_pkts_hit[hit];
2333 n_pkts_action = stats->n_pkts_action[action_id];
2335 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2338 hit ? "hit" : "miss",
2339 (uint32_t)action_id);
2341 t->action_id = action_id;
2342 t->structs[0] = action_data;
2344 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2345 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2348 thread_ip_action_call(p, t, action_id);
2352 instr_table_af_exec(struct rte_swx_pipeline *p)
2354 struct thread *t = &p->threads[p->thread_id];
2355 struct instruction *ip = t->ip;
2356 uint32_t table_id = ip->table.table_id;
2357 struct rte_swx_table_state *ts = &t->table_state[table_id];
2358 struct table_runtime *table = &t->tables[table_id];
2359 struct table_statistics *stats = &p->table_stats[table_id];
2360 uint64_t action_id, n_pkts_hit, n_pkts_action;
2361 uint8_t *action_data;
2362 action_func_t action_func;
2366 done = table->func(ts->obj,
2374 TRACE("[Thread %2u] table %u (not finalized)\n",
2382 action_id = hit ? action_id : ts->default_action_id;
2383 action_data = hit ? action_data : ts->default_action_data;
2384 action_func = p->action_funcs[action_id];
2385 n_pkts_hit = stats->n_pkts_hit[hit];
2386 n_pkts_action = stats->n_pkts_action[action_id];
2388 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2391 hit ? "hit" : "miss",
2392 (uint32_t)action_id);
2394 t->action_id = action_id;
2395 t->structs[0] = action_data;
2397 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2398 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2408 instr_selector_exec(struct rte_swx_pipeline *p)
2410 struct thread *t = &p->threads[p->thread_id];
2411 struct instruction *ip = t->ip;
2412 uint32_t selector_id = ip->table.table_id;
2413 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2414 struct selector_runtime *selector = &t->selectors[selector_id];
2415 struct selector_statistics *stats = &p->selector_stats[selector_id];
2416 uint64_t n_pkts = stats->n_pkts;
2420 done = rte_swx_table_selector_select(ts->obj,
2422 selector->group_id_buffer,
2423 selector->selector_buffer,
2424 selector->member_id_buffer);
2427 TRACE("[Thread %2u] selector %u (not finalized)\n",
2436 TRACE("[Thread %2u] selector %u\n",
2440 stats->n_pkts = n_pkts + 1;
2447 instr_learner_exec(struct rte_swx_pipeline *p)
2449 struct thread *t = &p->threads[p->thread_id];
2450 struct instruction *ip = t->ip;
2451 uint32_t learner_id = ip->table.table_id;
2452 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2453 p->n_selectors + learner_id];
2454 struct learner_runtime *l = &t->learners[learner_id];
2455 struct learner_statistics *stats = &p->learner_stats[learner_id];
2456 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2457 uint8_t *action_data;
2461 time = rte_get_tsc_cycles();
2463 done = rte_swx_table_learner_lookup(ts->obj,
2472 TRACE("[Thread %2u] learner %u (not finalized)\n",
2480 action_id = hit ? action_id : ts->default_action_id;
2481 action_data = hit ? action_data : ts->default_action_data;
2482 n_pkts_hit = stats->n_pkts_hit[hit];
2483 n_pkts_action = stats->n_pkts_action[action_id];
2485 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2488 hit ? "hit" : "miss",
2489 (uint32_t)action_id);
2491 t->action_id = action_id;
2492 t->structs[0] = action_data;
2494 t->learner_id = learner_id;
2496 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2497 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2500 thread_ip_action_call(p, t, action_id);
2504 instr_learner_af_exec(struct rte_swx_pipeline *p)
2506 struct thread *t = &p->threads[p->thread_id];
2507 struct instruction *ip = t->ip;
2508 uint32_t learner_id = ip->table.table_id;
2509 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2510 p->n_selectors + learner_id];
2511 struct learner_runtime *l = &t->learners[learner_id];
2512 struct learner_statistics *stats = &p->learner_stats[learner_id];
2513 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2514 uint8_t *action_data;
2515 action_func_t action_func;
2519 time = rte_get_tsc_cycles();
2521 done = rte_swx_table_learner_lookup(ts->obj,
2530 TRACE("[Thread %2u] learner %u (not finalized)\n",
2538 action_id = hit ? action_id : ts->default_action_id;
2539 action_data = hit ? action_data : ts->default_action_data;
2540 action_func = p->action_funcs[action_id];
2541 n_pkts_hit = stats->n_pkts_hit[hit];
2542 n_pkts_action = stats->n_pkts_action[action_id];
2544 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2547 hit ? "hit" : "miss",
2548 (uint32_t)action_id);
2550 t->action_id = action_id;
2551 t->structs[0] = action_data;
2553 t->learner_id = learner_id;
2555 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2556 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2559 thread_ip_action_call(p, t, action_id);
2568 static struct action *
2569 action_find(struct rte_swx_pipeline *p, const char *name);
2572 action_has_nbo_args(struct action *a);
2575 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2578 instr_learn_translate(struct rte_swx_pipeline *p,
2579 struct action *action,
2582 struct instruction *instr,
2583 struct instruction_data *data __rte_unused)
2586 const char *mf_name;
2587 uint32_t mf_offset = 0;
2589 CHECK(action, EINVAL);
2590 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2592 a = action_find(p, tokens[1]);
2594 CHECK(!action_has_nbo_args(a), EINVAL);
2596 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2597 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2602 mf = metadata_field_parse(p, mf_name);
2605 mf_offset = mf->offset / 8;
2608 instr->type = INSTR_LEARNER_LEARN;
2609 instr->learn.action_id = a->id;
2610 instr->learn.mf_offset = mf_offset;
2616 instr_learn_exec(struct rte_swx_pipeline *p)
2618 struct thread *t = &p->threads[p->thread_id];
2619 struct instruction *ip = t->ip;
2621 __instr_learn_exec(p, t, ip);
2631 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2632 struct action *action,
2633 char **tokens __rte_unused,
2635 struct instruction *instr,
2636 struct instruction_data *data __rte_unused)
2638 CHECK(action, EINVAL);
2639 CHECK(n_tokens == 1, EINVAL);
2641 instr->type = INSTR_LEARNER_FORGET;
2647 instr_forget_exec(struct rte_swx_pipeline *p)
2649 struct thread *t = &p->threads[p->thread_id];
2650 struct instruction *ip = t->ip;
2652 __instr_forget_exec(p, t, ip);
2662 instr_extern_translate(struct rte_swx_pipeline *p,
2663 struct action *action __rte_unused,
2666 struct instruction *instr,
2667 struct instruction_data *data __rte_unused)
2669 char *token = tokens[1];
2671 CHECK(n_tokens == 2, EINVAL);
2673 if (token[0] == 'e') {
2674 struct extern_obj *obj;
2675 struct extern_type_member_func *func;
2677 func = extern_obj_member_func_parse(p, token, &obj);
2678 CHECK(func, EINVAL);
2680 instr->type = INSTR_EXTERN_OBJ;
2681 instr->ext_obj.ext_obj_id = obj->id;
2682 instr->ext_obj.func_id = func->id;
2687 if (token[0] == 'f') {
2688 struct extern_func *func;
2690 func = extern_func_parse(p, token);
2691 CHECK(func, EINVAL);
2693 instr->type = INSTR_EXTERN_FUNC;
2694 instr->ext_func.ext_func_id = func->id;
2703 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2705 struct thread *t = &p->threads[p->thread_id];
2706 struct instruction *ip = t->ip;
2709 /* Extern object member function execute. */
2710 done = __instr_extern_obj_exec(p, t, ip);
2713 thread_ip_inc_cond(t, done);
2714 thread_yield_cond(p, done ^ 1);
2718 instr_extern_func_exec(struct rte_swx_pipeline *p)
2720 struct thread *t = &p->threads[p->thread_id];
2721 struct instruction *ip = t->ip;
2724 /* Extern function execute. */
2725 done = __instr_extern_func_exec(p, t, ip);
2728 thread_ip_inc_cond(t, done);
2729 thread_yield_cond(p, done ^ 1);
2736 instr_mov_translate(struct rte_swx_pipeline *p,
2737 struct action *action,
2740 struct instruction *instr,
2741 struct instruction_data *data __rte_unused)
2743 char *dst = tokens[1], *src = tokens[2];
2744 struct field *fdst, *fsrc;
2746 uint32_t dst_struct_id = 0, src_struct_id = 0;
2748 CHECK(n_tokens == 3, EINVAL);
2750 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2751 CHECK(fdst, EINVAL);
2752 CHECK(!fdst->var_size, EINVAL);
2754 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2755 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2757 CHECK(!fsrc->var_size, EINVAL);
2759 instr->type = INSTR_MOV;
2760 if (dst[0] != 'h' && src[0] == 'h')
2761 instr->type = INSTR_MOV_MH;
2762 if (dst[0] == 'h' && src[0] != 'h')
2763 instr->type = INSTR_MOV_HM;
2764 if (dst[0] == 'h' && src[0] == 'h')
2765 instr->type = INSTR_MOV_HH;
2767 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2768 instr->mov.dst.n_bits = fdst->n_bits;
2769 instr->mov.dst.offset = fdst->offset / 8;
2770 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2771 instr->mov.src.n_bits = fsrc->n_bits;
2772 instr->mov.src.offset = fsrc->offset / 8;
2777 src_val = strtoull(src, &src, 0);
2778 CHECK(!src[0], EINVAL);
2781 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2783 instr->type = INSTR_MOV_I;
2784 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2785 instr->mov.dst.n_bits = fdst->n_bits;
2786 instr->mov.dst.offset = fdst->offset / 8;
2787 instr->mov.src_val = src_val;
2792 instr_mov_exec(struct rte_swx_pipeline *p)
2794 struct thread *t = &p->threads[p->thread_id];
2795 struct instruction *ip = t->ip;
2797 __instr_mov_exec(p, t, ip);
2804 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2806 struct thread *t = &p->threads[p->thread_id];
2807 struct instruction *ip = t->ip;
2809 __instr_mov_mh_exec(p, t, ip);
2816 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2818 struct thread *t = &p->threads[p->thread_id];
2819 struct instruction *ip = t->ip;
2821 __instr_mov_hm_exec(p, t, ip);
2828 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2830 struct thread *t = &p->threads[p->thread_id];
2831 struct instruction *ip = t->ip;
2833 __instr_mov_hh_exec(p, t, ip);
2840 instr_mov_i_exec(struct rte_swx_pipeline *p)
2842 struct thread *t = &p->threads[p->thread_id];
2843 struct instruction *ip = t->ip;
2845 __instr_mov_i_exec(p, t, ip);
2855 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2857 struct thread *t = &p->threads[p->thread_id];
2858 struct instruction *ip = t->ip;
2860 __instr_dma_ht_exec(p, t, ip);
2867 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2869 struct thread *t = &p->threads[p->thread_id];
2870 struct instruction *ip = t->ip;
2872 __instr_dma_ht2_exec(p, t, ip);
2879 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2881 struct thread *t = &p->threads[p->thread_id];
2882 struct instruction *ip = t->ip;
2884 __instr_dma_ht3_exec(p, t, ip);
2891 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2893 struct thread *t = &p->threads[p->thread_id];
2894 struct instruction *ip = t->ip;
2896 __instr_dma_ht4_exec(p, t, ip);
2903 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2905 struct thread *t = &p->threads[p->thread_id];
2906 struct instruction *ip = t->ip;
2908 __instr_dma_ht5_exec(p, t, ip);
2915 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2917 struct thread *t = &p->threads[p->thread_id];
2918 struct instruction *ip = t->ip;
2920 __instr_dma_ht6_exec(p, t, ip);
2927 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2929 struct thread *t = &p->threads[p->thread_id];
2930 struct instruction *ip = t->ip;
2932 __instr_dma_ht7_exec(p, t, ip);
2939 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2941 struct thread *t = &p->threads[p->thread_id];
2942 struct instruction *ip = t->ip;
2944 __instr_dma_ht8_exec(p, t, ip);
2954 instr_alu_add_translate(struct rte_swx_pipeline *p,
2955 struct action *action,
2958 struct instruction *instr,
2959 struct instruction_data *data __rte_unused)
2961 char *dst = tokens[1], *src = tokens[2];
2962 struct field *fdst, *fsrc;
2964 uint32_t dst_struct_id = 0, src_struct_id = 0;
2966 CHECK(n_tokens == 3, EINVAL);
2968 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2969 CHECK(fdst, EINVAL);
2970 CHECK(!fdst->var_size, EINVAL);
2972 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2973 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2975 CHECK(!fsrc->var_size, EINVAL);
2977 instr->type = INSTR_ALU_ADD;
2978 if (dst[0] == 'h' && src[0] != 'h')
2979 instr->type = INSTR_ALU_ADD_HM;
2980 if (dst[0] != 'h' && src[0] == 'h')
2981 instr->type = INSTR_ALU_ADD_MH;
2982 if (dst[0] == 'h' && src[0] == 'h')
2983 instr->type = INSTR_ALU_ADD_HH;
2985 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2986 instr->alu.dst.n_bits = fdst->n_bits;
2987 instr->alu.dst.offset = fdst->offset / 8;
2988 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2989 instr->alu.src.n_bits = fsrc->n_bits;
2990 instr->alu.src.offset = fsrc->offset / 8;
2994 /* ADD_MI, ADD_HI. */
2995 src_val = strtoull(src, &src, 0);
2996 CHECK(!src[0], EINVAL);
2998 instr->type = INSTR_ALU_ADD_MI;
3000 instr->type = INSTR_ALU_ADD_HI;
3002 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3003 instr->alu.dst.n_bits = fdst->n_bits;
3004 instr->alu.dst.offset = fdst->offset / 8;
3005 instr->alu.src_val = src_val;
3010 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3011 struct action *action,
3014 struct instruction *instr,
3015 struct instruction_data *data __rte_unused)
3017 char *dst = tokens[1], *src = tokens[2];
3018 struct field *fdst, *fsrc;
3020 uint32_t dst_struct_id = 0, src_struct_id = 0;
3022 CHECK(n_tokens == 3, EINVAL);
3024 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3025 CHECK(fdst, EINVAL);
3026 CHECK(!fdst->var_size, EINVAL);
3028 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3029 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3031 CHECK(!fsrc->var_size, EINVAL);
3033 instr->type = INSTR_ALU_SUB;
3034 if (dst[0] == 'h' && src[0] != 'h')
3035 instr->type = INSTR_ALU_SUB_HM;
3036 if (dst[0] != 'h' && src[0] == 'h')
3037 instr->type = INSTR_ALU_SUB_MH;
3038 if (dst[0] == 'h' && src[0] == 'h')
3039 instr->type = INSTR_ALU_SUB_HH;
3041 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3042 instr->alu.dst.n_bits = fdst->n_bits;
3043 instr->alu.dst.offset = fdst->offset / 8;
3044 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3045 instr->alu.src.n_bits = fsrc->n_bits;
3046 instr->alu.src.offset = fsrc->offset / 8;
3050 /* SUB_MI, SUB_HI. */
3051 src_val = strtoull(src, &src, 0);
3052 CHECK(!src[0], EINVAL);
3054 instr->type = INSTR_ALU_SUB_MI;
3056 instr->type = INSTR_ALU_SUB_HI;
3058 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3059 instr->alu.dst.n_bits = fdst->n_bits;
3060 instr->alu.dst.offset = fdst->offset / 8;
3061 instr->alu.src_val = src_val;
3066 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3067 struct action *action __rte_unused,
3070 struct instruction *instr,
3071 struct instruction_data *data __rte_unused)
3073 char *dst = tokens[1], *src = tokens[2];
3074 struct header *hdst, *hsrc;
3075 struct field *fdst, *fsrc;
3077 CHECK(n_tokens == 3, EINVAL);
3079 fdst = header_field_parse(p, dst, &hdst);
3080 CHECK(fdst, EINVAL);
3081 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3084 fsrc = header_field_parse(p, src, &hsrc);
3086 CHECK(!fsrc->var_size, EINVAL);
3088 instr->type = INSTR_ALU_CKADD_FIELD;
3089 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3090 instr->alu.dst.n_bits = fdst->n_bits;
3091 instr->alu.dst.offset = fdst->offset / 8;
3092 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3093 instr->alu.src.n_bits = fsrc->n_bits;
3094 instr->alu.src.offset = fsrc->offset / 8;
3098 /* CKADD_STRUCT, CKADD_STRUCT20. */
3099 hsrc = header_parse(p, src);
3100 CHECK(hsrc, EINVAL);
3102 instr->type = INSTR_ALU_CKADD_STRUCT;
3103 if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
3104 instr->type = INSTR_ALU_CKADD_STRUCT20;
3106 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3107 instr->alu.dst.n_bits = fdst->n_bits;
3108 instr->alu.dst.offset = fdst->offset / 8;
3109 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3110 instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
3111 instr->alu.src.offset = 0; /* Unused. */
3116 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3117 struct action *action __rte_unused,
3120 struct instruction *instr,
3121 struct instruction_data *data __rte_unused)
3123 char *dst = tokens[1], *src = tokens[2];
3124 struct header *hdst, *hsrc;
3125 struct field *fdst, *fsrc;
3127 CHECK(n_tokens == 3, EINVAL);
3129 fdst = header_field_parse(p, dst, &hdst);
3130 CHECK(fdst, EINVAL);
3131 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
3133 fsrc = header_field_parse(p, src, &hsrc);
3134 CHECK(fsrc, EINVAL);
3135 CHECK(!fsrc->var_size, EINVAL);
3137 instr->type = INSTR_ALU_CKSUB_FIELD;
3138 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3139 instr->alu.dst.n_bits = fdst->n_bits;
3140 instr->alu.dst.offset = fdst->offset / 8;
3141 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3142 instr->alu.src.n_bits = fsrc->n_bits;
3143 instr->alu.src.offset = fsrc->offset / 8;
3148 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3149 struct action *action,
3152 struct instruction *instr,
3153 struct instruction_data *data __rte_unused)
3155 char *dst = tokens[1], *src = tokens[2];
3156 struct field *fdst, *fsrc;
3158 uint32_t dst_struct_id = 0, src_struct_id = 0;
3160 CHECK(n_tokens == 3, EINVAL);
3162 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3163 CHECK(fdst, EINVAL);
3164 CHECK(!fdst->var_size, EINVAL);
3166 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3167 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3169 CHECK(!fsrc->var_size, EINVAL);
3171 instr->type = INSTR_ALU_SHL;
3172 if (dst[0] == 'h' && src[0] != 'h')
3173 instr->type = INSTR_ALU_SHL_HM;
3174 if (dst[0] != 'h' && src[0] == 'h')
3175 instr->type = INSTR_ALU_SHL_MH;
3176 if (dst[0] == 'h' && src[0] == 'h')
3177 instr->type = INSTR_ALU_SHL_HH;
3179 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3180 instr->alu.dst.n_bits = fdst->n_bits;
3181 instr->alu.dst.offset = fdst->offset / 8;
3182 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3183 instr->alu.src.n_bits = fsrc->n_bits;
3184 instr->alu.src.offset = fsrc->offset / 8;
3188 /* SHL_MI, SHL_HI. */
3189 src_val = strtoull(src, &src, 0);
3190 CHECK(!src[0], EINVAL);
3192 instr->type = INSTR_ALU_SHL_MI;
3194 instr->type = INSTR_ALU_SHL_HI;
3196 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3197 instr->alu.dst.n_bits = fdst->n_bits;
3198 instr->alu.dst.offset = fdst->offset / 8;
3199 instr->alu.src_val = src_val;
3204 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3205 struct action *action,
3208 struct instruction *instr,
3209 struct instruction_data *data __rte_unused)
3211 char *dst = tokens[1], *src = tokens[2];
3212 struct field *fdst, *fsrc;
3214 uint32_t dst_struct_id = 0, src_struct_id = 0;
3216 CHECK(n_tokens == 3, EINVAL);
3218 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3219 CHECK(fdst, EINVAL);
3220 CHECK(!fdst->var_size, EINVAL);
3222 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3223 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3225 CHECK(!fsrc->var_size, EINVAL);
3227 instr->type = INSTR_ALU_SHR;
3228 if (dst[0] == 'h' && src[0] != 'h')
3229 instr->type = INSTR_ALU_SHR_HM;
3230 if (dst[0] != 'h' && src[0] == 'h')
3231 instr->type = INSTR_ALU_SHR_MH;
3232 if (dst[0] == 'h' && src[0] == 'h')
3233 instr->type = INSTR_ALU_SHR_HH;
3235 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3236 instr->alu.dst.n_bits = fdst->n_bits;
3237 instr->alu.dst.offset = fdst->offset / 8;
3238 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3239 instr->alu.src.n_bits = fsrc->n_bits;
3240 instr->alu.src.offset = fsrc->offset / 8;
3244 /* SHR_MI, SHR_HI. */
3245 src_val = strtoull(src, &src, 0);
3246 CHECK(!src[0], EINVAL);
3248 instr->type = INSTR_ALU_SHR_MI;
3250 instr->type = INSTR_ALU_SHR_HI;
3252 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3253 instr->alu.dst.n_bits = fdst->n_bits;
3254 instr->alu.dst.offset = fdst->offset / 8;
3255 instr->alu.src_val = src_val;
3260 instr_alu_and_translate(struct rte_swx_pipeline *p,
3261 struct action *action,
3264 struct instruction *instr,
3265 struct instruction_data *data __rte_unused)
3267 char *dst = tokens[1], *src = tokens[2];
3268 struct field *fdst, *fsrc;
3270 uint32_t dst_struct_id = 0, src_struct_id = 0;
3272 CHECK(n_tokens == 3, EINVAL);
3274 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3275 CHECK(fdst, EINVAL);
3276 CHECK(!fdst->var_size, EINVAL);
3278 /* AND, AND_MH, AND_HM, AND_HH. */
3279 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3281 CHECK(!fsrc->var_size, EINVAL);
3283 instr->type = INSTR_ALU_AND;
3284 if (dst[0] != 'h' && src[0] == 'h')
3285 instr->type = INSTR_ALU_AND_MH;
3286 if (dst[0] == 'h' && src[0] != 'h')
3287 instr->type = INSTR_ALU_AND_HM;
3288 if (dst[0] == 'h' && src[0] == 'h')
3289 instr->type = INSTR_ALU_AND_HH;
3291 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3292 instr->alu.dst.n_bits = fdst->n_bits;
3293 instr->alu.dst.offset = fdst->offset / 8;
3294 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3295 instr->alu.src.n_bits = fsrc->n_bits;
3296 instr->alu.src.offset = fsrc->offset / 8;
3301 src_val = strtoull(src, &src, 0);
3302 CHECK(!src[0], EINVAL);
3305 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3307 instr->type = INSTR_ALU_AND_I;
3308 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3309 instr->alu.dst.n_bits = fdst->n_bits;
3310 instr->alu.dst.offset = fdst->offset / 8;
3311 instr->alu.src_val = src_val;
3316 instr_alu_or_translate(struct rte_swx_pipeline *p,
3317 struct action *action,
3320 struct instruction *instr,
3321 struct instruction_data *data __rte_unused)
3323 char *dst = tokens[1], *src = tokens[2];
3324 struct field *fdst, *fsrc;
3326 uint32_t dst_struct_id = 0, src_struct_id = 0;
3328 CHECK(n_tokens == 3, EINVAL);
3330 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3331 CHECK(fdst, EINVAL);
3332 CHECK(!fdst->var_size, EINVAL);
3334 /* OR, OR_MH, OR_HM, OR_HH. */
3335 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3337 CHECK(!fsrc->var_size, EINVAL);
3339 instr->type = INSTR_ALU_OR;
3340 if (dst[0] != 'h' && src[0] == 'h')
3341 instr->type = INSTR_ALU_OR_MH;
3342 if (dst[0] == 'h' && src[0] != 'h')
3343 instr->type = INSTR_ALU_OR_HM;
3344 if (dst[0] == 'h' && src[0] == 'h')
3345 instr->type = INSTR_ALU_OR_HH;
3347 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3348 instr->alu.dst.n_bits = fdst->n_bits;
3349 instr->alu.dst.offset = fdst->offset / 8;
3350 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3351 instr->alu.src.n_bits = fsrc->n_bits;
3352 instr->alu.src.offset = fsrc->offset / 8;
3357 src_val = strtoull(src, &src, 0);
3358 CHECK(!src[0], EINVAL);
3361 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3363 instr->type = INSTR_ALU_OR_I;
3364 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3365 instr->alu.dst.n_bits = fdst->n_bits;
3366 instr->alu.dst.offset = fdst->offset / 8;
3367 instr->alu.src_val = src_val;
3372 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3373 struct action *action,
3376 struct instruction *instr,
3377 struct instruction_data *data __rte_unused)
3379 char *dst = tokens[1], *src = tokens[2];
3380 struct field *fdst, *fsrc;
3382 uint32_t dst_struct_id = 0, src_struct_id = 0;
3384 CHECK(n_tokens == 3, EINVAL);
3386 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3387 CHECK(fdst, EINVAL);
3388 CHECK(!fdst->var_size, EINVAL);
3390 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3391 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3393 CHECK(!fsrc->var_size, EINVAL);
3395 instr->type = INSTR_ALU_XOR;
3396 if (dst[0] != 'h' && src[0] == 'h')
3397 instr->type = INSTR_ALU_XOR_MH;
3398 if (dst[0] == 'h' && src[0] != 'h')
3399 instr->type = INSTR_ALU_XOR_HM;
3400 if (dst[0] == 'h' && src[0] == 'h')
3401 instr->type = INSTR_ALU_XOR_HH;
3403 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3404 instr->alu.dst.n_bits = fdst->n_bits;
3405 instr->alu.dst.offset = fdst->offset / 8;
3406 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3407 instr->alu.src.n_bits = fsrc->n_bits;
3408 instr->alu.src.offset = fsrc->offset / 8;
3413 src_val = strtoull(src, &src, 0);
3414 CHECK(!src[0], EINVAL);
3417 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3419 instr->type = INSTR_ALU_XOR_I;
3420 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3421 instr->alu.dst.n_bits = fdst->n_bits;
3422 instr->alu.dst.offset = fdst->offset / 8;
3423 instr->alu.src_val = src_val;
3428 instr_alu_add_exec(struct rte_swx_pipeline *p)
3430 struct thread *t = &p->threads[p->thread_id];
3431 struct instruction *ip = t->ip;
3434 __instr_alu_add_exec(p, t, ip);
3441 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3443 struct thread *t = &p->threads[p->thread_id];
3444 struct instruction *ip = t->ip;
3447 __instr_alu_add_mh_exec(p, t, ip);
3454 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3456 struct thread *t = &p->threads[p->thread_id];
3457 struct instruction *ip = t->ip;
3460 __instr_alu_add_hm_exec(p, t, ip);
3467 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3469 struct thread *t = &p->threads[p->thread_id];
3470 struct instruction *ip = t->ip;
3473 __instr_alu_add_hh_exec(p, t, ip);
3480 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3482 struct thread *t = &p->threads[p->thread_id];
3483 struct instruction *ip = t->ip;
3486 __instr_alu_add_mi_exec(p, t, ip);
3493 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3495 struct thread *t = &p->threads[p->thread_id];
3496 struct instruction *ip = t->ip;
3499 __instr_alu_add_hi_exec(p, t, ip);
3506 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3508 struct thread *t = &p->threads[p->thread_id];
3509 struct instruction *ip = t->ip;
3512 __instr_alu_sub_exec(p, t, ip);
3519 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3521 struct thread *t = &p->threads[p->thread_id];
3522 struct instruction *ip = t->ip;
3525 __instr_alu_sub_mh_exec(p, t, ip);
3532 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3534 struct thread *t = &p->threads[p->thread_id];
3535 struct instruction *ip = t->ip;
3538 __instr_alu_sub_hm_exec(p, t, ip);
3545 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3547 struct thread *t = &p->threads[p->thread_id];
3548 struct instruction *ip = t->ip;
3551 __instr_alu_sub_hh_exec(p, t, ip);
3558 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3560 struct thread *t = &p->threads[p->thread_id];
3561 struct instruction *ip = t->ip;
3564 __instr_alu_sub_mi_exec(p, t, ip);
3571 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3573 struct thread *t = &p->threads[p->thread_id];
3574 struct instruction *ip = t->ip;
3577 __instr_alu_sub_hi_exec(p, t, ip);
3584 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3586 struct thread *t = &p->threads[p->thread_id];
3587 struct instruction *ip = t->ip;
3590 __instr_alu_shl_exec(p, t, ip);
3597 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3599 struct thread *t = &p->threads[p->thread_id];
3600 struct instruction *ip = t->ip;
3603 __instr_alu_shl_mh_exec(p, t, ip);
3610 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3612 struct thread *t = &p->threads[p->thread_id];
3613 struct instruction *ip = t->ip;
3616 __instr_alu_shl_hm_exec(p, t, ip);
3623 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3625 struct thread *t = &p->threads[p->thread_id];
3626 struct instruction *ip = t->ip;
3629 __instr_alu_shl_hh_exec(p, t, ip);
3636 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3638 struct thread *t = &p->threads[p->thread_id];
3639 struct instruction *ip = t->ip;
3642 __instr_alu_shl_mi_exec(p, t, ip);
3649 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3651 struct thread *t = &p->threads[p->thread_id];
3652 struct instruction *ip = t->ip;
3655 __instr_alu_shl_hi_exec(p, t, ip);
3662 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3664 struct thread *t = &p->threads[p->thread_id];
3665 struct instruction *ip = t->ip;
3668 __instr_alu_shr_exec(p, t, ip);
3675 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3677 struct thread *t = &p->threads[p->thread_id];
3678 struct instruction *ip = t->ip;
3681 __instr_alu_shr_mh_exec(p, t, ip);
3688 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3690 struct thread *t = &p->threads[p->thread_id];
3691 struct instruction *ip = t->ip;
3694 __instr_alu_shr_hm_exec(p, t, ip);
3701 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3703 struct thread *t = &p->threads[p->thread_id];
3704 struct instruction *ip = t->ip;
3707 __instr_alu_shr_hh_exec(p, t, ip);
3714 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3716 struct thread *t = &p->threads[p->thread_id];
3717 struct instruction *ip = t->ip;
3720 __instr_alu_shr_mi_exec(p, t, ip);
3727 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3729 struct thread *t = &p->threads[p->thread_id];
3730 struct instruction *ip = t->ip;
3733 __instr_alu_shr_hi_exec(p, t, ip);
3740 instr_alu_and_exec(struct rte_swx_pipeline *p)
3742 struct thread *t = &p->threads[p->thread_id];
3743 struct instruction *ip = t->ip;
3746 __instr_alu_and_exec(p, t, ip);
3753 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3755 struct thread *t = &p->threads[p->thread_id];
3756 struct instruction *ip = t->ip;
3759 __instr_alu_and_mh_exec(p, t, ip);
3766 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3768 struct thread *t = &p->threads[p->thread_id];
3769 struct instruction *ip = t->ip;
3772 __instr_alu_and_hm_exec(p, t, ip);
3779 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3781 struct thread *t = &p->threads[p->thread_id];
3782 struct instruction *ip = t->ip;
3785 __instr_alu_and_hh_exec(p, t, ip);
3792 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3794 struct thread *t = &p->threads[p->thread_id];
3795 struct instruction *ip = t->ip;
3798 __instr_alu_and_i_exec(p, t, ip);
3805 instr_alu_or_exec(struct rte_swx_pipeline *p)
3807 struct thread *t = &p->threads[p->thread_id];
3808 struct instruction *ip = t->ip;
3811 __instr_alu_or_exec(p, t, ip);
3818 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3820 struct thread *t = &p->threads[p->thread_id];
3821 struct instruction *ip = t->ip;
3824 __instr_alu_or_mh_exec(p, t, ip);
3831 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3833 struct thread *t = &p->threads[p->thread_id];
3834 struct instruction *ip = t->ip;
3837 __instr_alu_or_hm_exec(p, t, ip);
3844 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3846 struct thread *t = &p->threads[p->thread_id];
3847 struct instruction *ip = t->ip;
3850 __instr_alu_or_hh_exec(p, t, ip);
3857 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3859 struct thread *t = &p->threads[p->thread_id];
3860 struct instruction *ip = t->ip;
3863 __instr_alu_or_i_exec(p, t, ip);
3870 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3872 struct thread *t = &p->threads[p->thread_id];
3873 struct instruction *ip = t->ip;
3876 __instr_alu_xor_exec(p, t, ip);
3883 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3885 struct thread *t = &p->threads[p->thread_id];
3886 struct instruction *ip = t->ip;
3889 __instr_alu_xor_mh_exec(p, t, ip);
3896 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3898 struct thread *t = &p->threads[p->thread_id];
3899 struct instruction *ip = t->ip;
3902 __instr_alu_xor_hm_exec(p, t, ip);
3909 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3911 struct thread *t = &p->threads[p->thread_id];
3912 struct instruction *ip = t->ip;
3915 __instr_alu_xor_hh_exec(p, t, ip);
3922 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3924 struct thread *t = &p->threads[p->thread_id];
3925 struct instruction *ip = t->ip;
3928 __instr_alu_xor_i_exec(p, t, ip);
3935 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3937 struct thread *t = &p->threads[p->thread_id];
3938 struct instruction *ip = t->ip;
3941 __instr_alu_ckadd_field_exec(p, t, ip);
3948 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3950 struct thread *t = &p->threads[p->thread_id];
3951 struct instruction *ip = t->ip;
3954 __instr_alu_cksub_field_exec(p, t, ip);
3961 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3963 struct thread *t = &p->threads[p->thread_id];
3964 struct instruction *ip = t->ip;
3967 __instr_alu_ckadd_struct20_exec(p, t, ip);
3974 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3976 struct thread *t = &p->threads[p->thread_id];
3977 struct instruction *ip = t->ip;
3980 __instr_alu_ckadd_struct_exec(p, t, ip);
3989 static struct regarray *
3990 regarray_find(struct rte_swx_pipeline *p, const char *name);
3993 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3994 struct action *action,
3997 struct instruction *instr,
3998 struct instruction_data *data __rte_unused)
4000 char *regarray = tokens[1], *idx = tokens[2];
4003 uint32_t idx_struct_id, idx_val;
4005 CHECK(n_tokens == 3, EINVAL);
4007 r = regarray_find(p, regarray);
4010 /* REGPREFETCH_RH, REGPREFETCH_RM. */
4011 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4013 CHECK(!fidx->var_size, EINVAL);
4015 instr->type = INSTR_REGPREFETCH_RM;
4017 instr->type = INSTR_REGPREFETCH_RH;
4019 instr->regarray.regarray_id = r->id;
4020 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4021 instr->regarray.idx.n_bits = fidx->n_bits;
4022 instr->regarray.idx.offset = fidx->offset / 8;
4023 instr->regarray.dstsrc_val = 0; /* Unused. */
4027 /* REGPREFETCH_RI. */
4028 idx_val = strtoul(idx, &idx, 0);
4029 CHECK(!idx[0], EINVAL);
4031 instr->type = INSTR_REGPREFETCH_RI;
4032 instr->regarray.regarray_id = r->id;
4033 instr->regarray.idx_val = idx_val;
4034 instr->regarray.dstsrc_val = 0; /* Unused. */
4039 instr_regrd_translate(struct rte_swx_pipeline *p,
4040 struct action *action,
4043 struct instruction *instr,
4044 struct instruction_data *data __rte_unused)
4046 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4048 struct field *fdst, *fidx;
4049 uint32_t dst_struct_id, idx_struct_id, idx_val;
4051 CHECK(n_tokens == 4, EINVAL);
4053 r = regarray_find(p, regarray);
4056 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4057 CHECK(fdst, EINVAL);
4058 CHECK(!fdst->var_size, EINVAL);
4060 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4061 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4063 CHECK(!fidx->var_size, EINVAL);
4065 instr->type = INSTR_REGRD_MRM;
4066 if (dst[0] == 'h' && idx[0] != 'h')
4067 instr->type = INSTR_REGRD_HRM;
4068 if (dst[0] != 'h' && idx[0] == 'h')
4069 instr->type = INSTR_REGRD_MRH;
4070 if (dst[0] == 'h' && idx[0] == 'h')
4071 instr->type = INSTR_REGRD_HRH;
4073 instr->regarray.regarray_id = r->id;
4074 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4075 instr->regarray.idx.n_bits = fidx->n_bits;
4076 instr->regarray.idx.offset = fidx->offset / 8;
4077 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4078 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4079 instr->regarray.dstsrc.offset = fdst->offset / 8;
4083 /* REGRD_MRI, REGRD_HRI. */
4084 idx_val = strtoul(idx, &idx, 0);
4085 CHECK(!idx[0], EINVAL);
4087 instr->type = INSTR_REGRD_MRI;
4089 instr->type = INSTR_REGRD_HRI;
4091 instr->regarray.regarray_id = r->id;
4092 instr->regarray.idx_val = idx_val;
4093 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4094 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4095 instr->regarray.dstsrc.offset = fdst->offset / 8;
4100 instr_regwr_translate(struct rte_swx_pipeline *p,
4101 struct action *action,
4104 struct instruction *instr,
4105 struct instruction_data *data __rte_unused)
4107 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4109 struct field *fidx, *fsrc;
4111 uint32_t idx_struct_id, idx_val, src_struct_id;
4113 CHECK(n_tokens == 4, EINVAL);
4115 r = regarray_find(p, regarray);
4118 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4119 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4120 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4122 CHECK(!fidx->var_size, EINVAL);
4123 CHECK(!fsrc->var_size, EINVAL);
4125 instr->type = INSTR_REGWR_RMM;
4126 if (idx[0] == 'h' && src[0] != 'h')
4127 instr->type = INSTR_REGWR_RHM;
4128 if (idx[0] != 'h' && src[0] == 'h')
4129 instr->type = INSTR_REGWR_RMH;
4130 if (idx[0] == 'h' && src[0] == 'h')
4131 instr->type = INSTR_REGWR_RHH;
4133 instr->regarray.regarray_id = r->id;
4134 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4135 instr->regarray.idx.n_bits = fidx->n_bits;
4136 instr->regarray.idx.offset = fidx->offset / 8;
4137 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4138 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4139 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4143 /* REGWR_RHI, REGWR_RMI. */
4144 if (fidx && !fsrc) {
4145 CHECK(!fidx->var_size, EINVAL);
4147 src_val = strtoull(src, &src, 0);
4148 CHECK(!src[0], EINVAL);
4150 instr->type = INSTR_REGWR_RMI;
4152 instr->type = INSTR_REGWR_RHI;
4154 instr->regarray.regarray_id = r->id;
4155 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4156 instr->regarray.idx.n_bits = fidx->n_bits;
4157 instr->regarray.idx.offset = fidx->offset / 8;
4158 instr->regarray.dstsrc_val = src_val;
4162 /* REGWR_RIH, REGWR_RIM. */
4163 if (!fidx && fsrc) {
4164 idx_val = strtoul(idx, &idx, 0);
4165 CHECK(!idx[0], EINVAL);
4167 CHECK(!fsrc->var_size, EINVAL);
4169 instr->type = INSTR_REGWR_RIM;
4171 instr->type = INSTR_REGWR_RIH;
4173 instr->regarray.regarray_id = r->id;
4174 instr->regarray.idx_val = idx_val;
4175 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4176 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4177 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4182 src_val = strtoull(src, &src, 0);
4183 CHECK(!src[0], EINVAL);
4185 idx_val = strtoul(idx, &idx, 0);
4186 CHECK(!idx[0], EINVAL);
4188 instr->type = INSTR_REGWR_RII;
4189 instr->regarray.idx_val = idx_val;
4190 instr->regarray.dstsrc_val = src_val;
4196 instr_regadd_translate(struct rte_swx_pipeline *p,
4197 struct action *action,
4200 struct instruction *instr,
4201 struct instruction_data *data __rte_unused)
4203 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4205 struct field *fidx, *fsrc;
4207 uint32_t idx_struct_id, idx_val, src_struct_id;
4209 CHECK(n_tokens == 4, EINVAL);
4211 r = regarray_find(p, regarray);
4214 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4215 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4216 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4218 CHECK(!fidx->var_size, EINVAL);
4219 CHECK(!fsrc->var_size, EINVAL);
4221 instr->type = INSTR_REGADD_RMM;
4222 if (idx[0] == 'h' && src[0] != 'h')
4223 instr->type = INSTR_REGADD_RHM;
4224 if (idx[0] != 'h' && src[0] == 'h')
4225 instr->type = INSTR_REGADD_RMH;
4226 if (idx[0] == 'h' && src[0] == 'h')
4227 instr->type = INSTR_REGADD_RHH;
4229 instr->regarray.regarray_id = r->id;
4230 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4231 instr->regarray.idx.n_bits = fidx->n_bits;
4232 instr->regarray.idx.offset = fidx->offset / 8;
4233 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4234 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4235 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4239 /* REGADD_RHI, REGADD_RMI. */
4240 if (fidx && !fsrc) {
4241 CHECK(!fidx->var_size, EINVAL);
4243 src_val = strtoull(src, &src, 0);
4244 CHECK(!src[0], EINVAL);
4246 instr->type = INSTR_REGADD_RMI;
4248 instr->type = INSTR_REGADD_RHI;
4250 instr->regarray.regarray_id = r->id;
4251 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4252 instr->regarray.idx.n_bits = fidx->n_bits;
4253 instr->regarray.idx.offset = fidx->offset / 8;
4254 instr->regarray.dstsrc_val = src_val;
4258 /* REGADD_RIH, REGADD_RIM. */
4259 if (!fidx && fsrc) {
4260 idx_val = strtoul(idx, &idx, 0);
4261 CHECK(!idx[0], EINVAL);
4263 CHECK(!fsrc->var_size, EINVAL);
4265 instr->type = INSTR_REGADD_RIM;
4267 instr->type = INSTR_REGADD_RIH;
4269 instr->regarray.regarray_id = r->id;
4270 instr->regarray.idx_val = idx_val;
4271 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4272 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4273 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4278 src_val = strtoull(src, &src, 0);
4279 CHECK(!src[0], EINVAL);
4281 idx_val = strtoul(idx, &idx, 0);
4282 CHECK(!idx[0], EINVAL);
4284 instr->type = INSTR_REGADD_RII;
4285 instr->regarray.idx_val = idx_val;
4286 instr->regarray.dstsrc_val = src_val;
4291 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4293 struct thread *t = &p->threads[p->thread_id];
4294 struct instruction *ip = t->ip;
4297 __instr_regprefetch_rh_exec(p, t, ip);
4304 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4306 struct thread *t = &p->threads[p->thread_id];
4307 struct instruction *ip = t->ip;
4310 __instr_regprefetch_rm_exec(p, t, ip);
4317 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4319 struct thread *t = &p->threads[p->thread_id];
4320 struct instruction *ip = t->ip;
4323 __instr_regprefetch_ri_exec(p, t, ip);
4330 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4332 struct thread *t = &p->threads[p->thread_id];
4333 struct instruction *ip = t->ip;
4336 __instr_regrd_hrh_exec(p, t, ip);
4343 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4345 struct thread *t = &p->threads[p->thread_id];
4346 struct instruction *ip = t->ip;
4349 __instr_regrd_hrm_exec(p, t, ip);
4356 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4358 struct thread *t = &p->threads[p->thread_id];
4359 struct instruction *ip = t->ip;
4362 __instr_regrd_mrh_exec(p, t, ip);
4369 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4371 struct thread *t = &p->threads[p->thread_id];
4372 struct instruction *ip = t->ip;
4375 __instr_regrd_mrm_exec(p, t, ip);
4382 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4384 struct thread *t = &p->threads[p->thread_id];
4385 struct instruction *ip = t->ip;
4388 __instr_regrd_hri_exec(p, t, ip);
4395 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4397 struct thread *t = &p->threads[p->thread_id];
4398 struct instruction *ip = t->ip;
4401 __instr_regrd_mri_exec(p, t, ip);
4408 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4410 struct thread *t = &p->threads[p->thread_id];
4411 struct instruction *ip = t->ip;
4414 __instr_regwr_rhh_exec(p, t, ip);
4421 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4423 struct thread *t = &p->threads[p->thread_id];
4424 struct instruction *ip = t->ip;
4427 __instr_regwr_rhm_exec(p, t, ip);
4434 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4436 struct thread *t = &p->threads[p->thread_id];
4437 struct instruction *ip = t->ip;
4440 __instr_regwr_rmh_exec(p, t, ip);
4447 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4449 struct thread *t = &p->threads[p->thread_id];
4450 struct instruction *ip = t->ip;
4453 __instr_regwr_rmm_exec(p, t, ip);
4460 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4462 struct thread *t = &p->threads[p->thread_id];
4463 struct instruction *ip = t->ip;
4466 __instr_regwr_rhi_exec(p, t, ip);
4473 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4475 struct thread *t = &p->threads[p->thread_id];
4476 struct instruction *ip = t->ip;
4479 __instr_regwr_rmi_exec(p, t, ip);
4486 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4488 struct thread *t = &p->threads[p->thread_id];
4489 struct instruction *ip = t->ip;
4492 __instr_regwr_rih_exec(p, t, ip);
4499 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4501 struct thread *t = &p->threads[p->thread_id];
4502 struct instruction *ip = t->ip;
4505 __instr_regwr_rim_exec(p, t, ip);
4512 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4514 struct thread *t = &p->threads[p->thread_id];
4515 struct instruction *ip = t->ip;
4518 __instr_regwr_rii_exec(p, t, ip);
4525 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4527 struct thread *t = &p->threads[p->thread_id];
4528 struct instruction *ip = t->ip;
4531 __instr_regadd_rhh_exec(p, t, ip);
4538 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4540 struct thread *t = &p->threads[p->thread_id];
4541 struct instruction *ip = t->ip;
4544 __instr_regadd_rhm_exec(p, t, ip);
4551 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4553 struct thread *t = &p->threads[p->thread_id];
4554 struct instruction *ip = t->ip;
4557 __instr_regadd_rmh_exec(p, t, ip);
4564 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4566 struct thread *t = &p->threads[p->thread_id];
4567 struct instruction *ip = t->ip;
4570 __instr_regadd_rmm_exec(p, t, ip);
4577 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4579 struct thread *t = &p->threads[p->thread_id];
4580 struct instruction *ip = t->ip;
4583 __instr_regadd_rhi_exec(p, t, ip);
4590 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4592 struct thread *t = &p->threads[p->thread_id];
4593 struct instruction *ip = t->ip;
4596 __instr_regadd_rmi_exec(p, t, ip);
4603 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4605 struct thread *t = &p->threads[p->thread_id];
4606 struct instruction *ip = t->ip;
4609 __instr_regadd_rih_exec(p, t, ip);
4616 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4618 struct thread *t = &p->threads[p->thread_id];
4619 struct instruction *ip = t->ip;
4622 __instr_regadd_rim_exec(p, t, ip);
4629 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4631 struct thread *t = &p->threads[p->thread_id];
4632 struct instruction *ip = t->ip;
4635 __instr_regadd_rii_exec(p, t, ip);
4644 static struct metarray *
4645 metarray_find(struct rte_swx_pipeline *p, const char *name);
4648 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4649 struct action *action,
4652 struct instruction *instr,
4653 struct instruction_data *data __rte_unused)
4655 char *metarray = tokens[1], *idx = tokens[2];
4658 uint32_t idx_struct_id, idx_val;
4660 CHECK(n_tokens == 3, EINVAL);
4662 m = metarray_find(p, metarray);
4665 /* METPREFETCH_H, METPREFETCH_M. */
4666 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4668 CHECK(!fidx->var_size, EINVAL);
4670 instr->type = INSTR_METPREFETCH_M;
4672 instr->type = INSTR_METPREFETCH_H;
4674 instr->meter.metarray_id = m->id;
4675 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4676 instr->meter.idx.n_bits = fidx->n_bits;
4677 instr->meter.idx.offset = fidx->offset / 8;
4681 /* METPREFETCH_I. */
4682 idx_val = strtoul(idx, &idx, 0);
4683 CHECK(!idx[0], EINVAL);
4685 instr->type = INSTR_METPREFETCH_I;
4686 instr->meter.metarray_id = m->id;
4687 instr->meter.idx_val = idx_val;
4692 instr_meter_translate(struct rte_swx_pipeline *p,
4693 struct action *action,
4696 struct instruction *instr,
4697 struct instruction_data *data __rte_unused)
4699 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4700 char *color_in = tokens[4], *color_out = tokens[5];
4702 struct field *fidx, *flength, *fcin, *fcout;
4703 uint32_t idx_struct_id, length_struct_id;
4704 uint32_t color_in_struct_id, color_out_struct_id;
4706 CHECK(n_tokens == 6, EINVAL);
4708 m = metarray_find(p, metarray);
4711 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4713 flength = struct_field_parse(p, action, length, &length_struct_id);
4714 CHECK(flength, EINVAL);
4715 CHECK(!flength->var_size, EINVAL);
4717 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4719 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4720 CHECK(fcout, EINVAL);
4721 CHECK(!fcout->var_size, EINVAL);
4723 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4725 CHECK(!fidx->var_size, EINVAL);
4726 CHECK(!fcin->var_size, EINVAL);
4728 instr->type = INSTR_METER_MMM;
4729 if (idx[0] == 'h' && length[0] == 'h')
4730 instr->type = INSTR_METER_HHM;
4731 if (idx[0] == 'h' && length[0] != 'h')
4732 instr->type = INSTR_METER_HMM;
4733 if (idx[0] != 'h' && length[0] == 'h')
4734 instr->type = INSTR_METER_MHM;
4736 instr->meter.metarray_id = m->id;
4738 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4739 instr->meter.idx.n_bits = fidx->n_bits;
4740 instr->meter.idx.offset = fidx->offset / 8;
4742 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4743 instr->meter.length.n_bits = flength->n_bits;
4744 instr->meter.length.offset = flength->offset / 8;
4746 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4747 instr->meter.color_in.n_bits = fcin->n_bits;
4748 instr->meter.color_in.offset = fcin->offset / 8;
4750 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4751 instr->meter.color_out.n_bits = fcout->n_bits;
4752 instr->meter.color_out.offset = fcout->offset / 8;
4755 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4756 if (fidx && !fcin) {
4757 uint32_t color_in_val;
4759 CHECK(!fidx->var_size, EINVAL);
4761 color_in_val = strtoul(color_in, &color_in, 0);
4762 CHECK(!color_in[0], EINVAL);
4764 instr->type = INSTR_METER_MMI;
4765 if (idx[0] == 'h' && length[0] == 'h')
4766 instr->type = INSTR_METER_HHI;
4767 if (idx[0] == 'h' && length[0] != 'h')
4768 instr->type = INSTR_METER_HMI;
4769 if (idx[0] != 'h' && length[0] == 'h')
4770 instr->type = INSTR_METER_MHI;
4772 instr->meter.metarray_id = m->id;
4774 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4775 instr->meter.idx.n_bits = fidx->n_bits;
4776 instr->meter.idx.offset = fidx->offset / 8;
4778 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4779 instr->meter.length.n_bits = flength->n_bits;
4780 instr->meter.length.offset = flength->offset / 8;
4782 instr->meter.color_in_val = color_in_val;
4784 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4785 instr->meter.color_out.n_bits = fcout->n_bits;
4786 instr->meter.color_out.offset = fcout->offset / 8;
4789 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4790 if (!fidx && fcin) {
4793 idx_val = strtoul(idx, &idx, 0);
4794 CHECK(!idx[0], EINVAL);
4796 CHECK(!fcin->var_size, EINVAL);
4798 instr->type = INSTR_METER_IMM;
4799 if (length[0] == 'h')
4800 instr->type = INSTR_METER_IHM;
4802 instr->meter.metarray_id = m->id;
4804 instr->meter.idx_val = idx_val;
4806 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4807 instr->meter.length.n_bits = flength->n_bits;
4808 instr->meter.length.offset = flength->offset / 8;
4810 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4811 instr->meter.color_in.n_bits = fcin->n_bits;
4812 instr->meter.color_in.offset = fcin->offset / 8;
4814 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4815 instr->meter.color_out.n_bits = fcout->n_bits;
4816 instr->meter.color_out.offset = fcout->offset / 8;
4819 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4820 if (!fidx && !fcin) {
4821 uint32_t idx_val, color_in_val;
4823 idx_val = strtoul(idx, &idx, 0);
4824 CHECK(!idx[0], EINVAL);
4826 color_in_val = strtoul(color_in, &color_in, 0);
4827 CHECK(!color_in[0], EINVAL);
4829 instr->type = INSTR_METER_IMI;
4830 if (length[0] == 'h')
4831 instr->type = INSTR_METER_IHI;
4833 instr->meter.metarray_id = m->id;
4835 instr->meter.idx_val = idx_val;
4837 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4838 instr->meter.length.n_bits = flength->n_bits;
4839 instr->meter.length.offset = flength->offset / 8;
4841 instr->meter.color_in_val = color_in_val;
4843 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4844 instr->meter.color_out.n_bits = fcout->n_bits;
4845 instr->meter.color_out.offset = fcout->offset / 8;
4852 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4854 struct thread *t = &p->threads[p->thread_id];
4855 struct instruction *ip = t->ip;
4858 __instr_metprefetch_h_exec(p, t, ip);
4865 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4867 struct thread *t = &p->threads[p->thread_id];
4868 struct instruction *ip = t->ip;
4871 __instr_metprefetch_m_exec(p, t, ip);
4878 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4880 struct thread *t = &p->threads[p->thread_id];
4881 struct instruction *ip = t->ip;
4884 __instr_metprefetch_i_exec(p, t, ip);
4891 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4893 struct thread *t = &p->threads[p->thread_id];
4894 struct instruction *ip = t->ip;
4897 __instr_meter_hhm_exec(p, t, ip);
4904 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4906 struct thread *t = &p->threads[p->thread_id];
4907 struct instruction *ip = t->ip;
4910 __instr_meter_hhi_exec(p, t, ip);
4917 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4919 struct thread *t = &p->threads[p->thread_id];
4920 struct instruction *ip = t->ip;
4923 __instr_meter_hmm_exec(p, t, ip);
4930 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4932 struct thread *t = &p->threads[p->thread_id];
4933 struct instruction *ip = t->ip;
4936 __instr_meter_hmi_exec(p, t, ip);
4943 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4945 struct thread *t = &p->threads[p->thread_id];
4946 struct instruction *ip = t->ip;
4949 __instr_meter_mhm_exec(p, t, ip);
4956 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4958 struct thread *t = &p->threads[p->thread_id];
4959 struct instruction *ip = t->ip;
4962 __instr_meter_mhi_exec(p, t, ip);
4969 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4971 struct thread *t = &p->threads[p->thread_id];
4972 struct instruction *ip = t->ip;
4975 __instr_meter_mmm_exec(p, t, ip);
4982 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4984 struct thread *t = &p->threads[p->thread_id];
4985 struct instruction *ip = t->ip;
4988 __instr_meter_mmi_exec(p, t, ip);
4995 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4997 struct thread *t = &p->threads[p->thread_id];
4998 struct instruction *ip = t->ip;
5001 __instr_meter_ihm_exec(p, t, ip);
5008 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
5010 struct thread *t = &p->threads[p->thread_id];
5011 struct instruction *ip = t->ip;
5014 __instr_meter_ihi_exec(p, t, ip);
5021 instr_meter_imm_exec(struct rte_swx_pipeline *p)
5023 struct thread *t = &p->threads[p->thread_id];
5024 struct instruction *ip = t->ip;
5027 __instr_meter_imm_exec(p, t, ip);
5034 instr_meter_imi_exec(struct rte_swx_pipeline *p)
5036 struct thread *t = &p->threads[p->thread_id];
5037 struct instruction *ip = t->ip;
5040 __instr_meter_imi_exec(p, t, ip);
5050 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
5051 struct action *action __rte_unused,
5054 struct instruction *instr,
5055 struct instruction_data *data)
5057 CHECK(n_tokens == 2, EINVAL);
5059 strcpy(data->jmp_label, tokens[1]);
5061 instr->type = INSTR_JMP;
5062 instr->jmp.ip = NULL; /* Resolved later. */
5067 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
5068 struct action *action __rte_unused,
5071 struct instruction *instr,
5072 struct instruction_data *data)
5076 CHECK(n_tokens == 3, EINVAL);
5078 strcpy(data->jmp_label, tokens[1]);
5080 h = header_parse(p, tokens[2]);
5083 instr->type = INSTR_JMP_VALID;
5084 instr->jmp.ip = NULL; /* Resolved later. */
5085 instr->jmp.header_id = h->id;
5090 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
5091 struct action *action __rte_unused,
5094 struct instruction *instr,
5095 struct instruction_data *data)
5099 CHECK(n_tokens == 3, EINVAL);
5101 strcpy(data->jmp_label, tokens[1]);
5103 h = header_parse(p, tokens[2]);
5106 instr->type = INSTR_JMP_INVALID;
5107 instr->jmp.ip = NULL; /* Resolved later. */
5108 instr->jmp.header_id = h->id;
5113 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
5114 struct action *action,
5117 struct instruction *instr,
5118 struct instruction_data *data)
5120 CHECK(!action, EINVAL);
5121 CHECK(n_tokens == 2, EINVAL);
5123 strcpy(data->jmp_label, tokens[1]);
5125 instr->type = INSTR_JMP_HIT;
5126 instr->jmp.ip = NULL; /* Resolved later. */
5131 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
5132 struct action *action,
5135 struct instruction *instr,
5136 struct instruction_data *data)
5138 CHECK(!action, EINVAL);
5139 CHECK(n_tokens == 2, EINVAL);
5141 strcpy(data->jmp_label, tokens[1]);
5143 instr->type = INSTR_JMP_MISS;
5144 instr->jmp.ip = NULL; /* Resolved later. */
5149 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
5150 struct action *action,
5153 struct instruction *instr,
5154 struct instruction_data *data)
5158 CHECK(!action, EINVAL);
5159 CHECK(n_tokens == 3, EINVAL);
5161 strcpy(data->jmp_label, tokens[1]);
5163 a = action_find(p, tokens[2]);
5166 instr->type = INSTR_JMP_ACTION_HIT;
5167 instr->jmp.ip = NULL; /* Resolved later. */
5168 instr->jmp.action_id = a->id;
5173 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
5174 struct action *action,
5177 struct instruction *instr,
5178 struct instruction_data *data)
5182 CHECK(!action, EINVAL);
5183 CHECK(n_tokens == 3, EINVAL);
5185 strcpy(data->jmp_label, tokens[1]);
5187 a = action_find(p, tokens[2]);
5190 instr->type = INSTR_JMP_ACTION_MISS;
5191 instr->jmp.ip = NULL; /* Resolved later. */
5192 instr->jmp.action_id = a->id;
5197 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5198 struct action *action,
5201 struct instruction *instr,
5202 struct instruction_data *data)
5204 char *a = tokens[2], *b = tokens[3];
5205 struct field *fa, *fb;
5207 uint32_t a_struct_id, b_struct_id;
5209 CHECK(n_tokens == 4, EINVAL);
5211 strcpy(data->jmp_label, tokens[1]);
5213 fa = struct_field_parse(p, action, a, &a_struct_id);
5215 CHECK(!fa->var_size, EINVAL);
5217 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5218 fb = struct_field_parse(p, action, b, &b_struct_id);
5220 CHECK(!fb->var_size, EINVAL);
5222 instr->type = INSTR_JMP_EQ;
5223 if (a[0] != 'h' && b[0] == 'h')
5224 instr->type = INSTR_JMP_EQ_MH;
5225 if (a[0] == 'h' && b[0] != 'h')
5226 instr->type = INSTR_JMP_EQ_HM;
5227 if (a[0] == 'h' && b[0] == 'h')
5228 instr->type = INSTR_JMP_EQ_HH;
5229 instr->jmp.ip = NULL; /* Resolved later. */
5231 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5232 instr->jmp.a.n_bits = fa->n_bits;
5233 instr->jmp.a.offset = fa->offset / 8;
5234 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5235 instr->jmp.b.n_bits = fb->n_bits;
5236 instr->jmp.b.offset = fb->offset / 8;
5241 b_val = strtoull(b, &b, 0);
5242 CHECK(!b[0], EINVAL);
5245 b_val = hton64(b_val) >> (64 - fa->n_bits);
5247 instr->type = INSTR_JMP_EQ_I;
5248 instr->jmp.ip = NULL; /* Resolved later. */
5249 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5250 instr->jmp.a.n_bits = fa->n_bits;
5251 instr->jmp.a.offset = fa->offset / 8;
5252 instr->jmp.b_val = b_val;
5257 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5258 struct action *action,
5261 struct instruction *instr,
5262 struct instruction_data *data)
5264 char *a = tokens[2], *b = tokens[3];
5265 struct field *fa, *fb;
5267 uint32_t a_struct_id, b_struct_id;
5269 CHECK(n_tokens == 4, EINVAL);
5271 strcpy(data->jmp_label, tokens[1]);
5273 fa = struct_field_parse(p, action, a, &a_struct_id);
5275 CHECK(!fa->var_size, EINVAL);
5277 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5278 fb = struct_field_parse(p, action, b, &b_struct_id);
5280 CHECK(!fb->var_size, EINVAL);
5282 instr->type = INSTR_JMP_NEQ;
5283 if (a[0] != 'h' && b[0] == 'h')
5284 instr->type = INSTR_JMP_NEQ_MH;
5285 if (a[0] == 'h' && b[0] != 'h')
5286 instr->type = INSTR_JMP_NEQ_HM;
5287 if (a[0] == 'h' && b[0] == 'h')
5288 instr->type = INSTR_JMP_NEQ_HH;
5289 instr->jmp.ip = NULL; /* Resolved later. */
5291 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5292 instr->jmp.a.n_bits = fa->n_bits;
5293 instr->jmp.a.offset = fa->offset / 8;
5294 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5295 instr->jmp.b.n_bits = fb->n_bits;
5296 instr->jmp.b.offset = fb->offset / 8;
5301 b_val = strtoull(b, &b, 0);
5302 CHECK(!b[0], EINVAL);
5305 b_val = hton64(b_val) >> (64 - fa->n_bits);
5307 instr->type = INSTR_JMP_NEQ_I;
5308 instr->jmp.ip = NULL; /* Resolved later. */
5309 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5310 instr->jmp.a.n_bits = fa->n_bits;
5311 instr->jmp.a.offset = fa->offset / 8;
5312 instr->jmp.b_val = b_val;
5317 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5318 struct action *action,
5321 struct instruction *instr,
5322 struct instruction_data *data)
5324 char *a = tokens[2], *b = tokens[3];
5325 struct field *fa, *fb;
5327 uint32_t a_struct_id, b_struct_id;
5329 CHECK(n_tokens == 4, EINVAL);
5331 strcpy(data->jmp_label, tokens[1]);
5333 fa = struct_field_parse(p, action, a, &a_struct_id);
5335 CHECK(!fa->var_size, EINVAL);
5337 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5338 fb = struct_field_parse(p, action, b, &b_struct_id);
5340 CHECK(!fb->var_size, EINVAL);
5342 instr->type = INSTR_JMP_LT;
5343 if (a[0] == 'h' && b[0] != 'h')
5344 instr->type = INSTR_JMP_LT_HM;
5345 if (a[0] != 'h' && b[0] == 'h')
5346 instr->type = INSTR_JMP_LT_MH;
5347 if (a[0] == 'h' && b[0] == 'h')
5348 instr->type = INSTR_JMP_LT_HH;
5349 instr->jmp.ip = NULL; /* Resolved later. */
5351 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5352 instr->jmp.a.n_bits = fa->n_bits;
5353 instr->jmp.a.offset = fa->offset / 8;
5354 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5355 instr->jmp.b.n_bits = fb->n_bits;
5356 instr->jmp.b.offset = fb->offset / 8;
5360 /* JMP_LT_MI, JMP_LT_HI. */
5361 b_val = strtoull(b, &b, 0);
5362 CHECK(!b[0], EINVAL);
5364 instr->type = INSTR_JMP_LT_MI;
5366 instr->type = INSTR_JMP_LT_HI;
5367 instr->jmp.ip = NULL; /* Resolved later. */
5369 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5370 instr->jmp.a.n_bits = fa->n_bits;
5371 instr->jmp.a.offset = fa->offset / 8;
5372 instr->jmp.b_val = b_val;
5377 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5378 struct action *action,
5381 struct instruction *instr,
5382 struct instruction_data *data)
5384 char *a = tokens[2], *b = tokens[3];
5385 struct field *fa, *fb;
5387 uint32_t a_struct_id, b_struct_id;
5389 CHECK(n_tokens == 4, EINVAL);
5391 strcpy(data->jmp_label, tokens[1]);
5393 fa = struct_field_parse(p, action, a, &a_struct_id);
5395 CHECK(!fa->var_size, EINVAL);
5397 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5398 fb = struct_field_parse(p, action, b, &b_struct_id);
5400 CHECK(!fb->var_size, EINVAL);
5402 instr->type = INSTR_JMP_GT;
5403 if (a[0] == 'h' && b[0] != 'h')
5404 instr->type = INSTR_JMP_GT_HM;
5405 if (a[0] != 'h' && b[0] == 'h')
5406 instr->type = INSTR_JMP_GT_MH;
5407 if (a[0] == 'h' && b[0] == 'h')
5408 instr->type = INSTR_JMP_GT_HH;
5409 instr->jmp.ip = NULL; /* Resolved later. */
5411 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5412 instr->jmp.a.n_bits = fa->n_bits;
5413 instr->jmp.a.offset = fa->offset / 8;
5414 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5415 instr->jmp.b.n_bits = fb->n_bits;
5416 instr->jmp.b.offset = fb->offset / 8;
5420 /* JMP_GT_MI, JMP_GT_HI. */
5421 b_val = strtoull(b, &b, 0);
5422 CHECK(!b[0], EINVAL);
5424 instr->type = INSTR_JMP_GT_MI;
5426 instr->type = INSTR_JMP_GT_HI;
5427 instr->jmp.ip = NULL; /* Resolved later. */
5429 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5430 instr->jmp.a.n_bits = fa->n_bits;
5431 instr->jmp.a.offset = fa->offset / 8;
5432 instr->jmp.b_val = b_val;
5437 instr_jmp_exec(struct rte_swx_pipeline *p)
5439 struct thread *t = &p->threads[p->thread_id];
5440 struct instruction *ip = t->ip;
5442 TRACE("[Thread %2u] jmp\n", p->thread_id);
5444 thread_ip_set(t, ip->jmp.ip);
5448 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5450 struct thread *t = &p->threads[p->thread_id];
5451 struct instruction *ip = t->ip;
5452 uint32_t header_id = ip->jmp.header_id;
5454 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5456 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5460 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5462 struct thread *t = &p->threads[p->thread_id];
5463 struct instruction *ip = t->ip;
5464 uint32_t header_id = ip->jmp.header_id;
5466 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5468 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5472 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5474 struct thread *t = &p->threads[p->thread_id];
5475 struct instruction *ip = t->ip;
5476 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5478 TRACE("[Thread %2u] jmph\n", p->thread_id);
5480 t->ip = ip_next[t->hit];
5484 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5486 struct thread *t = &p->threads[p->thread_id];
5487 struct instruction *ip = t->ip;
5488 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5490 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5492 t->ip = ip_next[t->hit];
5496 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5498 struct thread *t = &p->threads[p->thread_id];
5499 struct instruction *ip = t->ip;
5501 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5503 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5507 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5509 struct thread *t = &p->threads[p->thread_id];
5510 struct instruction *ip = t->ip;
5512 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5514 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5518 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5520 struct thread *t = &p->threads[p->thread_id];
5521 struct instruction *ip = t->ip;
5523 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5529 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5531 struct thread *t = &p->threads[p->thread_id];
5532 struct instruction *ip = t->ip;
5534 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5536 JMP_CMP_MH(t, ip, ==);
5540 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5542 struct thread *t = &p->threads[p->thread_id];
5543 struct instruction *ip = t->ip;
5545 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5547 JMP_CMP_HM(t, ip, ==);
5551 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5553 struct thread *t = &p->threads[p->thread_id];
5554 struct instruction *ip = t->ip;
5556 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5558 JMP_CMP_HH_FAST(t, ip, ==);
5562 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5564 struct thread *t = &p->threads[p->thread_id];
5565 struct instruction *ip = t->ip;
5567 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5569 JMP_CMP_I(t, ip, ==);
5573 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5575 struct thread *t = &p->threads[p->thread_id];
5576 struct instruction *ip = t->ip;
5578 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5584 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5586 struct thread *t = &p->threads[p->thread_id];
5587 struct instruction *ip = t->ip;
5589 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5591 JMP_CMP_MH(t, ip, !=);
5595 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5597 struct thread *t = &p->threads[p->thread_id];
5598 struct instruction *ip = t->ip;
5600 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5602 JMP_CMP_HM(t, ip, !=);
5606 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5608 struct thread *t = &p->threads[p->thread_id];
5609 struct instruction *ip = t->ip;
5611 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5613 JMP_CMP_HH_FAST(t, ip, !=);
5617 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5619 struct thread *t = &p->threads[p->thread_id];
5620 struct instruction *ip = t->ip;
5622 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5624 JMP_CMP_I(t, ip, !=);
5628 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5630 struct thread *t = &p->threads[p->thread_id];
5631 struct instruction *ip = t->ip;
5633 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5639 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5641 struct thread *t = &p->threads[p->thread_id];
5642 struct instruction *ip = t->ip;
5644 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5646 JMP_CMP_MH(t, ip, <);
5650 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5652 struct thread *t = &p->threads[p->thread_id];
5653 struct instruction *ip = t->ip;
5655 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5657 JMP_CMP_HM(t, ip, <);
5661 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5663 struct thread *t = &p->threads[p->thread_id];
5664 struct instruction *ip = t->ip;
5666 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5668 JMP_CMP_HH(t, ip, <);
5672 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5674 struct thread *t = &p->threads[p->thread_id];
5675 struct instruction *ip = t->ip;
5677 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5679 JMP_CMP_MI(t, ip, <);
5683 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5685 struct thread *t = &p->threads[p->thread_id];
5686 struct instruction *ip = t->ip;
5688 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5690 JMP_CMP_HI(t, ip, <);
5694 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5696 struct thread *t = &p->threads[p->thread_id];
5697 struct instruction *ip = t->ip;
5699 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5705 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5707 struct thread *t = &p->threads[p->thread_id];
5708 struct instruction *ip = t->ip;
5710 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5712 JMP_CMP_MH(t, ip, >);
5716 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5718 struct thread *t = &p->threads[p->thread_id];
5719 struct instruction *ip = t->ip;
5721 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5723 JMP_CMP_HM(t, ip, >);
5727 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5729 struct thread *t = &p->threads[p->thread_id];
5730 struct instruction *ip = t->ip;
5732 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5734 JMP_CMP_HH(t, ip, >);
5738 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5740 struct thread *t = &p->threads[p->thread_id];
5741 struct instruction *ip = t->ip;
5743 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5745 JMP_CMP_MI(t, ip, >);
5749 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5751 struct thread *t = &p->threads[p->thread_id];
5752 struct instruction *ip = t->ip;
5754 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5756 JMP_CMP_HI(t, ip, >);
5763 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5764 struct action *action,
5765 char **tokens __rte_unused,
5767 struct instruction *instr,
5768 struct instruction_data *data __rte_unused)
5770 CHECK(action, EINVAL);
5771 CHECK(n_tokens == 1, EINVAL);
5773 instr->type = INSTR_RETURN;
5778 instr_return_exec(struct rte_swx_pipeline *p)
5780 struct thread *t = &p->threads[p->thread_id];
5782 TRACE("[Thread %2u] return\n", p->thread_id);
5788 instr_translate(struct rte_swx_pipeline *p,
5789 struct action *action,
5791 struct instruction *instr,
5792 struct instruction_data *data)
5794 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5795 int n_tokens = 0, tpos = 0;
5797 /* Parse the instruction string into tokens. */
5801 token = strtok_r(string, " \t\v", &string);
5805 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5806 CHECK_NAME(token, EINVAL);
5808 tokens[n_tokens] = token;
5812 CHECK(n_tokens, EINVAL);
5814 /* Handle the optional instruction label. */
5815 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5816 strcpy(data->label, tokens[0]);
5819 CHECK(n_tokens - tpos, EINVAL);
5822 /* Identify the instruction type. */
5823 if (!strcmp(tokens[tpos], "rx"))
5824 return instr_rx_translate(p,
5831 if (!strcmp(tokens[tpos], "tx"))
5832 return instr_tx_translate(p,
5839 if (!strcmp(tokens[tpos], "drop"))
5840 return instr_drop_translate(p,
5847 if (!strcmp(tokens[tpos], "mirror"))
5848 return instr_mirror_translate(p,
5855 if (!strcmp(tokens[tpos], "recirculate"))
5856 return instr_recirculate_translate(p,
5863 if (!strcmp(tokens[tpos], "recircid"))
5864 return instr_recircid_translate(p,
5871 if (!strcmp(tokens[tpos], "extract"))
5872 return instr_hdr_extract_translate(p,
5879 if (!strcmp(tokens[tpos], "lookahead"))
5880 return instr_hdr_lookahead_translate(p,
5887 if (!strcmp(tokens[tpos], "emit"))
5888 return instr_hdr_emit_translate(p,
5895 if (!strcmp(tokens[tpos], "validate"))
5896 return instr_hdr_validate_translate(p,
5903 if (!strcmp(tokens[tpos], "invalidate"))
5904 return instr_hdr_invalidate_translate(p,
5911 if (!strcmp(tokens[tpos], "mov"))
5912 return instr_mov_translate(p,
5919 if (!strcmp(tokens[tpos], "add"))
5920 return instr_alu_add_translate(p,
5927 if (!strcmp(tokens[tpos], "sub"))
5928 return instr_alu_sub_translate(p,
5935 if (!strcmp(tokens[tpos], "ckadd"))
5936 return instr_alu_ckadd_translate(p,
5943 if (!strcmp(tokens[tpos], "cksub"))
5944 return instr_alu_cksub_translate(p,
5951 if (!strcmp(tokens[tpos], "and"))
5952 return instr_alu_and_translate(p,
5959 if (!strcmp(tokens[tpos], "or"))
5960 return instr_alu_or_translate(p,
5967 if (!strcmp(tokens[tpos], "xor"))
5968 return instr_alu_xor_translate(p,
5975 if (!strcmp(tokens[tpos], "shl"))
5976 return instr_alu_shl_translate(p,
5983 if (!strcmp(tokens[tpos], "shr"))
5984 return instr_alu_shr_translate(p,
5991 if (!strcmp(tokens[tpos], "regprefetch"))
5992 return instr_regprefetch_translate(p,
5999 if (!strcmp(tokens[tpos], "regrd"))
6000 return instr_regrd_translate(p,
6007 if (!strcmp(tokens[tpos], "regwr"))
6008 return instr_regwr_translate(p,
6015 if (!strcmp(tokens[tpos], "regadd"))
6016 return instr_regadd_translate(p,
6023 if (!strcmp(tokens[tpos], "metprefetch"))
6024 return instr_metprefetch_translate(p,
6031 if (!strcmp(tokens[tpos], "meter"))
6032 return instr_meter_translate(p,
6039 if (!strcmp(tokens[tpos], "table"))
6040 return instr_table_translate(p,
6047 if (!strcmp(tokens[tpos], "learn"))
6048 return instr_learn_translate(p,
6055 if (!strcmp(tokens[tpos], "forget"))
6056 return instr_forget_translate(p,
6063 if (!strcmp(tokens[tpos], "extern"))
6064 return instr_extern_translate(p,
6071 if (!strcmp(tokens[tpos], "jmp"))
6072 return instr_jmp_translate(p,
6079 if (!strcmp(tokens[tpos], "jmpv"))
6080 return instr_jmp_valid_translate(p,
6087 if (!strcmp(tokens[tpos], "jmpnv"))
6088 return instr_jmp_invalid_translate(p,
6095 if (!strcmp(tokens[tpos], "jmph"))
6096 return instr_jmp_hit_translate(p,
6103 if (!strcmp(tokens[tpos], "jmpnh"))
6104 return instr_jmp_miss_translate(p,
6111 if (!strcmp(tokens[tpos], "jmpa"))
6112 return instr_jmp_action_hit_translate(p,
6119 if (!strcmp(tokens[tpos], "jmpna"))
6120 return instr_jmp_action_miss_translate(p,
6127 if (!strcmp(tokens[tpos], "jmpeq"))
6128 return instr_jmp_eq_translate(p,
6135 if (!strcmp(tokens[tpos], "jmpneq"))
6136 return instr_jmp_neq_translate(p,
6143 if (!strcmp(tokens[tpos], "jmplt"))
6144 return instr_jmp_lt_translate(p,
6151 if (!strcmp(tokens[tpos], "jmpgt"))
6152 return instr_jmp_gt_translate(p,
6159 if (!strcmp(tokens[tpos], "return"))
6160 return instr_return_translate(p,
6170 static struct instruction_data *
6171 label_find(struct instruction_data *data, uint32_t n, const char *label)
6175 for (i = 0; i < n; i++)
6176 if (!strcmp(label, data[i].label))
6183 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
6185 uint32_t count = 0, i;
6190 for (i = 0; i < n; i++)
6191 if (!strcmp(label, data[i].jmp_label))
6198 instr_label_check(struct instruction_data *instruction_data,
6199 uint32_t n_instructions)
6203 /* Check that all instruction labels are unique. */
6204 for (i = 0; i < n_instructions; i++) {
6205 struct instruction_data *data = &instruction_data[i];
6206 char *label = data->label;
6212 for (j = i + 1; j < n_instructions; j++)
6213 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6216 /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6217 for (i = 0; i < n_instructions; i++) {
6218 struct instruction_data *data = &instruction_data[i];
6219 char *label = data->label;
6220 char *jmp_label = data->jmp_label;
6222 /* Continue if this instruction does not have a label or it is not a jump. */
6223 if (!label[0] || !jmp_label[0])
6226 CHECK(strcmp(label, jmp_label), EINVAL);
6229 /* Get users for each instruction label. */
6230 for (i = 0; i < n_instructions; i++) {
6231 struct instruction_data *data = &instruction_data[i];
6232 char *label = data->label;
6234 data->n_users = label_is_used(instruction_data,
6243 instr_jmp_resolve(struct instruction *instructions,
6244 struct instruction_data *instruction_data,
6245 uint32_t n_instructions)
6249 for (i = 0; i < n_instructions; i++) {
6250 struct instruction *instr = &instructions[i];
6251 struct instruction_data *data = &instruction_data[i];
6252 struct instruction_data *found;
6254 if (!instruction_is_jmp(instr))
6257 found = label_find(instruction_data,
6260 CHECK(found, EINVAL);
6262 instr->jmp.ip = &instructions[found - instruction_data];
6269 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6271 struct instruction *instr,
6272 struct instruction_data *data __rte_unused,
6273 uint32_t n_instructions)
6276 enum instruction_type type;
6279 /* Check that the first instruction is rx. */
6280 CHECK(instr[0].type == INSTR_RX, EINVAL);
6282 /* Check that there is at least one tx instruction. */
6283 for (i = 0; i < n_instructions; i++) {
6284 type = instr[i].type;
6286 if (instruction_is_tx(type))
6289 CHECK(i < n_instructions, EINVAL);
6291 /* Check that the last instruction is either tx or unconditional
6294 type = instr[n_instructions - 1].type;
6295 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6299 enum instruction_type type;
6302 /* Check that there is at least one return or tx instruction. */
6303 for (i = 0; i < n_instructions; i++) {
6304 type = instr[i].type;
6306 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6309 CHECK(i < n_instructions, EINVAL);
6316 instr_compact(struct instruction *instructions,
6317 struct instruction_data *instruction_data,
6318 uint32_t n_instructions)
6320 uint32_t i, pos = 0;
6322 /* Eliminate the invalid instructions that have been optimized out. */
6323 for (i = 0; i < n_instructions; i++) {
6324 struct instruction *instr = &instructions[i];
6325 struct instruction_data *data = &instruction_data[i];
6331 memcpy(&instructions[pos], instr, sizeof(*instr));
6332 memcpy(&instruction_data[pos], data, sizeof(*data));
6342 instr_pattern_extract_many_search(struct instruction *instr,
6343 struct instruction_data *data,
6345 uint32_t *n_pattern_instr)
6349 for (i = 0; i < n_instr; i++) {
6350 if (data[i].invalid)
6353 if (instr[i].type != INSTR_HDR_EXTRACT)
6356 if (i == RTE_DIM(instr->io.hdr.header_id))
6359 if (i && data[i].n_users)
6366 *n_pattern_instr = i;
6371 instr_pattern_extract_many_replace(struct instruction *instr,
6372 struct instruction_data *data,
6377 for (i = 1; i < n_instr; i++) {
6379 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6380 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6381 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6383 data[i].invalid = 1;
6388 instr_pattern_extract_many_optimize(struct instruction *instructions,
6389 struct instruction_data *instruction_data,
6390 uint32_t n_instructions)
6394 for (i = 0; i < n_instructions; ) {
6395 struct instruction *instr = &instructions[i];
6396 struct instruction_data *data = &instruction_data[i];
6397 uint32_t n_instr = 0;
6401 detected = instr_pattern_extract_many_search(instr,
6406 instr_pattern_extract_many_replace(instr,
6413 /* No pattern starting at the current instruction. */
6417 /* Eliminate the invalid instructions that have been optimized out. */
6418 n_instructions = instr_compact(instructions,
6422 return n_instructions;
6426 instr_pattern_emit_many_tx_search(struct instruction *instr,
6427 struct instruction_data *data,
6429 uint32_t *n_pattern_instr)
6433 for (i = 0; i < n_instr; i++) {
6434 if (data[i].invalid)
6437 if (instr[i].type != INSTR_HDR_EMIT)
6440 if (i == RTE_DIM(instr->io.hdr.header_id))
6443 if (i && data[i].n_users)
6450 if (instr[i].type != INSTR_TX)
6453 if (data[i].n_users)
6458 *n_pattern_instr = i;
6463 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6464 struct instruction_data *data,
6469 /* Any emit instruction in addition to the first one. */
6470 for (i = 1; i < n_instr - 1; i++) {
6472 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6473 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6474 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6476 data[i].invalid = 1;
6479 /* The TX instruction is the last one in the pattern. */
6481 instr[0].io.io.offset = instr[i].io.io.offset;
6482 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6483 data[i].invalid = 1;
6487 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6488 struct instruction_data *instruction_data,
6489 uint32_t n_instructions)
6493 for (i = 0; i < n_instructions; ) {
6494 struct instruction *instr = &instructions[i];
6495 struct instruction_data *data = &instruction_data[i];
6496 uint32_t n_instr = 0;
6499 /* Emit many + TX. */
6500 detected = instr_pattern_emit_many_tx_search(instr,
6505 instr_pattern_emit_many_tx_replace(instr,
6512 /* No pattern starting at the current instruction. */
6516 /* Eliminate the invalid instructions that have been optimized out. */
6517 n_instructions = instr_compact(instructions,
6521 return n_instructions;
6525 action_arg_src_mov_count(struct action *a,
6527 struct instruction *instructions,
6528 struct instruction_data *instruction_data,
6529 uint32_t n_instructions);
6532 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6534 struct instruction *instr,
6535 struct instruction_data *data,
6537 struct instruction *instructions,
6538 struct instruction_data *instruction_data,
6539 uint32_t n_instructions,
6540 uint32_t *n_pattern_instr)
6543 uint32_t src_field_id, i, j;
6545 /* Prerequisites. */
6549 /* First instruction: MOV_HM. */
6550 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6553 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6554 if (!h || h->st->var_size)
6557 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6558 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6561 if (src_field_id == a->st->n_fields)
6564 if (instr[0].mov.dst.offset ||
6565 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6566 instr[0].mov.src.struct_id ||
6567 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6568 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6571 if ((n_instr < h->st->n_fields + 1) ||
6572 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6575 /* Subsequent instructions: MOV_HM. */
6576 for (i = 1; i < h->st->n_fields; i++)
6577 if (data[i].invalid ||
6579 (instr[i].type != INSTR_MOV_HM) ||
6580 (instr[i].mov.dst.struct_id != h->struct_id) ||
6581 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6582 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6583 instr[i].mov.src.struct_id ||
6584 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6585 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6586 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6589 /* Last instruction: HDR_VALIDATE. */
6590 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6591 (instr[i].valid.header_id != h->id))
6594 /* Check that none of the action args that are used as source for this
6595 * DMA transfer are not used as source in any other mov instruction.
6597 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6600 n_users = action_arg_src_mov_count(a,
6609 *n_pattern_instr = 1 + i;
6614 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6616 struct instruction *instr,
6617 struct instruction_data *data,
6621 uint32_t src_field_id, src_offset, i;
6623 /* Read from the instructions before they are modified. */
6624 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6628 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6629 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6632 if (src_field_id == a->st->n_fields)
6635 src_offset = instr[0].mov.src.offset;
6637 /* Modify the instructions. */
6638 instr[0].type = INSTR_DMA_HT;
6639 instr[0].dma.dst.header_id[0] = h->id;
6640 instr[0].dma.dst.struct_id[0] = h->struct_id;
6641 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6642 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6644 for (i = 1; i < n_instr; i++)
6645 data[i].invalid = 1;
6647 /* Update the endianness of the action arguments to header endianness. */
6648 for (i = 0; i < h->st->n_fields; i++)
6649 a->args_endianness[src_field_id + i] = 1;
6653 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6655 struct instruction *instructions,
6656 struct instruction_data *instruction_data,
6657 uint32_t n_instructions)
6662 return n_instructions;
6664 for (i = 0; i < n_instructions; ) {
6665 struct instruction *instr = &instructions[i];
6666 struct instruction_data *data = &instruction_data[i];
6667 uint32_t n_instr = 0;
6670 /* Mov all + validate. */
6671 detected = instr_pattern_mov_all_validate_search(p,
6681 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6686 /* No pattern starting at the current instruction. */
6690 /* Eliminate the invalid instructions that have been optimized out. */
6691 n_instructions = instr_compact(instructions,
6695 return n_instructions;
6699 instr_pattern_dma_many_search(struct instruction *instr,
6700 struct instruction_data *data,
6702 uint32_t *n_pattern_instr)
6706 for (i = 0; i < n_instr; i++) {
6707 if (data[i].invalid)
6710 if (instr[i].type != INSTR_DMA_HT)
6713 if (i == RTE_DIM(instr->dma.dst.header_id))
6716 if (i && data[i].n_users)
6723 *n_pattern_instr = i;
6728 instr_pattern_dma_many_replace(struct instruction *instr,
6729 struct instruction_data *data,
6734 for (i = 1; i < n_instr; i++) {
6736 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6737 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6738 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6739 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6741 data[i].invalid = 1;
6746 instr_pattern_dma_many_optimize(struct instruction *instructions,
6747 struct instruction_data *instruction_data,
6748 uint32_t n_instructions)
6752 for (i = 0; i < n_instructions; ) {
6753 struct instruction *instr = &instructions[i];
6754 struct instruction_data *data = &instruction_data[i];
6755 uint32_t n_instr = 0;
6759 detected = instr_pattern_dma_many_search(instr,
6764 instr_pattern_dma_many_replace(instr, data, n_instr);
6769 /* No pattern starting at the current instruction. */
6773 /* Eliminate the invalid instructions that have been optimized out. */
6774 n_instructions = instr_compact(instructions,
6778 return n_instructions;
6782 instr_optimize(struct rte_swx_pipeline *p,
6784 struct instruction *instructions,
6785 struct instruction_data *instruction_data,
6786 uint32_t n_instructions)
6789 n_instructions = instr_pattern_extract_many_optimize(instructions,
6793 /* Emit many + TX. */
6794 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6798 /* Mov all + validate. */
6799 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6806 n_instructions = instr_pattern_dma_many_optimize(instructions,
6810 return n_instructions;
6814 instruction_config(struct rte_swx_pipeline *p,
6816 const char **instructions,
6817 uint32_t n_instructions)
6819 struct instruction *instr = NULL;
6820 struct instruction_data *data = NULL;
6824 CHECK(n_instructions, EINVAL);
6825 CHECK(instructions, EINVAL);
6826 for (i = 0; i < n_instructions; i++)
6827 CHECK_INSTRUCTION(instructions[i], EINVAL);
6829 /* Memory allocation. */
6830 instr = calloc(n_instructions, sizeof(struct instruction));
6836 data = calloc(n_instructions, sizeof(struct instruction_data));
6842 for (i = 0; i < n_instructions; i++) {
6843 char *string = strdup(instructions[i]);
6849 err = instr_translate(p, a, string, &instr[i], &data[i]);
6858 err = instr_label_check(data, n_instructions);
6862 err = instr_verify(p, a, instr, data, n_instructions);
6866 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6868 err = instr_jmp_resolve(instr, data, n_instructions);
6873 a->instructions = instr;
6874 a->instruction_data = data;
6875 a->n_instructions = n_instructions;
6877 p->instructions = instr;
6878 p->instruction_data = data;
6879 p->n_instructions = n_instructions;
6890 static instr_exec_t instruction_table[] = {
6891 [INSTR_RX] = instr_rx_exec,
6892 [INSTR_TX] = instr_tx_exec,
6893 [INSTR_TX_I] = instr_tx_i_exec,
6894 [INSTR_DROP] = instr_drop_exec,
6895 [INSTR_MIRROR] = instr_mirror_exec,
6896 [INSTR_RECIRCULATE] = instr_recirculate_exec,
6897 [INSTR_RECIRCID] = instr_recircid_exec,
6899 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6900 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6901 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6902 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6903 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6904 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6905 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6906 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6907 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6908 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6910 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6911 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6912 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6913 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6914 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6915 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6916 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6917 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6918 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6920 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6921 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6923 [INSTR_MOV] = instr_mov_exec,
6924 [INSTR_MOV_MH] = instr_mov_mh_exec,
6925 [INSTR_MOV_HM] = instr_mov_hm_exec,
6926 [INSTR_MOV_HH] = instr_mov_hh_exec,
6927 [INSTR_MOV_I] = instr_mov_i_exec,
6929 [INSTR_DMA_HT] = instr_dma_ht_exec,
6930 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6931 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6932 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6933 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6934 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6935 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6936 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6938 [INSTR_ALU_ADD] = instr_alu_add_exec,
6939 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6940 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6941 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6942 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6943 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6945 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6946 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6947 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6948 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6949 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6950 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6952 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6953 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6954 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6955 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6957 [INSTR_ALU_AND] = instr_alu_and_exec,
6958 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6959 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6960 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6961 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6963 [INSTR_ALU_OR] = instr_alu_or_exec,
6964 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6965 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6966 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6967 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6969 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6970 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6971 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6972 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6973 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6975 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6976 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6977 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6978 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6979 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6980 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6982 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6983 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6984 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6985 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6986 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6987 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6989 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6990 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6991 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6993 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6994 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6995 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6996 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6997 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6998 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
7000 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
7001 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
7002 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
7003 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
7004 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
7005 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
7006 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
7007 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
7008 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
7010 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
7011 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
7012 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
7013 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
7014 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
7015 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
7016 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
7017 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
7018 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
7020 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
7021 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
7022 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
7024 [INSTR_METER_HHM] = instr_meter_hhm_exec,
7025 [INSTR_METER_HHI] = instr_meter_hhi_exec,
7026 [INSTR_METER_HMM] = instr_meter_hmm_exec,
7027 [INSTR_METER_HMI] = instr_meter_hmi_exec,
7028 [INSTR_METER_MHM] = instr_meter_mhm_exec,
7029 [INSTR_METER_MHI] = instr_meter_mhi_exec,
7030 [INSTR_METER_MMM] = instr_meter_mmm_exec,
7031 [INSTR_METER_MMI] = instr_meter_mmi_exec,
7032 [INSTR_METER_IHM] = instr_meter_ihm_exec,
7033 [INSTR_METER_IHI] = instr_meter_ihi_exec,
7034 [INSTR_METER_IMM] = instr_meter_imm_exec,
7035 [INSTR_METER_IMI] = instr_meter_imi_exec,
7037 [INSTR_TABLE] = instr_table_exec,
7038 [INSTR_TABLE_AF] = instr_table_af_exec,
7039 [INSTR_SELECTOR] = instr_selector_exec,
7040 [INSTR_LEARNER] = instr_learner_exec,
7041 [INSTR_LEARNER_AF] = instr_learner_af_exec,
7042 [INSTR_LEARNER_LEARN] = instr_learn_exec,
7043 [INSTR_LEARNER_FORGET] = instr_forget_exec,
7044 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
7045 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
7047 [INSTR_JMP] = instr_jmp_exec,
7048 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
7049 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
7050 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
7051 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
7052 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
7053 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
7055 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
7056 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
7057 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
7058 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
7059 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
7061 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
7062 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
7063 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
7064 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
7065 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
7067 [INSTR_JMP_LT] = instr_jmp_lt_exec,
7068 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
7069 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
7070 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
7071 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
7072 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
7074 [INSTR_JMP_GT] = instr_jmp_gt_exec,
7075 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
7076 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
7077 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
7078 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
7079 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
7081 [INSTR_RETURN] = instr_return_exec,
7085 instruction_table_build(struct rte_swx_pipeline *p)
7087 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
7088 sizeof(struct instr_exec_t *));
7089 if (!p->instruction_table)
7092 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
7098 instruction_table_build_free(struct rte_swx_pipeline *p)
7100 if (!p->instruction_table)
7103 free(p->instruction_table);
7104 p->instruction_table = NULL;
7108 instruction_table_free(struct rte_swx_pipeline *p)
7110 instruction_table_build_free(p);
7114 instr_exec(struct rte_swx_pipeline *p)
7116 struct thread *t = &p->threads[p->thread_id];
7117 struct instruction *ip = t->ip;
7118 instr_exec_t instr = p->instruction_table[ip->type];
7126 static struct action *
7127 action_find(struct rte_swx_pipeline *p, const char *name)
7129 struct action *elem;
7134 TAILQ_FOREACH(elem, &p->actions, node)
7135 if (strcmp(elem->name, name) == 0)
7141 static struct action *
7142 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7144 struct action *action = NULL;
7146 TAILQ_FOREACH(action, &p->actions, node)
7147 if (action->id == id)
7153 static struct field *
7154 action_field_find(struct action *a, const char *name)
7156 return a->st ? struct_type_field_find(a->st, name) : NULL;
7159 static struct field *
7160 action_field_parse(struct action *action, const char *name)
7162 if (name[0] != 't' || name[1] != '.')
7165 return action_field_find(action, &name[2]);
7169 action_has_nbo_args(struct action *a)
7173 /* Return if the action does not have any args. */
7175 return 0; /* FALSE */
7177 for (i = 0; i < a->st->n_fields; i++)
7178 if (a->args_endianness[i])
7179 return 1; /* TRUE */
7181 return 0; /* FALSE */
7185 action_does_learning(struct action *a)
7189 for (i = 0; i < a->n_instructions; i++)
7190 switch (a->instructions[i].type) {
7191 case INSTR_LEARNER_LEARN:
7192 return 1; /* TRUE */
7194 case INSTR_LEARNER_FORGET:
7195 return 1; /* TRUE */
7201 return 0; /* FALSE */
7205 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
7207 const char *args_struct_type_name,
7208 const char **instructions,
7209 uint32_t n_instructions)
7211 struct struct_type *args_struct_type = NULL;
7212 struct action *a = NULL;
7217 CHECK_NAME(name, EINVAL);
7218 CHECK(!action_find(p, name), EEXIST);
7220 if (args_struct_type_name) {
7221 CHECK_NAME(args_struct_type_name, EINVAL);
7222 args_struct_type = struct_type_find(p, args_struct_type_name);
7223 CHECK(args_struct_type, EINVAL);
7224 CHECK(!args_struct_type->var_size, EINVAL);
7227 /* Node allocation. */
7228 a = calloc(1, sizeof(struct action));
7234 if (args_struct_type) {
7235 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7236 if (!a->args_endianness) {
7242 /* Node initialization. */
7243 strcpy(a->name, name);
7244 a->st = args_struct_type;
7245 a->id = p->n_actions;
7247 /* Instruction translation. */
7248 status = instruction_config(p, a, instructions, n_instructions);
7252 /* Node add to tailq. */
7253 TAILQ_INSERT_TAIL(&p->actions, a, node);
7262 free(a->args_endianness);
7263 free(a->instructions);
7264 free(a->instruction_data);
7271 action_build(struct rte_swx_pipeline *p)
7273 struct action *action;
7275 /* p->action_instructions. */
7276 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7277 CHECK(p->action_instructions, ENOMEM);
7279 TAILQ_FOREACH(action, &p->actions, node)
7280 p->action_instructions[action->id] = action->instructions;
7282 /* p->action_funcs. */
7283 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7284 CHECK(p->action_funcs, ENOMEM);
7290 action_build_free(struct rte_swx_pipeline *p)
7292 free(p->action_funcs);
7293 p->action_funcs = NULL;
7295 free(p->action_instructions);
7296 p->action_instructions = NULL;
7300 action_free(struct rte_swx_pipeline *p)
7302 action_build_free(p);
7305 struct action *action;
7307 action = TAILQ_FIRST(&p->actions);
7311 TAILQ_REMOVE(&p->actions, action, node);
7312 free(action->args_endianness);
7313 free(action->instructions);
7314 free(action->instruction_data);
7320 action_arg_src_mov_count(struct action *a,
7322 struct instruction *instructions,
7323 struct instruction_data *instruction_data,
7324 uint32_t n_instructions)
7326 uint32_t offset, n_users = 0, i;
7329 (arg_id >= a->st->n_fields) ||
7331 !instruction_data ||
7335 offset = a->st->fields[arg_id].offset / 8;
7337 for (i = 0; i < n_instructions; i++) {
7338 struct instruction *instr = &instructions[i];
7339 struct instruction_data *data = &instruction_data[i];
7341 if (data->invalid ||
7342 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7343 instr->mov.src.struct_id ||
7344 (instr->mov.src.offset != offset))
7353 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
7354 #define field_ntoh(val, n_bits) (ntoh64((val) << (64 - n_bits)))
7355 #define field_hton(val, n_bits) (hton64((val) << (64 - n_bits)))
7357 #define field_ntoh(val, n_bits) (val)
7358 #define field_hton(val, n_bits) (val)
7361 #define ACTION_ARGS_TOKENS_MAX 256
7364 action_args_parse(struct action *a, const char *args, uint8_t *data)
7366 char *tokens[ACTION_ARGS_TOKENS_MAX], *s0 = NULL, *s;
7367 uint32_t n_tokens = 0, offset = 0, i;
7371 if (!a->st || !args || !args[0]) {
7376 /* Memory allocation. */
7383 /* Parse the string into tokens. */
7387 token = strtok_r(s, " \f\n\r\t\v", &s);
7391 if (n_tokens >= RTE_DIM(tokens)) {
7396 tokens[n_tokens] = token;
7401 if (n_tokens != a->st->n_fields * 2) {
7406 /* Process the action arguments. */
7407 for (i = 0; i < a->st->n_fields; i++) {
7408 struct field *f = &a->st->fields[i];
7409 char *arg_name = tokens[i * 2];
7410 char *arg_val = tokens[i * 2 + 1];
7413 if (strcmp(arg_name, f->name)) {
7418 val = strtoull(arg_val, &arg_val, 0);
7424 /* Endianness conversion. */
7425 if (a->args_endianness[i])
7426 val = field_hton(val, f->n_bits);
7428 /* Copy to entry. */
7429 memcpy(&data[offset], (uint8_t *)&val, f->n_bits / 8);
7430 offset += f->n_bits / 8;
7441 static struct table_type *
7442 table_type_find(struct rte_swx_pipeline *p, const char *name)
7444 struct table_type *elem;
7446 TAILQ_FOREACH(elem, &p->table_types, node)
7447 if (strcmp(elem->name, name) == 0)
7453 static struct table_type *
7454 table_type_resolve(struct rte_swx_pipeline *p,
7455 const char *recommended_type_name,
7456 enum rte_swx_table_match_type match_type)
7458 struct table_type *elem;
7460 /* Only consider the recommended type if the match type is correct. */
7461 if (recommended_type_name)
7462 TAILQ_FOREACH(elem, &p->table_types, node)
7463 if (!strcmp(elem->name, recommended_type_name) &&
7464 (elem->match_type == match_type))
7467 /* Ignore the recommended type and get the first element with this match
7470 TAILQ_FOREACH(elem, &p->table_types, node)
7471 if (elem->match_type == match_type)
7477 static struct table *
7478 table_find(struct rte_swx_pipeline *p, const char *name)
7482 TAILQ_FOREACH(elem, &p->tables, node)
7483 if (strcmp(elem->name, name) == 0)
7489 static struct table *
7490 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7492 struct table *table = NULL;
7494 TAILQ_FOREACH(table, &p->tables, node)
7495 if (table->id == id)
7502 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7504 enum rte_swx_table_match_type match_type,
7505 struct rte_swx_table_ops *ops)
7507 struct table_type *elem;
7511 CHECK_NAME(name, EINVAL);
7512 CHECK(!table_type_find(p, name), EEXIST);
7515 CHECK(ops->create, EINVAL);
7516 CHECK(ops->lkp, EINVAL);
7517 CHECK(ops->free, EINVAL);
7519 /* Node allocation. */
7520 elem = calloc(1, sizeof(struct table_type));
7521 CHECK(elem, ENOMEM);
7523 /* Node initialization. */
7524 strcpy(elem->name, name);
7525 elem->match_type = match_type;
7526 memcpy(&elem->ops, ops, sizeof(*ops));
7528 /* Node add to tailq. */
7529 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7535 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7537 enum rte_swx_table_match_type *match_type)
7539 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7541 for (i = 0; i < n_fields; i++) {
7542 struct rte_swx_match_field_params *f = &fields[i];
7544 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7547 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7551 if ((n_fields_lpm > 1) ||
7552 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7555 *match_type = (n_fields_em == n_fields) ?
7556 RTE_SWX_TABLE_MATCH_EXACT :
7557 RTE_SWX_TABLE_MATCH_WILDCARD;
7563 table_match_fields_check(struct rte_swx_pipeline *p,
7564 struct rte_swx_pipeline_table_params *params,
7565 struct header **header)
7567 struct header *h0 = NULL;
7568 struct field *hf, *mf;
7569 uint32_t *offset = NULL, i;
7572 /* Return if no match fields. */
7573 if (!params->n_fields) {
7574 if (params->fields) {
7585 /* Memory allocation. */
7586 offset = calloc(params->n_fields, sizeof(uint32_t));
7592 /* Check that all the match fields belong to either the same header or
7595 hf = header_field_parse(p, params->fields[0].name, &h0);
7596 mf = metadata_field_parse(p, params->fields[0].name);
7597 if ((!hf && !mf) || (hf && hf->var_size)) {
7602 offset[0] = h0 ? hf->offset : mf->offset;
7604 for (i = 1; i < params->n_fields; i++)
7608 hf = header_field_parse(p, params->fields[i].name, &h);
7609 if (!hf || (h->id != h0->id) || hf->var_size) {
7614 offset[i] = hf->offset;
7616 mf = metadata_field_parse(p, params->fields[i].name);
7622 offset[i] = mf->offset;
7625 /* Check that there are no duplicated match fields. */
7626 for (i = 0; i < params->n_fields; i++) {
7629 for (j = 0; j < i; j++)
7630 if (offset[j] == offset[i]) {
7646 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7648 struct rte_swx_pipeline_table_params *params,
7649 const char *recommended_table_type_name,
7653 struct table_type *type;
7654 struct table *t = NULL;
7655 struct action *default_action;
7656 struct header *header = NULL;
7657 uint32_t action_data_size_max = 0, i;
7662 CHECK_NAME(name, EINVAL);
7663 CHECK(!table_find(p, name), EEXIST);
7664 CHECK(!selector_find(p, name), EEXIST);
7665 CHECK(!learner_find(p, name), EEXIST);
7667 CHECK(params, EINVAL);
7670 status = table_match_fields_check(p, params, &header);
7674 /* Action checks. */
7675 CHECK(params->n_actions, EINVAL);
7676 CHECK(params->action_names, EINVAL);
7677 for (i = 0; i < params->n_actions; i++) {
7678 const char *action_name = params->action_names[i];
7680 uint32_t action_data_size;
7681 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7683 CHECK_NAME(action_name, EINVAL);
7685 a = action_find(p, action_name);
7687 CHECK(!action_does_learning(a), EINVAL);
7689 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7690 if (action_data_size > action_data_size_max)
7691 action_data_size_max = action_data_size;
7693 if (params->action_is_for_table_entries)
7694 action_is_for_table_entries = params->action_is_for_table_entries[i];
7695 if (params->action_is_for_default_entry)
7696 action_is_for_default_entry = params->action_is_for_default_entry[i];
7697 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7700 CHECK_NAME(params->default_action_name, EINVAL);
7701 for (i = 0; i < p->n_actions; i++)
7702 if (!strcmp(params->action_names[i],
7703 params->default_action_name))
7705 CHECK(i < params->n_actions, EINVAL);
7706 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7709 default_action = action_find(p, params->default_action_name);
7710 CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
7713 /* Table type checks. */
7714 if (recommended_table_type_name)
7715 CHECK_NAME(recommended_table_type_name, EINVAL);
7717 if (params->n_fields) {
7718 enum rte_swx_table_match_type match_type;
7720 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7724 type = table_type_resolve(p, recommended_table_type_name, match_type);
7725 CHECK(type, EINVAL);
7730 /* Memory allocation. */
7731 t = calloc(1, sizeof(struct table));
7737 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7743 t->actions = calloc(params->n_actions, sizeof(struct action *));
7749 if (action_data_size_max) {
7750 t->default_action_data = calloc(1, action_data_size_max);
7751 if (!t->default_action_data) {
7757 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7758 if (!t->action_is_for_table_entries) {
7763 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7764 if (!t->action_is_for_default_entry) {
7769 /* Node initialization. */
7770 strcpy(t->name, name);
7771 if (args && args[0])
7772 strcpy(t->args, args);
7775 for (i = 0; i < params->n_fields; i++) {
7776 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7777 struct match_field *f = &t->fields[i];
7779 f->match_type = field->match_type;
7781 header_field_parse(p, field->name, NULL) :
7782 metadata_field_parse(p, field->name);
7784 t->n_fields = params->n_fields;
7787 for (i = 0; i < params->n_actions; i++) {
7788 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7790 if (params->action_is_for_table_entries)
7791 action_is_for_table_entries = params->action_is_for_table_entries[i];
7792 if (params->action_is_for_default_entry)
7793 action_is_for_default_entry = params->action_is_for_default_entry[i];
7795 t->actions[i] = action_find(p, params->action_names[i]);
7796 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7797 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7799 t->default_action = default_action;
7800 if (default_action->st) {
7801 status = action_args_parse(default_action,
7802 params->default_action_args,
7803 t->default_action_data);
7808 t->n_actions = params->n_actions;
7809 t->default_action_is_const = params->default_action_is_const;
7810 t->action_data_size_max = action_data_size_max;
7813 t->id = p->n_tables;
7815 /* Node add to tailq. */
7816 TAILQ_INSERT_TAIL(&p->tables, t, node);
7825 free(t->action_is_for_default_entry);
7826 free(t->action_is_for_table_entries);
7827 free(t->default_action_data);
7835 static struct rte_swx_table_params *
7836 table_params_get(struct table *table)
7838 struct rte_swx_table_params *params;
7839 struct field *first, *last;
7841 uint32_t key_size, key_offset, action_data_size, i;
7843 /* Memory allocation. */
7844 params = calloc(1, sizeof(struct rte_swx_table_params));
7848 /* Find first (smallest offset) and last (biggest offset) match fields. */
7849 first = table->fields[0].field;
7850 last = table->fields[0].field;
7852 for (i = 0; i < table->n_fields; i++) {
7853 struct field *f = table->fields[i].field;
7855 if (f->offset < first->offset)
7858 if (f->offset > last->offset)
7862 /* Key offset and size. */
7863 key_offset = first->offset / 8;
7864 key_size = (last->offset + last->n_bits - first->offset) / 8;
7866 /* Memory allocation. */
7867 key_mask = calloc(1, key_size);
7874 for (i = 0; i < table->n_fields; i++) {
7875 struct field *f = table->fields[i].field;
7876 uint32_t start = (f->offset - first->offset) / 8;
7877 size_t size = f->n_bits / 8;
7879 memset(&key_mask[start], 0xFF, size);
7882 /* Action data size. */
7883 action_data_size = 0;
7884 for (i = 0; i < table->n_actions; i++) {
7885 struct action *action = table->actions[i];
7886 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7888 if (ads > action_data_size)
7889 action_data_size = ads;
7893 params->match_type = table->type->match_type;
7894 params->key_size = key_size;
7895 params->key_offset = key_offset;
7896 params->key_mask0 = key_mask;
7897 params->action_data_size = action_data_size;
7898 params->n_keys_max = table->size;
7904 table_params_free(struct rte_swx_table_params *params)
7909 free(params->key_mask0);
7914 table_stub_lkp(void *table __rte_unused,
7915 void *mailbox __rte_unused,
7916 uint8_t **key __rte_unused,
7917 uint64_t *action_id __rte_unused,
7918 uint8_t **action_data __rte_unused,
7922 return 1; /* DONE. */
7926 table_build(struct rte_swx_pipeline *p)
7930 /* Per pipeline: table statistics. */
7931 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7932 CHECK(p->table_stats, ENOMEM);
7934 for (i = 0; i < p->n_tables; i++) {
7935 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7936 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7939 /* Per thread: table runt-time. */
7940 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7941 struct thread *t = &p->threads[i];
7942 struct table *table;
7944 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7945 CHECK(t->tables, ENOMEM);
7947 TAILQ_FOREACH(table, &p->tables, node) {
7948 struct table_runtime *r = &t->tables[table->id];
7953 size = table->type->ops.mailbox_size_get();
7956 r->func = table->type->ops.lkp;
7960 r->mailbox = calloc(1, size);
7961 CHECK(r->mailbox, ENOMEM);
7965 r->key = table->header ?
7966 &t->structs[table->header->struct_id] :
7967 &t->structs[p->metadata_struct_id];
7969 r->func = table_stub_lkp;
7978 table_build_free(struct rte_swx_pipeline *p)
7982 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7983 struct thread *t = &p->threads[i];
7989 for (j = 0; j < p->n_tables; j++) {
7990 struct table_runtime *r = &t->tables[j];
7999 if (p->table_stats) {
8000 for (i = 0; i < p->n_tables; i++)
8001 free(p->table_stats[i].n_pkts_action);
8003 free(p->table_stats);
8008 table_free(struct rte_swx_pipeline *p)
8010 table_build_free(p);
8016 elem = TAILQ_FIRST(&p->tables);
8020 TAILQ_REMOVE(&p->tables, elem, node);
8022 free(elem->actions);
8023 free(elem->default_action_data);
8029 struct table_type *elem;
8031 elem = TAILQ_FIRST(&p->table_types);
8035 TAILQ_REMOVE(&p->table_types, elem, node);
8043 static struct selector *
8044 selector_find(struct rte_swx_pipeline *p, const char *name)
8048 TAILQ_FOREACH(s, &p->selectors, node)
8049 if (strcmp(s->name, name) == 0)
8055 static struct selector *
8056 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8058 struct selector *s = NULL;
8060 TAILQ_FOREACH(s, &p->selectors, node)
8068 selector_fields_check(struct rte_swx_pipeline *p,
8069 struct rte_swx_pipeline_selector_params *params,
8070 struct header **header)
8072 struct header *h0 = NULL;
8073 struct field *hf, *mf;
8076 /* Return if no selector fields. */
8077 if (!params->n_selector_fields || !params->selector_field_names)
8080 /* Check that all the selector fields either belong to the same header
8081 * or are all meta-data fields.
8083 hf = header_field_parse(p, params->selector_field_names[0], &h0);
8084 mf = metadata_field_parse(p, params->selector_field_names[0]);
8088 for (i = 1; i < params->n_selector_fields; i++)
8092 hf = header_field_parse(p, params->selector_field_names[i], &h);
8093 if (!hf || (h->id != h0->id))
8096 mf = metadata_field_parse(p, params->selector_field_names[i]);
8101 /* Check that there are no duplicated match fields. */
8102 for (i = 0; i < params->n_selector_fields; i++) {
8103 const char *field_name = params->selector_field_names[i];
8106 for (j = i + 1; j < params->n_selector_fields; j++)
8107 if (!strcmp(params->selector_field_names[j], field_name))
8119 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
8121 struct rte_swx_pipeline_selector_params *params)
8124 struct header *selector_header = NULL;
8125 struct field *group_id_field, *member_id_field;
8131 CHECK_NAME(name, EINVAL);
8132 CHECK(!table_find(p, name), EEXIST);
8133 CHECK(!selector_find(p, name), EEXIST);
8134 CHECK(!learner_find(p, name), EEXIST);
8136 CHECK(params, EINVAL);
8138 CHECK_NAME(params->group_id_field_name, EINVAL);
8139 group_id_field = metadata_field_parse(p, params->group_id_field_name);
8140 CHECK(group_id_field, EINVAL);
8142 for (i = 0; i < params->n_selector_fields; i++) {
8143 const char *field_name = params->selector_field_names[i];
8145 CHECK_NAME(field_name, EINVAL);
8147 status = selector_fields_check(p, params, &selector_header);
8151 CHECK_NAME(params->member_id_field_name, EINVAL);
8152 member_id_field = metadata_field_parse(p, params->member_id_field_name);
8153 CHECK(member_id_field, EINVAL);
8155 CHECK(params->n_groups_max, EINVAL);
8157 CHECK(params->n_members_per_group_max, EINVAL);
8159 /* Memory allocation. */
8160 s = calloc(1, sizeof(struct selector));
8166 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
8167 if (!s->selector_fields) {
8172 /* Node initialization. */
8173 strcpy(s->name, name);
8175 s->group_id_field = group_id_field;
8177 for (i = 0; i < params->n_selector_fields; i++) {
8178 const char *field_name = params->selector_field_names[i];
8180 s->selector_fields[i] = selector_header ?
8181 header_field_parse(p, field_name, NULL) :
8182 metadata_field_parse(p, field_name);
8185 s->n_selector_fields = params->n_selector_fields;
8187 s->selector_header = selector_header;
8189 s->member_id_field = member_id_field;
8191 s->n_groups_max = params->n_groups_max;
8193 s->n_members_per_group_max = params->n_members_per_group_max;
8195 s->id = p->n_selectors;
8197 /* Node add to tailq. */
8198 TAILQ_INSERT_TAIL(&p->selectors, s, node);
8207 free(s->selector_fields);
8215 selector_params_free(struct rte_swx_table_selector_params *params)
8220 free(params->selector_mask);
8225 static struct rte_swx_table_selector_params *
8226 selector_table_params_get(struct selector *s)
8228 struct rte_swx_table_selector_params *params = NULL;
8229 struct field *first, *last;
8232 /* Memory allocation. */
8233 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
8238 params->group_id_offset = s->group_id_field->offset / 8;
8240 /* Find first (smallest offset) and last (biggest offset) selector fields. */
8241 first = s->selector_fields[0];
8242 last = s->selector_fields[0];
8244 for (i = 0; i < s->n_selector_fields; i++) {
8245 struct field *f = s->selector_fields[i];
8247 if (f->offset < first->offset)
8250 if (f->offset > last->offset)
8254 /* Selector offset and size. */
8255 params->selector_offset = first->offset / 8;
8256 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
8258 /* Memory allocation. */
8259 params->selector_mask = calloc(1, params->selector_size);
8260 if (!params->selector_mask)
8263 /* Selector mask. */
8264 for (i = 0; i < s->n_selector_fields; i++) {
8265 struct field *f = s->selector_fields[i];
8266 uint32_t start = (f->offset - first->offset) / 8;
8267 size_t size = f->n_bits / 8;
8269 memset(¶ms->selector_mask[start], 0xFF, size);
8273 params->member_id_offset = s->member_id_field->offset / 8;
8275 /* Maximum number of groups. */
8276 params->n_groups_max = s->n_groups_max;
8278 /* Maximum number of members per group. */
8279 params->n_members_per_group_max = s->n_members_per_group_max;
8284 selector_params_free(params);
8289 selector_build_free(struct rte_swx_pipeline *p)
8293 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8294 struct thread *t = &p->threads[i];
8300 for (j = 0; j < p->n_selectors; j++) {
8301 struct selector_runtime *r = &t->selectors[j];
8307 t->selectors = NULL;
8310 free(p->selector_stats);
8311 p->selector_stats = NULL;
8315 selector_build(struct rte_swx_pipeline *p)
8320 /* Per pipeline: selector statistics. */
8321 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
8322 if (!p->selector_stats) {
8327 /* Per thread: selector run-time. */
8328 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8329 struct thread *t = &p->threads[i];
8332 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8333 if (!t->selectors) {
8338 TAILQ_FOREACH(s, &p->selectors, node) {
8339 struct selector_runtime *r = &t->selectors[s->id];
8343 size = rte_swx_table_selector_mailbox_size_get();
8345 r->mailbox = calloc(1, size);
8352 /* r->group_id_buffer. */
8353 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8355 /* r->selector_buffer. */
8356 r->selector_buffer = s->selector_header ?
8357 &t->structs[s->selector_header->struct_id] :
8358 &t->structs[p->metadata_struct_id];
8360 /* r->member_id_buffer. */
8361 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8368 selector_build_free(p);
8373 selector_free(struct rte_swx_pipeline *p)
8375 selector_build_free(p);
8377 /* Selector tables. */
8379 struct selector *elem;
8381 elem = TAILQ_FIRST(&p->selectors);
8385 TAILQ_REMOVE(&p->selectors, elem, node);
8386 free(elem->selector_fields);
8394 static struct learner *
8395 learner_find(struct rte_swx_pipeline *p, const char *name)
8399 TAILQ_FOREACH(l, &p->learners, node)
8400 if (!strcmp(l->name, name))
8406 static struct learner *
8407 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8409 struct learner *l = NULL;
8411 TAILQ_FOREACH(l, &p->learners, node)
8419 learner_match_fields_check(struct rte_swx_pipeline *p,
8420 struct rte_swx_pipeline_learner_params *params,
8421 struct header **header)
8423 struct header *h0 = NULL;
8424 struct field *hf, *mf;
8427 /* Return if no match fields. */
8428 if (!params->n_fields || !params->field_names)
8431 /* Check that all the match fields either belong to the same header
8432 * or are all meta-data fields.
8434 hf = header_field_parse(p, params->field_names[0], &h0);
8435 mf = metadata_field_parse(p, params->field_names[0]);
8439 for (i = 1; i < params->n_fields; i++)
8443 hf = header_field_parse(p, params->field_names[i], &h);
8444 if (!hf || (h->id != h0->id))
8447 mf = metadata_field_parse(p, params->field_names[i]);
8452 /* Check that there are no duplicated match fields. */
8453 for (i = 0; i < params->n_fields; i++) {
8454 const char *field_name = params->field_names[i];
8457 for (j = i + 1; j < params->n_fields; j++)
8458 if (!strcmp(params->field_names[j], field_name))
8470 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8472 struct struct_type *mst = p->metadata_st, *ast = a->st;
8473 struct field *mf, *af;
8483 /* Check that mf_name is the name of a valid meta-data field. */
8484 CHECK_NAME(mf_name, EINVAL);
8485 mf = metadata_field_parse(p, mf_name);
8488 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8489 * all the action arguments.
8491 mf_pos = mf - mst->fields;
8492 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8494 /* Check that the size of each of the identified meta-data fields matches exactly the size
8495 * of the corresponding action argument.
8497 for (i = 0; i < ast->n_fields; i++) {
8498 mf = &mst->fields[mf_pos + i];
8499 af = &ast->fields[i];
8501 CHECK(mf->n_bits == af->n_bits, EINVAL);
8508 learner_action_learning_check(struct rte_swx_pipeline *p,
8509 struct action *action,
8510 const char **action_names,
8515 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8516 * the action passed as argument to the "learn" instruction) is also enabled for the
8517 * current learner table.
8519 for (i = 0; i < action->n_instructions; i++) {
8520 struct instruction *instr = &action->instructions[i];
8521 uint32_t found = 0, j;
8523 if (instr->type != INSTR_LEARNER_LEARN)
8526 for (j = 0; j < n_actions; j++) {
8529 a = action_find(p, action_names[j]);
8533 if (a->id == instr->learn.action_id)
8545 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8547 struct rte_swx_pipeline_learner_params *params,
8551 struct learner *l = NULL;
8552 struct action *default_action;
8553 struct header *header = NULL;
8554 uint32_t action_data_size_max = 0, i;
8559 CHECK_NAME(name, EINVAL);
8560 CHECK(!table_find(p, name), EEXIST);
8561 CHECK(!selector_find(p, name), EEXIST);
8562 CHECK(!learner_find(p, name), EEXIST);
8564 CHECK(params, EINVAL);
8567 status = learner_match_fields_check(p, params, &header);
8571 /* Action checks. */
8572 CHECK(params->n_actions, EINVAL);
8573 CHECK(params->action_names, EINVAL);
8574 for (i = 0; i < params->n_actions; i++) {
8575 const char *action_name = params->action_names[i];
8577 uint32_t action_data_size;
8578 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8580 CHECK_NAME(action_name, EINVAL);
8582 a = action_find(p, action_name);
8585 status = learner_action_learning_check(p,
8587 params->action_names,
8592 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8593 if (action_data_size > action_data_size_max)
8594 action_data_size_max = action_data_size;
8596 if (params->action_is_for_table_entries)
8597 action_is_for_table_entries = params->action_is_for_table_entries[i];
8598 if (params->action_is_for_default_entry)
8599 action_is_for_default_entry = params->action_is_for_default_entry[i];
8600 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8603 CHECK_NAME(params->default_action_name, EINVAL);
8604 for (i = 0; i < p->n_actions; i++)
8605 if (!strcmp(params->action_names[i],
8606 params->default_action_name))
8608 CHECK(i < params->n_actions, EINVAL);
8609 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8612 default_action = action_find(p, params->default_action_name);
8613 CHECK((default_action->st && params->default_action_args) || !params->default_action_args,
8616 /* Any other checks. */
8617 CHECK(size, EINVAL);
8618 CHECK(timeout, EINVAL);
8620 /* Memory allocation. */
8621 l = calloc(1, sizeof(struct learner));
8627 l->fields = calloc(params->n_fields, sizeof(struct field *));
8633 l->actions = calloc(params->n_actions, sizeof(struct action *));
8639 if (action_data_size_max) {
8640 l->default_action_data = calloc(1, action_data_size_max);
8641 if (!l->default_action_data) {
8647 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8648 if (!l->action_is_for_table_entries) {
8653 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8654 if (!l->action_is_for_default_entry) {
8659 /* Node initialization. */
8660 strcpy(l->name, name);
8662 for (i = 0; i < params->n_fields; i++) {
8663 const char *field_name = params->field_names[i];
8665 l->fields[i] = header ?
8666 header_field_parse(p, field_name, NULL) :
8667 metadata_field_parse(p, field_name);
8670 l->n_fields = params->n_fields;
8674 for (i = 0; i < params->n_actions; i++) {
8675 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8677 if (params->action_is_for_table_entries)
8678 action_is_for_table_entries = params->action_is_for_table_entries[i];
8679 if (params->action_is_for_default_entry)
8680 action_is_for_default_entry = params->action_is_for_default_entry[i];
8682 l->actions[i] = action_find(p, params->action_names[i]);
8683 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8684 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8687 l->default_action = default_action;
8689 if (default_action->st) {
8690 status = action_args_parse(default_action,
8691 params->default_action_args,
8692 l->default_action_data);
8697 l->n_actions = params->n_actions;
8699 l->default_action_is_const = params->default_action_is_const;
8701 l->action_data_size_max = action_data_size_max;
8705 l->timeout = timeout;
8707 l->id = p->n_learners;
8709 /* Node add to tailq. */
8710 TAILQ_INSERT_TAIL(&p->learners, l, node);
8719 free(l->action_is_for_default_entry);
8720 free(l->action_is_for_table_entries);
8721 free(l->default_action_data);
8730 learner_params_free(struct rte_swx_table_learner_params *params)
8735 free(params->key_mask0);
8740 static struct rte_swx_table_learner_params *
8741 learner_params_get(struct learner *l)
8743 struct rte_swx_table_learner_params *params = NULL;
8744 struct field *first, *last;
8747 /* Memory allocation. */
8748 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8752 /* Find first (smallest offset) and last (biggest offset) match fields. */
8753 first = l->fields[0];
8754 last = l->fields[0];
8756 for (i = 0; i < l->n_fields; i++) {
8757 struct field *f = l->fields[i];
8759 if (f->offset < first->offset)
8762 if (f->offset > last->offset)
8766 /* Key offset and size. */
8767 params->key_offset = first->offset / 8;
8768 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8770 /* Memory allocation. */
8771 params->key_mask0 = calloc(1, params->key_size);
8772 if (!params->key_mask0)
8776 for (i = 0; i < l->n_fields; i++) {
8777 struct field *f = l->fields[i];
8778 uint32_t start = (f->offset - first->offset) / 8;
8779 size_t size = f->n_bits / 8;
8781 memset(¶ms->key_mask0[start], 0xFF, size);
8784 /* Action data size. */
8785 params->action_data_size = l->action_data_size_max;
8787 /* Maximum number of keys. */
8788 params->n_keys_max = l->size;
8791 params->key_timeout[0] = l->timeout;
8792 params->n_key_timeouts = 1;
8797 learner_params_free(params);
8802 learner_build_free(struct rte_swx_pipeline *p)
8806 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8807 struct thread *t = &p->threads[i];
8813 for (j = 0; j < p->n_learners; j++) {
8814 struct learner_runtime *r = &t->learners[j];
8823 if (p->learner_stats) {
8824 for (i = 0; i < p->n_learners; i++)
8825 free(p->learner_stats[i].n_pkts_action);
8827 free(p->learner_stats);
8832 learner_build(struct rte_swx_pipeline *p)
8837 /* Per pipeline: learner statistics. */
8838 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8839 CHECK(p->learner_stats, ENOMEM);
8841 for (i = 0; i < p->n_learners; i++) {
8842 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8843 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8846 /* Per thread: learner run-time. */
8847 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8848 struct thread *t = &p->threads[i];
8851 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8857 TAILQ_FOREACH(l, &p->learners, node) {
8858 struct learner_runtime *r = &t->learners[l->id];
8862 size = rte_swx_table_learner_mailbox_size_get();
8864 r->mailbox = calloc(1, size);
8872 r->key = l->header ?
8873 &t->structs[l->header->struct_id] :
8874 &t->structs[p->metadata_struct_id];
8881 learner_build_free(p);
8886 learner_free(struct rte_swx_pipeline *p)
8888 learner_build_free(p);
8890 /* Learner tables. */
8894 l = TAILQ_FIRST(&p->learners);
8898 TAILQ_REMOVE(&p->learners, l, node);
8901 free(l->default_action_data);
8910 table_state_build(struct rte_swx_pipeline *p)
8912 struct table *table;
8916 p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
8917 sizeof(struct rte_swx_table_state));
8918 CHECK(p->table_state, ENOMEM);
8920 TAILQ_FOREACH(table, &p->tables, node) {
8921 struct rte_swx_table_state *ts = &p->table_state[table->id];
8924 struct rte_swx_table_params *params;
8927 params = table_params_get(table);
8928 CHECK(params, ENOMEM);
8930 ts->obj = table->type->ops.create(params,
8935 table_params_free(params);
8936 CHECK(ts->obj, ENODEV);
8939 /* ts->default_action_data. */
8940 if (table->action_data_size_max) {
8941 ts->default_action_data =
8942 malloc(table->action_data_size_max);
8943 CHECK(ts->default_action_data, ENOMEM);
8945 memcpy(ts->default_action_data,
8946 table->default_action_data,
8947 table->action_data_size_max);
8950 /* ts->default_action_id. */
8951 ts->default_action_id = table->default_action->id;
8954 TAILQ_FOREACH(s, &p->selectors, node) {
8955 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8956 struct rte_swx_table_selector_params *params;
8959 params = selector_table_params_get(s);
8960 CHECK(params, ENOMEM);
8962 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8964 selector_params_free(params);
8965 CHECK(ts->obj, ENODEV);
8968 TAILQ_FOREACH(l, &p->learners, node) {
8969 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8970 p->n_selectors + l->id];
8971 struct rte_swx_table_learner_params *params;
8974 params = learner_params_get(l);
8975 CHECK(params, ENOMEM);
8977 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8978 learner_params_free(params);
8979 CHECK(ts->obj, ENODEV);
8981 /* ts->default_action_data. */
8982 if (l->action_data_size_max) {
8983 ts->default_action_data = malloc(l->action_data_size_max);
8984 CHECK(ts->default_action_data, ENOMEM);
8986 memcpy(ts->default_action_data,
8987 l->default_action_data,
8988 l->action_data_size_max);
8991 /* ts->default_action_id. */
8992 ts->default_action_id = l->default_action->id;
8999 table_state_build_free(struct rte_swx_pipeline *p)
9003 if (!p->table_state)
9006 for (i = 0; i < p->n_tables; i++) {
9007 struct rte_swx_table_state *ts = &p->table_state[i];
9008 struct table *table = table_find_by_id(p, i);
9011 if (table->type && ts->obj)
9012 table->type->ops.free(ts->obj);
9014 /* ts->default_action_data. */
9015 free(ts->default_action_data);
9018 for (i = 0; i < p->n_selectors; i++) {
9019 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
9023 rte_swx_table_selector_free(ts->obj);
9026 for (i = 0; i < p->n_learners; i++) {
9027 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
9031 rte_swx_table_learner_free(ts->obj);
9033 /* ts->default_action_data. */
9034 free(ts->default_action_data);
9037 free(p->table_state);
9038 p->table_state = NULL;
9042 table_state_free(struct rte_swx_pipeline *p)
9044 table_state_build_free(p);
9050 static struct regarray *
9051 regarray_find(struct rte_swx_pipeline *p, const char *name)
9053 struct regarray *elem;
9055 TAILQ_FOREACH(elem, &p->regarrays, node)
9056 if (!strcmp(elem->name, name))
9062 static struct regarray *
9063 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9065 struct regarray *elem = NULL;
9067 TAILQ_FOREACH(elem, &p->regarrays, node)
9075 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9084 CHECK_NAME(name, EINVAL);
9085 CHECK(!regarray_find(p, name), EEXIST);
9087 CHECK(size, EINVAL);
9088 size = rte_align32pow2(size);
9090 /* Memory allocation. */
9091 r = calloc(1, sizeof(struct regarray));
9094 /* Node initialization. */
9095 strcpy(r->name, name);
9096 r->init_val = init_val;
9098 r->id = p->n_regarrays;
9100 /* Node add to tailq. */
9101 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9108 regarray_build(struct rte_swx_pipeline *p)
9110 struct regarray *regarray;
9112 if (!p->n_regarrays)
9115 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9116 CHECK(p->regarray_runtime, ENOMEM);
9118 TAILQ_FOREACH(regarray, &p->regarrays, node) {
9119 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9122 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9123 RTE_CACHE_LINE_SIZE,
9125 CHECK(r->regarray, ENOMEM);
9127 if (regarray->init_val)
9128 for (i = 0; i < regarray->size; i++)
9129 r->regarray[i] = regarray->init_val;
9131 r->size_mask = regarray->size - 1;
9138 regarray_build_free(struct rte_swx_pipeline *p)
9142 if (!p->regarray_runtime)
9145 for (i = 0; i < p->n_regarrays; i++) {
9146 struct regarray *regarray = regarray_find_by_id(p, i);
9147 struct regarray_runtime *r = &p->regarray_runtime[i];
9149 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9152 free(p->regarray_runtime);
9153 p->regarray_runtime = NULL;
9157 regarray_free(struct rte_swx_pipeline *p)
9159 regarray_build_free(p);
9162 struct regarray *elem;
9164 elem = TAILQ_FIRST(&p->regarrays);
9168 TAILQ_REMOVE(&p->regarrays, elem, node);
9176 static struct meter_profile *
9177 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9179 struct meter_profile *elem;
9181 TAILQ_FOREACH(elem, &p->meter_profiles, node)
9182 if (!strcmp(elem->name, name))
9188 static struct metarray *
9189 metarray_find(struct rte_swx_pipeline *p, const char *name)
9191 struct metarray *elem;
9193 TAILQ_FOREACH(elem, &p->metarrays, node)
9194 if (!strcmp(elem->name, name))
9200 static struct metarray *
9201 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9203 struct metarray *elem = NULL;
9205 TAILQ_FOREACH(elem, &p->metarrays, node)
9213 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9221 CHECK_NAME(name, EINVAL);
9222 CHECK(!metarray_find(p, name), EEXIST);
9224 CHECK(size, EINVAL);
9225 size = rte_align32pow2(size);
9227 /* Memory allocation. */
9228 m = calloc(1, sizeof(struct metarray));
9231 /* Node initialization. */
9232 strcpy(m->name, name);
9234 m->id = p->n_metarrays;
9236 /* Node add to tailq. */
9237 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9243 struct meter_profile meter_profile_default = {
9252 .cir_bytes_per_period = 1,
9254 .pir_bytes_per_period = 1,
9261 meter_init(struct meter *m)
9263 memset(m, 0, sizeof(struct meter));
9264 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9265 m->profile = &meter_profile_default;
9266 m->color_mask = RTE_COLOR_GREEN;
9268 meter_profile_default.n_users++;
9272 metarray_build(struct rte_swx_pipeline *p)
9276 if (!p->n_metarrays)
9279 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9280 CHECK(p->metarray_runtime, ENOMEM);
9282 TAILQ_FOREACH(m, &p->metarrays, node) {
9283 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9286 r->metarray = env_malloc(m->size * sizeof(struct meter),
9287 RTE_CACHE_LINE_SIZE,
9289 CHECK(r->metarray, ENOMEM);
9291 for (i = 0; i < m->size; i++)
9292 meter_init(&r->metarray[i]);
9294 r->size_mask = m->size - 1;
9301 metarray_build_free(struct rte_swx_pipeline *p)
9305 if (!p->metarray_runtime)
9308 for (i = 0; i < p->n_metarrays; i++) {
9309 struct metarray *m = metarray_find_by_id(p, i);
9310 struct metarray_runtime *r = &p->metarray_runtime[i];
9312 env_free(r->metarray, m->size * sizeof(struct meter));
9315 free(p->metarray_runtime);
9316 p->metarray_runtime = NULL;
9320 metarray_free(struct rte_swx_pipeline *p)
9322 metarray_build_free(p);
9326 struct metarray *elem;
9328 elem = TAILQ_FIRST(&p->metarrays);
9332 TAILQ_REMOVE(&p->metarrays, elem, node);
9336 /* Meter profiles. */
9338 struct meter_profile *elem;
9340 elem = TAILQ_FIRST(&p->meter_profiles);
9344 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9353 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9362 free(p->instruction_data);
9363 free(p->instructions);
9367 table_state_free(p);
9372 instruction_table_free(p);
9375 extern_func_free(p);
9389 port_in_types_register(struct rte_swx_pipeline *p)
9393 status = rte_swx_pipeline_port_in_type_register(p,
9395 &rte_swx_port_ethdev_reader_ops);
9399 status = rte_swx_pipeline_port_in_type_register(p,
9401 &rte_swx_port_ring_reader_ops);
9405 #ifdef RTE_PORT_PCAP
9406 status = rte_swx_pipeline_port_in_type_register(p,
9408 &rte_swx_port_source_ops);
9413 status = rte_swx_pipeline_port_in_type_register(p,
9415 &rte_swx_port_fd_reader_ops);
9423 port_out_types_register(struct rte_swx_pipeline *p)
9427 status = rte_swx_pipeline_port_out_type_register(p,
9429 &rte_swx_port_ethdev_writer_ops);
9433 status = rte_swx_pipeline_port_out_type_register(p,
9435 &rte_swx_port_ring_writer_ops);
9439 status = rte_swx_pipeline_port_out_type_register(p,
9441 &rte_swx_port_sink_ops);
9445 status = rte_swx_pipeline_port_out_type_register(p,
9447 &rte_swx_port_fd_writer_ops);
9455 table_types_register(struct rte_swx_pipeline *p)
9459 status = rte_swx_pipeline_table_type_register(p,
9461 RTE_SWX_TABLE_MATCH_EXACT,
9462 &rte_swx_table_exact_match_ops);
9466 status = rte_swx_pipeline_table_type_register(p,
9468 RTE_SWX_TABLE_MATCH_WILDCARD,
9469 &rte_swx_table_wildcard_match_ops);
9477 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9479 struct rte_swx_pipeline *pipeline = NULL;
9482 /* Check input parameters. */
9485 /* Memory allocation. */
9486 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9492 /* Initialization. */
9493 TAILQ_INIT(&pipeline->struct_types);
9494 TAILQ_INIT(&pipeline->port_in_types);
9495 TAILQ_INIT(&pipeline->ports_in);
9496 TAILQ_INIT(&pipeline->port_out_types);
9497 TAILQ_INIT(&pipeline->ports_out);
9498 TAILQ_INIT(&pipeline->extern_types);
9499 TAILQ_INIT(&pipeline->extern_objs);
9500 TAILQ_INIT(&pipeline->extern_funcs);
9501 TAILQ_INIT(&pipeline->headers);
9502 TAILQ_INIT(&pipeline->actions);
9503 TAILQ_INIT(&pipeline->table_types);
9504 TAILQ_INIT(&pipeline->tables);
9505 TAILQ_INIT(&pipeline->selectors);
9506 TAILQ_INIT(&pipeline->learners);
9507 TAILQ_INIT(&pipeline->regarrays);
9508 TAILQ_INIT(&pipeline->meter_profiles);
9509 TAILQ_INIT(&pipeline->metarrays);
9511 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9512 pipeline->numa_node = numa_node;
9514 status = port_in_types_register(pipeline);
9518 status = port_out_types_register(pipeline);
9522 status = table_types_register(pipeline);
9530 rte_swx_pipeline_free(pipeline);
9535 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9536 const char **instructions,
9537 uint32_t n_instructions)
9542 err = instruction_config(p, NULL, instructions, n_instructions);
9546 /* Thread instruction pointer reset. */
9547 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9548 struct thread *t = &p->threads[i];
9550 thread_ip_reset(p, t);
9557 pipeline_compile(struct rte_swx_pipeline *p);
9560 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9562 struct rte_swx_port_sink_params drop_port_params = {
9568 CHECK(p->build_done == 0, EEXIST);
9570 status = port_in_build(p);
9575 status = rte_swx_pipeline_port_out_config(p,
9582 status = port_out_build(p);
9586 status = mirroring_build(p);
9590 status = struct_build(p);
9594 status = extern_obj_build(p);
9598 status = extern_func_build(p);
9602 status = header_build(p);
9606 status = metadata_build(p);
9610 status = instruction_table_build(p);
9614 status = action_build(p);
9618 status = table_build(p);
9622 status = selector_build(p);
9626 status = learner_build(p);
9630 status = table_state_build(p);
9634 status = regarray_build(p);
9638 status = metarray_build(p);
9644 pipeline_compile(p);
9649 metarray_build_free(p);
9650 regarray_build_free(p);
9651 table_state_build_free(p);
9652 learner_build_free(p);
9653 selector_build_free(p);
9654 table_build_free(p);
9655 action_build_free(p);
9656 instruction_table_build_free(p);
9657 metadata_build_free(p);
9658 header_build_free(p);
9659 extern_func_build_free(p);
9660 extern_obj_build_free(p);
9661 mirroring_build_free(p);
9662 port_out_build_free(p);
9663 port_in_build_free(p);
9664 struct_build_free(p);
9670 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9674 for (i = 0; i < n_instructions; i++)
9679 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9683 for (i = 0; i < p->n_ports_out; i++) {
9684 struct port_out_runtime *port = &p->out[i];
9687 port->flush(port->obj);
9695 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9696 struct rte_swx_ctl_pipeline_info *pipeline)
9698 struct action *action;
9699 struct table *table;
9700 uint32_t n_actions = 0, n_tables = 0;
9702 if (!p || !pipeline)
9705 TAILQ_FOREACH(action, &p->actions, node)
9708 TAILQ_FOREACH(table, &p->tables, node)
9711 pipeline->n_ports_in = p->n_ports_in;
9712 pipeline->n_ports_out = p->n_ports_out;
9713 pipeline->n_mirroring_slots = p->n_mirroring_slots;
9714 pipeline->n_mirroring_sessions = p->n_mirroring_sessions;
9715 pipeline->n_actions = n_actions;
9716 pipeline->n_tables = n_tables;
9717 pipeline->n_selectors = p->n_selectors;
9718 pipeline->n_learners = p->n_learners;
9719 pipeline->n_regarrays = p->n_regarrays;
9720 pipeline->n_metarrays = p->n_metarrays;
9726 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9728 if (!p || !numa_node)
9731 *numa_node = p->numa_node;
9736 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9738 struct rte_swx_ctl_action_info *action)
9740 struct action *a = NULL;
9742 if (!p || (action_id >= p->n_actions) || !action)
9745 a = action_find_by_id(p, action_id);
9749 strcpy(action->name, a->name);
9750 action->n_args = a->st ? a->st->n_fields : 0;
9755 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9757 uint32_t action_arg_id,
9758 struct rte_swx_ctl_action_arg_info *action_arg)
9760 struct action *a = NULL;
9761 struct field *arg = NULL;
9763 if (!p || (action_id >= p->n_actions) || !action_arg)
9766 a = action_find_by_id(p, action_id);
9767 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9770 arg = &a->st->fields[action_arg_id];
9771 strcpy(action_arg->name, arg->name);
9772 action_arg->n_bits = arg->n_bits;
9773 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9779 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9781 struct rte_swx_ctl_table_info *table)
9783 struct table *t = NULL;
9788 t = table_find_by_id(p, table_id);
9792 strcpy(table->name, t->name);
9793 strcpy(table->args, t->args);
9794 table->n_match_fields = t->n_fields;
9795 table->n_actions = t->n_actions;
9796 table->default_action_is_const = t->default_action_is_const;
9797 table->size = t->size;
9802 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9804 uint32_t match_field_id,
9805 struct rte_swx_ctl_table_match_field_info *match_field)
9808 struct match_field *f;
9810 if (!p || (table_id >= p->n_tables) || !match_field)
9813 t = table_find_by_id(p, table_id);
9814 if (!t || (match_field_id >= t->n_fields))
9817 f = &t->fields[match_field_id];
9818 match_field->match_type = f->match_type;
9819 match_field->is_header = t->header ? 1 : 0;
9820 match_field->n_bits = f->field->n_bits;
9821 match_field->offset = f->field->offset;
9827 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9829 uint32_t table_action_id,
9830 struct rte_swx_ctl_table_action_info *table_action)
9834 if (!p || (table_id >= p->n_tables) || !table_action)
9837 t = table_find_by_id(p, table_id);
9838 if (!t || (table_action_id >= t->n_actions))
9841 table_action->action_id = t->actions[table_action_id]->id;
9843 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9844 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9850 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9852 struct rte_swx_table_ops *table_ops,
9857 if (!p || (table_id >= p->n_tables))
9860 t = table_find_by_id(p, table_id);
9866 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9876 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9877 uint32_t selector_id,
9878 struct rte_swx_ctl_selector_info *selector)
9880 struct selector *s = NULL;
9882 if (!p || !selector)
9885 s = selector_find_by_id(p, selector_id);
9889 strcpy(selector->name, s->name);
9891 selector->n_selector_fields = s->n_selector_fields;
9892 selector->n_groups_max = s->n_groups_max;
9893 selector->n_members_per_group_max = s->n_members_per_group_max;
9899 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9900 uint32_t selector_id,
9901 struct rte_swx_ctl_table_match_field_info *field)
9905 if (!p || (selector_id >= p->n_selectors) || !field)
9908 s = selector_find_by_id(p, selector_id);
9912 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9913 field->is_header = 0;
9914 field->n_bits = s->group_id_field->n_bits;
9915 field->offset = s->group_id_field->offset;
9921 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9922 uint32_t selector_id,
9923 uint32_t selector_field_id,
9924 struct rte_swx_ctl_table_match_field_info *field)
9929 if (!p || (selector_id >= p->n_selectors) || !field)
9932 s = selector_find_by_id(p, selector_id);
9933 if (!s || (selector_field_id >= s->n_selector_fields))
9936 f = s->selector_fields[selector_field_id];
9937 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9938 field->is_header = s->selector_header ? 1 : 0;
9939 field->n_bits = f->n_bits;
9940 field->offset = f->offset;
9946 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9947 uint32_t selector_id,
9948 struct rte_swx_ctl_table_match_field_info *field)
9952 if (!p || (selector_id >= p->n_selectors) || !field)
9955 s = selector_find_by_id(p, selector_id);
9959 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9960 field->is_header = 0;
9961 field->n_bits = s->member_id_field->n_bits;
9962 field->offset = s->member_id_field->offset;
9968 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9969 uint32_t learner_id,
9970 struct rte_swx_ctl_learner_info *learner)
9972 struct learner *l = NULL;
9977 l = learner_find_by_id(p, learner_id);
9981 strcpy(learner->name, l->name);
9983 learner->n_match_fields = l->n_fields;
9984 learner->n_actions = l->n_actions;
9985 learner->default_action_is_const = l->default_action_is_const;
9986 learner->size = l->size;
9992 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9993 uint32_t learner_id,
9994 uint32_t match_field_id,
9995 struct rte_swx_ctl_table_match_field_info *match_field)
10000 if (!p || (learner_id >= p->n_learners) || !match_field)
10003 l = learner_find_by_id(p, learner_id);
10004 if (!l || (match_field_id >= l->n_fields))
10007 f = l->fields[match_field_id];
10008 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10009 match_field->is_header = l->header ? 1 : 0;
10010 match_field->n_bits = f->n_bits;
10011 match_field->offset = f->offset;
10017 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
10018 uint32_t learner_id,
10019 uint32_t learner_action_id,
10020 struct rte_swx_ctl_table_action_info *learner_action)
10024 if (!p || (learner_id >= p->n_learners) || !learner_action)
10027 l = learner_find_by_id(p, learner_id);
10028 if (!l || (learner_action_id >= l->n_actions))
10031 learner_action->action_id = l->actions[learner_action_id]->id;
10033 learner_action->action_is_for_table_entries =
10034 l->action_is_for_table_entries[learner_action_id];
10036 learner_action->action_is_for_default_entry =
10037 l->action_is_for_default_entry[learner_action_id];
10043 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10044 struct rte_swx_table_state **table_state)
10046 if (!p || !table_state || !p->build_done)
10049 *table_state = p->table_state;
10054 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10055 struct rte_swx_table_state *table_state)
10057 if (!p || !table_state || !p->build_done)
10060 p->table_state = table_state;
10065 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10067 struct rte_swx_port_in_stats *stats)
10069 struct port_in *port;
10074 port = port_in_find(p, port_id);
10078 port->type->ops.stats_read(port->obj, stats);
10083 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10085 struct rte_swx_port_out_stats *stats)
10087 struct port_out *port;
10092 port = port_out_find(p, port_id);
10096 port->type->ops.stats_read(port->obj, stats);
10101 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10102 const char *table_name,
10103 struct rte_swx_table_stats *stats)
10105 struct table *table;
10106 struct table_statistics *table_stats;
10108 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10111 table = table_find(p, table_name);
10115 table_stats = &p->table_stats[table->id];
10117 memcpy(stats->n_pkts_action,
10118 table_stats->n_pkts_action,
10119 p->n_actions * sizeof(uint64_t));
10121 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10122 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10128 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
10129 const char *selector_name,
10130 struct rte_swx_pipeline_selector_stats *stats)
10132 struct selector *s;
10134 if (!p || !selector_name || !selector_name[0] || !stats)
10137 s = selector_find(p, selector_name);
10141 stats->n_pkts = p->selector_stats[s->id].n_pkts;
10147 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
10148 const char *learner_name,
10149 struct rte_swx_learner_stats *stats)
10152 struct learner_statistics *learner_stats;
10154 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
10157 l = learner_find(p, learner_name);
10161 learner_stats = &p->learner_stats[l->id];
10163 memcpy(stats->n_pkts_action,
10164 learner_stats->n_pkts_action,
10165 p->n_actions * sizeof(uint64_t));
10167 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
10168 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
10170 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
10171 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
10173 stats->n_pkts_forget = learner_stats->n_pkts_forget;
10179 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10180 uint32_t regarray_id,
10181 struct rte_swx_ctl_regarray_info *regarray)
10183 struct regarray *r;
10185 if (!p || !regarray)
10188 r = regarray_find_by_id(p, regarray_id);
10192 strcpy(regarray->name, r->name);
10193 regarray->size = r->size;
10198 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10199 const char *regarray_name,
10200 uint32_t regarray_index,
10203 struct regarray *regarray;
10204 struct regarray_runtime *r;
10206 if (!p || !regarray_name || !value)
10209 regarray = regarray_find(p, regarray_name);
10210 if (!regarray || (regarray_index >= regarray->size))
10213 r = &p->regarray_runtime[regarray->id];
10214 *value = r->regarray[regarray_index];
10219 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10220 const char *regarray_name,
10221 uint32_t regarray_index,
10224 struct regarray *regarray;
10225 struct regarray_runtime *r;
10227 if (!p || !regarray_name)
10230 regarray = regarray_find(p, regarray_name);
10231 if (!regarray || (regarray_index >= regarray->size))
10234 r = &p->regarray_runtime[regarray->id];
10235 r->regarray[regarray_index] = value;
10240 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10241 uint32_t metarray_id,
10242 struct rte_swx_ctl_metarray_info *metarray)
10244 struct metarray *m;
10246 if (!p || !metarray)
10249 m = metarray_find_by_id(p, metarray_id);
10253 strcpy(metarray->name, m->name);
10254 metarray->size = m->size;
10259 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10261 struct rte_meter_trtcm_params *params)
10263 struct meter_profile *mp;
10267 CHECK_NAME(name, EINVAL);
10268 CHECK(params, EINVAL);
10269 CHECK(!meter_profile_find(p, name), EEXIST);
10271 /* Node allocation. */
10272 mp = calloc(1, sizeof(struct meter_profile));
10275 /* Node initialization. */
10276 strcpy(mp->name, name);
10277 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10278 status = rte_meter_trtcm_profile_config(&mp->profile, params);
10284 /* Node add to tailq. */
10285 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10291 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10294 struct meter_profile *mp;
10297 CHECK_NAME(name, EINVAL);
10299 mp = meter_profile_find(p, name);
10301 CHECK(!mp->n_users, EBUSY);
10303 /* Remove node from tailq. */
10304 TAILQ_REMOVE(&p->meter_profiles, mp, node);
10311 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10312 const char *metarray_name,
10313 uint32_t metarray_index)
10315 struct meter_profile *mp_old;
10316 struct metarray *metarray;
10317 struct metarray_runtime *metarray_runtime;
10321 CHECK_NAME(metarray_name, EINVAL);
10323 metarray = metarray_find(p, metarray_name);
10324 CHECK(metarray, EINVAL);
10325 CHECK(metarray_index < metarray->size, EINVAL);
10327 metarray_runtime = &p->metarray_runtime[metarray->id];
10328 m = &metarray_runtime->metarray[metarray_index];
10329 mp_old = m->profile;
10339 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10340 const char *metarray_name,
10341 uint32_t metarray_index,
10342 const char *profile_name)
10344 struct meter_profile *mp, *mp_old;
10345 struct metarray *metarray;
10346 struct metarray_runtime *metarray_runtime;
10350 CHECK_NAME(metarray_name, EINVAL);
10352 metarray = metarray_find(p, metarray_name);
10353 CHECK(metarray, EINVAL);
10354 CHECK(metarray_index < metarray->size, EINVAL);
10356 mp = meter_profile_find(p, profile_name);
10359 metarray_runtime = &p->metarray_runtime[metarray->id];
10360 m = &metarray_runtime->metarray[metarray_index];
10361 mp_old = m->profile;
10363 memset(m, 0, sizeof(struct meter));
10364 rte_meter_trtcm_config(&m->m, &mp->profile);
10366 m->color_mask = RTE_COLORS;
10375 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10376 const char *metarray_name,
10377 uint32_t metarray_index,
10378 struct rte_swx_ctl_meter_stats *stats)
10380 struct metarray *metarray;
10381 struct metarray_runtime *metarray_runtime;
10385 CHECK_NAME(metarray_name, EINVAL);
10387 metarray = metarray_find(p, metarray_name);
10388 CHECK(metarray, EINVAL);
10389 CHECK(metarray_index < metarray->size, EINVAL);
10391 CHECK(stats, EINVAL);
10393 metarray_runtime = &p->metarray_runtime[metarray->id];
10394 m = &metarray_runtime->metarray[metarray_index];
10396 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10397 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10403 rte_swx_ctl_pipeline_mirroring_session_set(struct rte_swx_pipeline *p,
10404 uint32_t session_id,
10405 struct rte_swx_pipeline_mirroring_session_params *params)
10407 struct mirroring_session *s;
10410 CHECK(p->build_done, EEXIST);
10411 CHECK(session_id < p->n_mirroring_sessions, EINVAL);
10412 CHECK(params, EINVAL);
10413 CHECK(params->port_id < p->n_ports_out, EINVAL);
10415 s = &p->mirroring_sessions[session_id];
10416 s->port_id = params->port_id;
10417 s->fast_clone = params->fast_clone;
10418 s->truncation_length = params->truncation_length ? params->truncation_length : UINT32_MAX;
10424 * Pipeline compilation.
10426 static const char *
10427 instr_type_to_name(struct instruction *instr)
10429 switch (instr->type) {
10430 case INSTR_RX: return "INSTR_RX";
10432 case INSTR_TX: return "INSTR_TX";
10433 case INSTR_TX_I: return "INSTR_TX_I";
10434 case INSTR_DROP: return "INSTR_DROP";
10435 case INSTR_MIRROR: return "INSTR_MIRROR";
10436 case INSTR_RECIRCULATE: return "INSTR_RECIRCULATE";
10437 case INSTR_RECIRCID: return "INSTR_RECIRCID";
10439 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10440 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10441 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10442 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10443 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10444 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10445 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10446 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10448 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10450 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10452 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10453 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10454 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10455 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10456 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10457 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10458 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10459 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10460 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10462 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10463 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10465 case INSTR_MOV: return "INSTR_MOV";
10466 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10467 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10468 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10469 case INSTR_MOV_I: return "INSTR_MOV_I";
10471 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10472 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10473 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10474 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10475 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10476 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10477 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10478 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10480 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10481 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10482 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10483 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10484 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10485 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10487 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10488 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10489 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10490 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10491 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10492 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10494 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10495 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10496 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10497 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10499 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10500 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10501 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10502 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10503 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10505 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10506 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10507 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10508 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10509 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10511 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10512 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10513 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10514 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10515 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10517 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10518 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10519 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10520 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10521 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10522 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10524 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10525 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10526 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10527 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10528 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10529 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10531 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10532 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10533 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10535 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10536 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10537 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10538 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10539 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10540 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10542 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10543 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10544 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10545 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10546 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10547 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10548 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10549 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10550 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10552 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10553 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10554 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10555 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10556 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10557 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10558 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10559 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10560 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10562 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10563 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10564 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10566 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10567 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10568 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10569 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10570 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10571 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10572 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10573 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10574 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10575 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10576 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10577 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10579 case INSTR_TABLE: return "INSTR_TABLE";
10580 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10581 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10582 case INSTR_LEARNER: return "INSTR_LEARNER";
10583 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10585 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10586 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10588 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10589 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10591 case INSTR_JMP: return "INSTR_JMP";
10592 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10593 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10594 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10595 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10596 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10597 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10598 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10599 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10600 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10601 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10602 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10603 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10604 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10605 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10606 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10607 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10608 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10609 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10610 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10611 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10612 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10613 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10614 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10615 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10616 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10617 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10618 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10619 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10621 case INSTR_RETURN: return "INSTR_RETURN";
10623 default: return "INSTR_UNKNOWN";
10628 (*instruction_export_t)(struct instruction *, FILE *);
10631 instr_io_export(struct instruction *instr, FILE *f)
10633 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10635 /* n_io, n_io_imm, n_hdrs. */
10636 if (instr->type == INSTR_RX ||
10637 instr->type == INSTR_TX ||
10638 instr->type == INSTR_HDR_EXTRACT_M ||
10639 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10642 if (instr->type == INSTR_TX_I)
10645 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10646 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10648 if (instr->type == INSTR_HDR_EXTRACT_M ||
10649 instr->type == INSTR_HDR_LOOKAHEAD ||
10650 instr->type == INSTR_HDR_EMIT)
10653 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10654 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10659 "\t\t.type = %s,\n",
10660 instr_type_to_name(instr));
10663 if (n_io || n_io_imm || n_hdrs)
10671 "\t\t\t\t.offset = %u,\n"
10672 "\t\t\t\t.n_bits = %u,\n"
10674 instr->io.io.offset,
10675 instr->io.io.n_bits);
10680 "\t\t\t\t.val = %u,\n"
10684 /* instr.io.hdr. */
10689 /* instr.io.hdr.header_id. */
10691 "\t\t\t.header_id = {");
10693 for (i = 0; i < n_hdrs; i++)
10696 instr->io.hdr.header_id[i]);
10701 /* instr.io.hdr.struct_id. */
10703 "\t\t\t.struct_id = {");
10705 for (i = 0; i < n_hdrs; i++)
10708 instr->io.hdr.struct_id[i]);
10713 /* instr.io.hdr.n_bytes. */
10715 "\t\t\t.n_bytes = {");
10717 for (i = 0; i < n_hdrs; i++)
10720 instr->io.hdr.n_bytes[i]);
10725 /* instr.io.hdr - closing curly brace. */
10730 /* instr.io - closing curly brace. */
10731 if (n_io || n_io_imm || n_hdrs)
10735 /* instr - closing curly brace. */
10741 instr_mirror_export(struct instruction *instr, FILE *f)
10745 "\t\t.type = %s,\n"
10746 "\t\t.mirror = {\n"
10748 "\t\t\t\t.struct_id = %u,\n"
10749 "\t\t\t\t.n_bits = %u,\n"
10750 "\t\t\t\t.offset = %u,\n"
10753 "\t\t\t\t.struct_id = %u,\n"
10754 "\t\t\t\t.n_bits = %u,\n"
10755 "\t\t\t\t.offset = %u,\n"
10759 instr_type_to_name(instr),
10760 instr->mirror.dst.struct_id,
10761 instr->mirror.dst.n_bits,
10762 instr->mirror.dst.offset,
10763 instr->mirror.src.struct_id,
10764 instr->mirror.src.n_bits,
10765 instr->mirror.src.offset);
10769 instr_recirculate_export(struct instruction *instr, FILE *f)
10773 "\t\t.type = %s,\n"
10775 instr_type_to_name(instr));
10779 instr_recircid_export(struct instruction *instr, FILE *f)
10783 "\t\t.type = %s,\n"
10785 "\t\t\t.offset = %u,\n"
10786 "\t\t\t.n_bits = %u,\n"
10789 instr_type_to_name(instr),
10790 instr->io.io.offset,
10791 instr->io.io.n_bits);
10795 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10799 "\t\t.type = %s,\n"
10801 "\t\t\t.header_id = %u,\n"
10804 instr_type_to_name(instr),
10805 instr->valid.header_id);
10809 instr_mov_export(struct instruction *instr, FILE *f)
10811 if (instr->type != INSTR_MOV_I)
10814 "\t\t.type = %s,\n"
10817 "\t\t\t\t.struct_id = %u,\n"
10818 "\t\t\t\t.n_bits = %u,\n"
10819 "\t\t\t\t.offset = %u,\n"
10822 "\t\t\t\t.struct_id = %u,\n"
10823 "\t\t\t\t.n_bits = %u,\n"
10824 "\t\t\t\t.offset = %u,\n"
10828 instr_type_to_name(instr),
10829 instr->mov.dst.struct_id,
10830 instr->mov.dst.n_bits,
10831 instr->mov.dst.offset,
10832 instr->mov.src.struct_id,
10833 instr->mov.src.n_bits,
10834 instr->mov.src.offset);
10838 "\t\t.type = %s,\n"
10841 "\t\t\t\t.struct_id = %u,\n"
10842 "\t\t\t\t.n_bits = %u,\n"
10843 "\t\t\t\t.offset = %u,\n"
10845 "\t\t\t.src_val = %" PRIu64 ",\n"
10848 instr_type_to_name(instr),
10849 instr->mov.dst.struct_id,
10850 instr->mov.dst.n_bits,
10851 instr->mov.dst.offset,
10852 instr->mov.src_val);
10856 instr_dma_ht_export(struct instruction *instr, FILE *f)
10858 uint32_t n_dma = 0, i;
10861 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10866 "\t\t.type = %s,\n",
10867 instr_type_to_name(instr));
10873 /* instr.dma.dst. */
10875 "\t\t\t.dst = {\n");
10877 /* instr.dma.dst.header_id. */
10879 "\t\t\t\t.header_id = {");
10881 for (i = 0; i < n_dma; i++)
10884 instr->dma.dst.header_id[i]);
10889 /* instr.dma.dst.struct_id. */
10891 "\t\t\t\t.struct_id = {");
10893 for (i = 0; i < n_dma; i++)
10896 instr->dma.dst.struct_id[i]);
10901 /* instr.dma.dst - closing curly brace. */
10905 /* instr.dma.src. */
10907 "\t\t\t.src = {\n");
10909 /* instr.dma.src.offset. */
10911 "\t\t\t\t.offset = {");
10913 for (i = 0; i < n_dma; i++)
10916 instr->dma.src.offset[i]);
10921 /* instr.dma.src - closing curly brace. */
10925 /* instr.dma.n_bytes. */
10927 "\t\t\t.n_bytes = {");
10929 for (i = 0; i < n_dma; i++)
10932 instr->dma.n_bytes[i]);
10937 /* instr.dma - closing curly brace. */
10941 /* instr - closing curly brace. */
10947 instr_alu_export(struct instruction *instr, FILE *f)
10951 if (instr->type == INSTR_ALU_ADD_MI ||
10952 instr->type == INSTR_ALU_ADD_HI ||
10953 instr->type == INSTR_ALU_SUB_MI ||
10954 instr->type == INSTR_ALU_SUB_HI ||
10955 instr->type == INSTR_ALU_SHL_MI ||
10956 instr->type == INSTR_ALU_SHL_HI ||
10957 instr->type == INSTR_ALU_SHR_MI ||
10958 instr->type == INSTR_ALU_SHR_HI ||
10959 instr->type == INSTR_ALU_AND_I ||
10960 instr->type == INSTR_ALU_OR_I ||
10961 instr->type == INSTR_ALU_XOR_I)
10967 "\t\t.type = %s,\n"
10970 "\t\t\t\t.struct_id = %u,\n"
10971 "\t\t\t\t.n_bits = %u,\n"
10972 "\t\t\t\t.offset = %u,\n"
10975 "\t\t\t\t.struct_id = %u,\n"
10976 "\t\t\t\t.n_bits = %u,\n"
10977 "\t\t\t\t.offset = %u,\n"
10981 instr_type_to_name(instr),
10982 instr->alu.dst.struct_id,
10983 instr->alu.dst.n_bits,
10984 instr->alu.dst.offset,
10985 instr->alu.src.struct_id,
10986 instr->alu.src.n_bits,
10987 instr->alu.src.offset);
10991 "\t\t.type = %s,\n"
10994 "\t\t\t\t.struct_id = %u,\n"
10995 "\t\t\t\t.n_bits = %u,\n"
10996 "\t\t\t\t.offset = %u,\n"
10998 "\t\t\t.src_val = %" PRIu64 ",\n"
11001 instr_type_to_name(instr),
11002 instr->alu.dst.struct_id,
11003 instr->alu.dst.n_bits,
11004 instr->alu.dst.offset,
11005 instr->alu.src_val);
11009 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11011 int prefetch = 0, idx_imm = 0, src_imm = 0;
11013 if (instr->type == INSTR_REGPREFETCH_RH ||
11014 instr->type == INSTR_REGPREFETCH_RM ||
11015 instr->type == INSTR_REGPREFETCH_RI)
11018 /* index is the 3rd operand for the regrd instruction and the 2nd
11019 * operand for the regwr and regadd instructions.
11021 if (instr->type == INSTR_REGPREFETCH_RI ||
11022 instr->type == INSTR_REGRD_HRI ||
11023 instr->type == INSTR_REGRD_MRI ||
11024 instr->type == INSTR_REGWR_RIH ||
11025 instr->type == INSTR_REGWR_RIM ||
11026 instr->type == INSTR_REGWR_RII ||
11027 instr->type == INSTR_REGADD_RIH ||
11028 instr->type == INSTR_REGADD_RIM ||
11029 instr->type == INSTR_REGADD_RII)
11032 /* src is the 3rd operand for the regwr and regadd instructions. */
11033 if (instr->type == INSTR_REGWR_RHI ||
11034 instr->type == INSTR_REGWR_RMI ||
11035 instr->type == INSTR_REGWR_RII ||
11036 instr->type == INSTR_REGADD_RHI ||
11037 instr->type == INSTR_REGADD_RMI ||
11038 instr->type == INSTR_REGADD_RII)
11041 /* instr.regarray.regarray_id. */
11044 "\t\t.type = %s,\n"
11045 "\t\t.regarray = {\n"
11046 "\t\t\t.regarray_id = %u,\n",
11047 instr_type_to_name(instr),
11048 instr->regarray.regarray_id);
11050 /* instr.regarray.idx / instr.regarray.idx_val. */
11053 "\t\t\t\t.idx = {\n"
11054 "\t\t\t\t\t.struct_id = %u,\n"
11055 "\t\t\t\t\t.n_bits = %u,\n"
11056 "\t\t\t\t\t.offset = %u,\n"
11058 instr->regarray.idx.struct_id,
11059 instr->regarray.idx.n_bits,
11060 instr->regarray.idx.offset);
11063 "\t\t\t\t.idx_val = %u,\n",
11064 instr->regarray.idx_val);
11066 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
11070 "\t\t\t\t.dstsrc = {\n"
11071 "\t\t\t\t\t.struct_id = %u,\n"
11072 "\t\t\t\t\t.n_bits = %u,\n"
11073 "\t\t\t\t\t.offset = %u,\n"
11075 instr->regarray.dstsrc.struct_id,
11076 instr->regarray.dstsrc.n_bits,
11077 instr->regarray.dstsrc.offset);
11080 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
11081 instr->regarray.dstsrc_val);
11084 /* instr.regarray and instr - closing curly braces. */
11091 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
11093 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
11095 if (instr->type == INSTR_METPREFETCH_H ||
11096 instr->type == INSTR_METPREFETCH_M ||
11097 instr->type == INSTR_METPREFETCH_I)
11101 if (instr->type == INSTR_METPREFETCH_I ||
11102 instr->type == INSTR_METER_IHM ||
11103 instr->type == INSTR_METER_IHI ||
11104 instr->type == INSTR_METER_IMM ||
11105 instr->type == INSTR_METER_IMI)
11108 /* color_in_imm. */
11109 if (instr->type == INSTR_METER_HHI ||
11110 instr->type == INSTR_METER_HMI ||
11111 instr->type == INSTR_METER_MHI ||
11112 instr->type == INSTR_METER_MMI ||
11113 instr->type == INSTR_METER_IHI ||
11114 instr->type == INSTR_METER_IMI)
11117 /* instr.meter.metarray_id. */
11120 "\t\t.type = %s,\n"
11122 "\t\t\t.metarray_id = %u,\n",
11123 instr_type_to_name(instr),
11124 instr->meter.metarray_id);
11126 /* instr.meter.idx / instr.meter.idx_val. */
11130 "\t\t\t\t.struct_id = %u,\n"
11131 "\t\t\t\t.n_bits = %u,\n"
11132 "\t\t\t\t.offset = %u,\n"
11134 instr->meter.idx.struct_id,
11135 instr->meter.idx.n_bits,
11136 instr->meter.idx.offset);
11139 "\t\t\t.idx_val = %u,\n",
11140 instr->meter.idx_val);
11143 /* instr.meter.length. */
11145 "\t\t\t.length = {\n"
11146 "\t\t\t\t.struct_id = %u,\n"
11147 "\t\t\t\t.n_bits = %u,\n"
11148 "\t\t\t\t.offset = %u,\n"
11150 instr->meter.length.struct_id,
11151 instr->meter.length.n_bits,
11152 instr->meter.length.offset);
11154 /* instr.meter.color_in / instr.meter.color_in_val. */
11157 "\t\t\t.color_in = {\n"
11158 "\t\t\t\t.struct_id = %u,\n"
11159 "\t\t\t\t.n_bits = %u,\n"
11160 "\t\t\t\t.offset = %u,\n"
11162 instr->meter.color_in.struct_id,
11163 instr->meter.color_in.n_bits,
11164 instr->meter.color_in.offset);
11167 "\t\t\t.color_in_val = %u,\n",
11168 (uint32_t)instr->meter.color_in_val);
11170 /* instr.meter.color_out. */
11172 "\t\t\t.color_out = {\n"
11173 "\t\t\t\t.struct_id = %u,\n"
11174 "\t\t\t\t.n_bits = %u,\n"
11175 "\t\t\t\t.offset = %u,\n"
11177 instr->meter.color_out.struct_id,
11178 instr->meter.color_out.n_bits,
11179 instr->meter.color_out.offset);
11182 /* instr.meter and instr - closing curly braces. */
11189 instr_table_export(struct instruction *instr,
11194 "\t\t.type = %s,\n"
11196 "\t\t\t.table_id = %u,\n"
11199 instr_type_to_name(instr),
11200 instr->table.table_id);
11204 instr_learn_export(struct instruction *instr, FILE *f)
11208 "\t\t.type = %s,\n"
11210 "\t\t\t\t.action_id = %u,\n"
11213 instr_type_to_name(instr),
11214 instr->learn.action_id);
11218 instr_forget_export(struct instruction *instr, FILE *f)
11222 "\t\t.type = %s,\n"
11224 instr_type_to_name(instr));
11228 instr_extern_export(struct instruction *instr, FILE *f)
11230 if (instr->type == INSTR_EXTERN_OBJ)
11233 "\t\t.type = %s,\n"
11234 "\t\t.ext_obj = {\n"
11235 "\t\t\t.ext_obj_id = %u,\n"
11236 "\t\t\t.func_id = %u,\n"
11239 instr_type_to_name(instr),
11240 instr->ext_obj.ext_obj_id,
11241 instr->ext_obj.func_id);
11245 "\t\t.type = %s,\n"
11246 "\t\t.ext_func = {\n"
11247 "\t\t\t.ext_func_id = %u,\n"
11250 instr_type_to_name(instr),
11251 instr->ext_func.ext_func_id);
11255 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
11259 "\t\t.type = %s,\n"
11261 "\t\t\t.ip = NULL,\n",
11262 instr_type_to_name(instr));
11264 switch (instr->type) {
11265 case INSTR_JMP_VALID:
11266 case INSTR_JMP_INVALID:
11268 "\t\t\t.header_id = %u,\n",
11269 instr->jmp.header_id);
11272 case INSTR_JMP_ACTION_HIT:
11273 case INSTR_JMP_ACTION_MISS:
11275 "\t\t\t.action_id = %u,\n",
11276 instr->jmp.action_id);
11280 case INSTR_JMP_EQ_MH:
11281 case INSTR_JMP_EQ_HM:
11282 case INSTR_JMP_EQ_HH:
11283 case INSTR_JMP_NEQ:
11284 case INSTR_JMP_NEQ_MH:
11285 case INSTR_JMP_NEQ_HM:
11286 case INSTR_JMP_NEQ_HH:
11288 case INSTR_JMP_LT_MH:
11289 case INSTR_JMP_LT_HM:
11290 case INSTR_JMP_LT_HH:
11292 case INSTR_JMP_GT_MH:
11293 case INSTR_JMP_GT_HM:
11294 case INSTR_JMP_GT_HH:
11297 "\t\t\t\t.struct_id = %u,\n"
11298 "\t\t\t\t.n_bits = %u,\n"
11299 "\t\t\t\t.offset = %u,\n"
11302 "\t\t\t\t.struct_id = %u,\n"
11303 "\t\t\t\t.n_bits = %u,\n"
11304 "\t\t\t\t.offset = %u,\n"
11306 instr->jmp.a.struct_id,
11307 instr->jmp.a.n_bits,
11308 instr->jmp.a.offset,
11309 instr->jmp.b.struct_id,
11310 instr->jmp.b.n_bits,
11311 instr->jmp.b.offset);
11314 case INSTR_JMP_EQ_I:
11315 case INSTR_JMP_NEQ_I:
11316 case INSTR_JMP_LT_MI:
11317 case INSTR_JMP_LT_HI:
11318 case INSTR_JMP_GT_MI:
11319 case INSTR_JMP_GT_HI:
11322 "\t\t\t\t.struct_id = %u,\n"
11323 "\t\t\t\t.n_bits = %u,\n"
11324 "\t\t\t\t.offset = %u,\n"
11326 "\t\t\t.b_val = %" PRIu64 ",\n",
11327 instr->jmp.a.struct_id,
11328 instr->jmp.a.n_bits,
11329 instr->jmp.a.offset,
11343 instr_return_export(struct instruction *instr,
11348 "\t\t.type = %s,\n",
11349 instr_type_to_name(instr));
11355 static instruction_export_t export_table[] = {
11356 [INSTR_RX] = instr_io_export,
11358 [INSTR_TX] = instr_io_export,
11359 [INSTR_TX_I] = instr_io_export,
11360 [INSTR_DROP] = instr_io_export,
11361 [INSTR_MIRROR] = instr_mirror_export,
11362 [INSTR_RECIRCULATE] = instr_recirculate_export,
11363 [INSTR_RECIRCID] = instr_recircid_export,
11365 [INSTR_HDR_EXTRACT] = instr_io_export,
11366 [INSTR_HDR_EXTRACT2] = instr_io_export,
11367 [INSTR_HDR_EXTRACT3] = instr_io_export,
11368 [INSTR_HDR_EXTRACT4] = instr_io_export,
11369 [INSTR_HDR_EXTRACT5] = instr_io_export,
11370 [INSTR_HDR_EXTRACT6] = instr_io_export,
11371 [INSTR_HDR_EXTRACT7] = instr_io_export,
11372 [INSTR_HDR_EXTRACT8] = instr_io_export,
11374 [INSTR_HDR_EXTRACT_M] = instr_io_export,
11376 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
11378 [INSTR_HDR_EMIT] = instr_io_export,
11379 [INSTR_HDR_EMIT_TX] = instr_io_export,
11380 [INSTR_HDR_EMIT2_TX] = instr_io_export,
11381 [INSTR_HDR_EMIT3_TX] = instr_io_export,
11382 [INSTR_HDR_EMIT4_TX] = instr_io_export,
11383 [INSTR_HDR_EMIT5_TX] = instr_io_export,
11384 [INSTR_HDR_EMIT6_TX] = instr_io_export,
11385 [INSTR_HDR_EMIT7_TX] = instr_io_export,
11386 [INSTR_HDR_EMIT8_TX] = instr_io_export,
11388 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
11389 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
11391 [INSTR_MOV] = instr_mov_export,
11392 [INSTR_MOV_MH] = instr_mov_export,
11393 [INSTR_MOV_HM] = instr_mov_export,
11394 [INSTR_MOV_HH] = instr_mov_export,
11395 [INSTR_MOV_I] = instr_mov_export,
11397 [INSTR_DMA_HT] = instr_dma_ht_export,
11398 [INSTR_DMA_HT2] = instr_dma_ht_export,
11399 [INSTR_DMA_HT3] = instr_dma_ht_export,
11400 [INSTR_DMA_HT4] = instr_dma_ht_export,
11401 [INSTR_DMA_HT5] = instr_dma_ht_export,
11402 [INSTR_DMA_HT6] = instr_dma_ht_export,
11403 [INSTR_DMA_HT7] = instr_dma_ht_export,
11404 [INSTR_DMA_HT8] = instr_dma_ht_export,
11406 [INSTR_ALU_ADD] = instr_alu_export,
11407 [INSTR_ALU_ADD_MH] = instr_alu_export,
11408 [INSTR_ALU_ADD_HM] = instr_alu_export,
11409 [INSTR_ALU_ADD_HH] = instr_alu_export,
11410 [INSTR_ALU_ADD_MI] = instr_alu_export,
11411 [INSTR_ALU_ADD_HI] = instr_alu_export,
11413 [INSTR_ALU_SUB] = instr_alu_export,
11414 [INSTR_ALU_SUB_MH] = instr_alu_export,
11415 [INSTR_ALU_SUB_HM] = instr_alu_export,
11416 [INSTR_ALU_SUB_HH] = instr_alu_export,
11417 [INSTR_ALU_SUB_MI] = instr_alu_export,
11418 [INSTR_ALU_SUB_HI] = instr_alu_export,
11420 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
11421 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
11422 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
11423 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
11425 [INSTR_ALU_AND] = instr_alu_export,
11426 [INSTR_ALU_AND_MH] = instr_alu_export,
11427 [INSTR_ALU_AND_HM] = instr_alu_export,
11428 [INSTR_ALU_AND_HH] = instr_alu_export,
11429 [INSTR_ALU_AND_I] = instr_alu_export,
11431 [INSTR_ALU_OR] = instr_alu_export,
11432 [INSTR_ALU_OR_MH] = instr_alu_export,
11433 [INSTR_ALU_OR_HM] = instr_alu_export,
11434 [INSTR_ALU_OR_HH] = instr_alu_export,
11435 [INSTR_ALU_OR_I] = instr_alu_export,
11437 [INSTR_ALU_XOR] = instr_alu_export,
11438 [INSTR_ALU_XOR_MH] = instr_alu_export,
11439 [INSTR_ALU_XOR_HM] = instr_alu_export,
11440 [INSTR_ALU_XOR_HH] = instr_alu_export,
11441 [INSTR_ALU_XOR_I] = instr_alu_export,
11443 [INSTR_ALU_SHL] = instr_alu_export,
11444 [INSTR_ALU_SHL_MH] = instr_alu_export,
11445 [INSTR_ALU_SHL_HM] = instr_alu_export,
11446 [INSTR_ALU_SHL_HH] = instr_alu_export,
11447 [INSTR_ALU_SHL_MI] = instr_alu_export,
11448 [INSTR_ALU_SHL_HI] = instr_alu_export,
11450 [INSTR_ALU_SHR] = instr_alu_export,
11451 [INSTR_ALU_SHR_MH] = instr_alu_export,
11452 [INSTR_ALU_SHR_HM] = instr_alu_export,
11453 [INSTR_ALU_SHR_HH] = instr_alu_export,
11454 [INSTR_ALU_SHR_MI] = instr_alu_export,
11455 [INSTR_ALU_SHR_HI] = instr_alu_export,
11457 [INSTR_REGPREFETCH_RH] = instr_reg_export,
11458 [INSTR_REGPREFETCH_RM] = instr_reg_export,
11459 [INSTR_REGPREFETCH_RI] = instr_reg_export,
11461 [INSTR_REGRD_HRH] = instr_reg_export,
11462 [INSTR_REGRD_HRM] = instr_reg_export,
11463 [INSTR_REGRD_MRH] = instr_reg_export,
11464 [INSTR_REGRD_MRM] = instr_reg_export,
11465 [INSTR_REGRD_HRI] = instr_reg_export,
11466 [INSTR_REGRD_MRI] = instr_reg_export,
11468 [INSTR_REGWR_RHH] = instr_reg_export,
11469 [INSTR_REGWR_RHM] = instr_reg_export,
11470 [INSTR_REGWR_RMH] = instr_reg_export,
11471 [INSTR_REGWR_RMM] = instr_reg_export,
11472 [INSTR_REGWR_RHI] = instr_reg_export,
11473 [INSTR_REGWR_RMI] = instr_reg_export,
11474 [INSTR_REGWR_RIH] = instr_reg_export,
11475 [INSTR_REGWR_RIM] = instr_reg_export,
11476 [INSTR_REGWR_RII] = instr_reg_export,
11478 [INSTR_REGADD_RHH] = instr_reg_export,
11479 [INSTR_REGADD_RHM] = instr_reg_export,
11480 [INSTR_REGADD_RMH] = instr_reg_export,
11481 [INSTR_REGADD_RMM] = instr_reg_export,
11482 [INSTR_REGADD_RHI] = instr_reg_export,
11483 [INSTR_REGADD_RMI] = instr_reg_export,
11484 [INSTR_REGADD_RIH] = instr_reg_export,
11485 [INSTR_REGADD_RIM] = instr_reg_export,
11486 [INSTR_REGADD_RII] = instr_reg_export,
11488 [INSTR_METPREFETCH_H] = instr_meter_export,
11489 [INSTR_METPREFETCH_M] = instr_meter_export,
11490 [INSTR_METPREFETCH_I] = instr_meter_export,
11492 [INSTR_METER_HHM] = instr_meter_export,
11493 [INSTR_METER_HHI] = instr_meter_export,
11494 [INSTR_METER_HMM] = instr_meter_export,
11495 [INSTR_METER_HMI] = instr_meter_export,
11496 [INSTR_METER_MHM] = instr_meter_export,
11497 [INSTR_METER_MHI] = instr_meter_export,
11498 [INSTR_METER_MMM] = instr_meter_export,
11499 [INSTR_METER_MMI] = instr_meter_export,
11500 [INSTR_METER_IHM] = instr_meter_export,
11501 [INSTR_METER_IHI] = instr_meter_export,
11502 [INSTR_METER_IMM] = instr_meter_export,
11503 [INSTR_METER_IMI] = instr_meter_export,
11505 [INSTR_TABLE] = instr_table_export,
11506 [INSTR_TABLE_AF] = instr_table_export,
11507 [INSTR_SELECTOR] = instr_table_export,
11508 [INSTR_LEARNER] = instr_table_export,
11509 [INSTR_LEARNER_AF] = instr_table_export,
11511 [INSTR_LEARNER_LEARN] = instr_learn_export,
11512 [INSTR_LEARNER_FORGET] = instr_forget_export,
11514 [INSTR_EXTERN_OBJ] = instr_extern_export,
11515 [INSTR_EXTERN_FUNC] = instr_extern_export,
11517 [INSTR_JMP] = instr_jmp_export,
11518 [INSTR_JMP_VALID] = instr_jmp_export,
11519 [INSTR_JMP_INVALID] = instr_jmp_export,
11520 [INSTR_JMP_HIT] = instr_jmp_export,
11521 [INSTR_JMP_MISS] = instr_jmp_export,
11522 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11523 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11525 [INSTR_JMP_EQ] = instr_jmp_export,
11526 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11527 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11528 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11529 [INSTR_JMP_EQ_I] = instr_jmp_export,
11531 [INSTR_JMP_NEQ] = instr_jmp_export,
11532 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11533 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11534 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11535 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11537 [INSTR_JMP_LT] = instr_jmp_export,
11538 [INSTR_JMP_LT_MH] = instr_jmp_export,
11539 [INSTR_JMP_LT_HM] = instr_jmp_export,
11540 [INSTR_JMP_LT_HH] = instr_jmp_export,
11541 [INSTR_JMP_LT_MI] = instr_jmp_export,
11542 [INSTR_JMP_LT_HI] = instr_jmp_export,
11544 [INSTR_JMP_GT] = instr_jmp_export,
11545 [INSTR_JMP_GT_MH] = instr_jmp_export,
11546 [INSTR_JMP_GT_HM] = instr_jmp_export,
11547 [INSTR_JMP_GT_HH] = instr_jmp_export,
11548 [INSTR_JMP_GT_MI] = instr_jmp_export,
11549 [INSTR_JMP_GT_HI] = instr_jmp_export,
11551 [INSTR_RETURN] = instr_return_export,
11555 action_data_codegen(struct action *a, FILE *f)
11560 "static const struct instruction action_%s_instructions[] = {\n",
11563 for (i = 0; i < a->n_instructions; i++) {
11564 struct instruction *instr = &a->instructions[i];
11565 instruction_export_t func = export_table[instr->type];
11570 fprintf(f, "};\n");
11573 static const char *
11574 instr_type_to_func(struct instruction *instr)
11576 switch (instr->type) {
11577 case INSTR_RX: return NULL;
11579 case INSTR_TX: return "__instr_tx_exec";
11580 case INSTR_TX_I: return "__instr_tx_i_exec";
11581 case INSTR_DROP: return "__instr_drop_exec";
11582 case INSTR_MIRROR: return "__instr_mirror_exec";
11583 case INSTR_RECIRCULATE: return "__instr_recirculate_exec";
11584 case INSTR_RECIRCID: return "__instr_recircid_exec";
11586 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11587 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11588 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11589 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11590 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11591 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11592 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11593 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11595 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11597 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11599 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11600 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11601 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11602 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11603 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11604 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11605 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11606 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11607 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11609 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11610 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11612 case INSTR_MOV: return "__instr_mov_exec";
11613 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11614 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11615 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11616 case INSTR_MOV_I: return "__instr_mov_i_exec";
11618 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11619 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11620 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11621 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11622 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11623 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11624 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11625 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11627 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11628 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11629 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11630 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11631 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11632 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11634 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11635 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11636 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11637 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11638 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11639 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11641 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11642 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11643 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11644 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11646 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11647 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11648 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11649 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11650 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11652 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11653 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11654 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11655 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11656 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11658 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11659 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11660 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11661 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11662 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11664 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11665 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11666 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11667 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11668 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11669 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11671 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11672 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11673 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11674 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11675 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11676 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11678 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11679 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11680 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11682 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11683 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11684 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11685 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11686 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11687 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11689 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11690 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11691 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11692 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11693 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11694 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11695 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11696 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11697 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11699 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11700 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11701 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11702 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11703 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11704 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11705 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11706 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11707 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11709 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11710 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11711 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11713 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11714 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11715 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11716 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11717 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11718 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11719 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11720 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11721 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11722 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11723 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11724 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11726 case INSTR_TABLE: return NULL;
11727 case INSTR_TABLE_AF: return NULL;
11728 case INSTR_SELECTOR: return NULL;
11729 case INSTR_LEARNER: return NULL;
11730 case INSTR_LEARNER_AF: return NULL;
11732 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11733 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11735 case INSTR_EXTERN_OBJ: return NULL;
11736 case INSTR_EXTERN_FUNC: return NULL;
11738 case INSTR_JMP: return NULL;
11739 case INSTR_JMP_VALID: return NULL;
11740 case INSTR_JMP_INVALID: return NULL;
11741 case INSTR_JMP_HIT: return NULL;
11742 case INSTR_JMP_MISS: return NULL;
11743 case INSTR_JMP_ACTION_HIT: return NULL;
11744 case INSTR_JMP_ACTION_MISS: return NULL;
11745 case INSTR_JMP_EQ: return NULL;
11746 case INSTR_JMP_EQ_MH: return NULL;
11747 case INSTR_JMP_EQ_HM: return NULL;
11748 case INSTR_JMP_EQ_HH: return NULL;
11749 case INSTR_JMP_EQ_I: return NULL;
11750 case INSTR_JMP_NEQ: return NULL;
11751 case INSTR_JMP_NEQ_MH: return NULL;
11752 case INSTR_JMP_NEQ_HM: return NULL;
11753 case INSTR_JMP_NEQ_HH: return NULL;
11754 case INSTR_JMP_NEQ_I: return NULL;
11755 case INSTR_JMP_LT: return NULL;
11756 case INSTR_JMP_LT_MH: return NULL;
11757 case INSTR_JMP_LT_HM: return NULL;
11758 case INSTR_JMP_LT_HH: return NULL;
11759 case INSTR_JMP_LT_MI: return NULL;
11760 case INSTR_JMP_LT_HI: return NULL;
11761 case INSTR_JMP_GT: return NULL;
11762 case INSTR_JMP_GT_MH: return NULL;
11763 case INSTR_JMP_GT_HM: return NULL;
11764 case INSTR_JMP_GT_HH: return NULL;
11765 case INSTR_JMP_GT_MI: return NULL;
11766 case INSTR_JMP_GT_HI: return NULL;
11768 case INSTR_RETURN: return NULL;
11770 default: return NULL;
11775 action_instr_does_tx_codegen(struct action *a,
11776 uint32_t instr_pos,
11777 struct instruction *instr,
11781 "%s(p, t, &action_%s_instructions[%u]);\n"
11782 "\tthread_ip_reset(p, t);\n"
11783 "\tinstr_rx_exec(p);\n"
11785 instr_type_to_func(instr),
11791 action_instr_extern_obj_codegen(struct action *a,
11792 uint32_t instr_pos,
11796 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11802 action_instr_extern_func_codegen(struct action *a,
11803 uint32_t instr_pos,
11807 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11813 action_instr_jmp_codegen(struct action *a,
11814 uint32_t instr_pos,
11815 struct instruction *instr,
11816 struct instruction_data *data,
11819 switch (instr->type) {
11826 case INSTR_JMP_VALID:
11828 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11835 case INSTR_JMP_INVALID:
11837 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11844 case INSTR_JMP_HIT:
11851 case INSTR_JMP_MISS:
11858 case INSTR_JMP_ACTION_HIT:
11860 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11867 case INSTR_JMP_ACTION_MISS:
11869 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11878 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11879 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11888 case INSTR_JMP_EQ_MH:
11890 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11891 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11900 case INSTR_JMP_EQ_HM:
11902 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11903 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11912 case INSTR_JMP_EQ_HH:
11914 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11915 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11924 case INSTR_JMP_EQ_I:
11926 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11927 "action_%s_instructions[%u].jmp.b_val)\n"
11936 case INSTR_JMP_NEQ:
11938 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11939 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11948 case INSTR_JMP_NEQ_MH:
11950 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11951 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11960 case INSTR_JMP_NEQ_HM:
11962 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11963 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11972 case INSTR_JMP_NEQ_HH:
11974 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11975 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11984 case INSTR_JMP_NEQ_I:
11986 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11987 "action_%s_instructions[%u].jmp.b_val)\n"
11998 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11999 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12008 case INSTR_JMP_LT_MH:
12010 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12011 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12020 case INSTR_JMP_LT_HM:
12022 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12023 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12032 case INSTR_JMP_LT_HH:
12034 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12035 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12044 case INSTR_JMP_LT_MI:
12046 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
12047 "action_%s_instructions[%u].jmp.b_val)\n"
12056 case INSTR_JMP_LT_HI:
12058 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
12059 "action_%s_instructions[%u].jmp.b_val)\n"
12070 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12071 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12080 case INSTR_JMP_GT_MH:
12082 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12083 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12092 case INSTR_JMP_GT_HM:
12094 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12095 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
12104 case INSTR_JMP_GT_HH:
12106 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12107 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
12116 case INSTR_JMP_GT_MI:
12118 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
12119 "action_%s_instructions[%u].jmp.b_val)\n"
12128 case INSTR_JMP_GT_HI:
12130 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
12131 "action_%s_instructions[%u].jmp.b_val)\n"
12146 action_instr_return_codegen(FILE *f)
12153 action_instr_codegen(struct action *a, FILE *f)
12159 "action_%s_run(struct rte_swx_pipeline *p)\n"
12161 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12165 for (i = 0; i < a->n_instructions; i++) {
12166 struct instruction *instr = &a->instructions[i];
12167 struct instruction_data *data = &a->instruction_data[i];
12169 /* Label, if present. */
12170 if (data->label[0])
12171 fprintf(f, "\n%s : ", data->label);
12173 fprintf(f, "\n\t");
12175 /* TX instruction type. */
12176 if (instruction_does_tx(instr)) {
12177 action_instr_does_tx_codegen(a, i, instr, f);
12181 /* Extern object/function instruction type. */
12182 if (instr->type == INSTR_EXTERN_OBJ) {
12183 action_instr_extern_obj_codegen(a, i, f);
12187 if (instr->type == INSTR_EXTERN_FUNC) {
12188 action_instr_extern_func_codegen(a, i, f);
12192 /* Jump instruction type. */
12193 if (instruction_is_jmp(instr)) {
12194 action_instr_jmp_codegen(a, i, instr, data, f);
12198 /* Return instruction type. */
12199 if (instr->type == INSTR_RETURN) {
12200 action_instr_return_codegen(f);
12204 /* Any other instruction type. */
12206 "%s(p, t, &action_%s_instructions[%u]);\n",
12207 instr_type_to_func(instr),
12212 fprintf(f, "}\n\n");
12215 struct instruction_group {
12216 TAILQ_ENTRY(instruction_group) node;
12220 uint32_t first_instr_id;
12222 uint32_t last_instr_id;
12227 TAILQ_HEAD(instruction_group_list, instruction_group);
12229 static struct instruction_group *
12230 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
12232 struct instruction_group *g;
12234 TAILQ_FOREACH(g, igl, node)
12235 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
12242 instruction_group_list_free(struct instruction_group_list *igl)
12248 struct instruction_group *g;
12250 g = TAILQ_FIRST(igl);
12254 TAILQ_REMOVE(igl, g, node);
12261 static struct instruction_group_list *
12262 instruction_group_list_create(struct rte_swx_pipeline *p)
12264 struct instruction_group_list *igl = NULL;
12265 struct instruction_group *g = NULL;
12266 uint32_t n_groups = 0, i;
12268 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
12272 igl = calloc(1, sizeof(struct instruction_group_list));
12278 /* Allocate the first group. */
12279 g = calloc(1, sizeof(struct instruction_group));
12283 /* Iteration 1: Separate the instructions into groups based on the thread yield
12284 * instructions. Do not worry about the jump instructions at this point.
12286 for (i = 0; i < p->n_instructions; i++) {
12287 struct instruction *instr = &p->instructions[i];
12289 /* Check for thread yield instructions. */
12290 if (!instruction_does_thread_yield(instr))
12293 /* If the current group contains at least one instruction, then finalize it (with
12294 * the previous instruction), add it to the list and allocate a new group (that
12295 * starts with the current instruction).
12297 if (i - g->first_instr_id) {
12298 /* Finalize the group. */
12299 g->last_instr_id = i - 1;
12301 /* Add the group to the list. Advance the number of groups. */
12302 TAILQ_INSERT_TAIL(igl, g, node);
12305 /* Allocate a new group. */
12306 g = calloc(1, sizeof(struct instruction_group));
12310 /* Initialize the new group. */
12311 g->group_id = n_groups;
12312 g->first_instr_id = i;
12315 /* Finalize the current group (with the current instruction, therefore this group
12316 * contains just the current thread yield instruction), add it to the list and
12317 * allocate a new group (that starts with the next instruction).
12320 /* Finalize the group. */
12321 g->last_instr_id = i;
12323 /* Add the group to the list. Advance the number of groups. */
12324 TAILQ_INSERT_TAIL(igl, g, node);
12327 /* Allocate a new group. */
12328 g = calloc(1, sizeof(struct instruction_group));
12332 /* Initialize the new group. */
12333 g->group_id = n_groups;
12334 g->first_instr_id = i + 1;
12337 /* Handle the last group. */
12338 if (i - g->first_instr_id) {
12339 /* Finalize the group. */
12340 g->last_instr_id = i - 1;
12342 /* Add the group to the list. Advance the number of groups. */
12343 TAILQ_INSERT_TAIL(igl, g, node);
12350 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
12351 * the destination of a jump instruction located in a different group ("far jump"), then the
12352 * current group has to be split, so that the instruction representing the far jump
12353 * destination is at the start of its group.
12356 int is_modified = 0;
12358 for (i = 0; i < p->n_instructions; i++) {
12359 struct instruction_data *data = &p->instruction_data[i];
12360 struct instruction_group *g;
12363 /* Continue when the current instruction is not a jump destination. */
12364 if (!data->n_users)
12367 g = instruction_group_list_group_find(igl, i);
12371 /* Find out all the jump instructions with this destination. */
12372 for (j = 0; j < p->n_instructions; j++) {
12373 struct instruction *jmp_instr = &p->instructions[j];
12374 struct instruction_data *jmp_data = &p->instruction_data[j];
12375 struct instruction_group *jmp_g, *new_g;
12377 /* Continue when not a jump instruction. Even when jump instruction,
12378 * continue when the jump destination is not this instruction.
12380 if (!instruction_is_jmp(jmp_instr) ||
12381 strcmp(jmp_data->jmp_label, data->label))
12384 jmp_g = instruction_group_list_group_find(igl, j);
12388 /* Continue when both the jump instruction and the jump destination
12389 * instruction are in the same group. Even when in different groups,
12390 * still continue if the jump destination instruction is already the
12391 * first instruction of its group.
12393 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
12396 /* Split the group of the current jump destination instruction to
12397 * make this instruction the first instruction of a new group.
12399 new_g = calloc(1, sizeof(struct instruction_group));
12403 new_g->group_id = n_groups;
12404 new_g->first_instr_id = i;
12405 new_g->last_instr_id = g->last_instr_id;
12407 g->last_instr_id = i - 1;
12409 TAILQ_INSERT_AFTER(igl, g, new_g, node);
12413 /* The decision to split this group (to make the current instruction
12414 * the first instruction of a new group) is already taken and fully
12415 * implemented, so no need to search for more reasons to do it.
12421 /* Re-evaluate everything, as at least one group got split, so some jumps that were
12422 * previously considered local (i.e. the jump destination is in the same group as
12423 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
12424 * different group than the jump instruction). Wost case scenario: each instruction
12425 * that is a jump destination ends up as the first instruction of its group.
12431 /* Re-assign the group IDs to be in incremental order. */
12433 TAILQ_FOREACH(g, igl, node) {
12442 instruction_group_list_free(igl);
12450 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12451 uint32_t instr_pos,
12452 struct instruction *instr,
12456 "%s(p, t, &pipeline_instructions[%u]);\n"
12457 "\tthread_ip_reset(p, t);\n"
12458 "\tinstr_rx_exec(p);\n"
12460 instr_type_to_func(instr),
12465 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12466 struct instruction_group_list *igl,
12467 uint32_t jmp_instr_id,
12468 struct instruction *jmp_instr,
12469 struct instruction_data *jmp_data,
12472 struct instruction_group *jmp_g, *g;
12473 struct instruction_data *data;
12476 switch (jmp_instr->type) {
12480 case INSTR_JMP_VALID:
12482 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12486 case INSTR_JMP_INVALID:
12488 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12492 case INSTR_JMP_HIT:
12497 case INSTR_JMP_MISS:
12502 case INSTR_JMP_ACTION_HIT:
12504 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12508 case INSTR_JMP_ACTION_MISS:
12510 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12516 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12517 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12522 case INSTR_JMP_EQ_MH:
12524 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12525 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12530 case INSTR_JMP_EQ_HM:
12532 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12533 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12538 case INSTR_JMP_EQ_HH:
12540 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12541 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12546 case INSTR_JMP_EQ_I:
12548 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12549 "pipeline_instructions[%u].jmp.b_val)",
12554 case INSTR_JMP_NEQ:
12556 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12557 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12562 case INSTR_JMP_NEQ_MH:
12564 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12565 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12570 case INSTR_JMP_NEQ_HM:
12572 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12573 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12578 case INSTR_JMP_NEQ_HH:
12580 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12581 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12586 case INSTR_JMP_NEQ_I:
12588 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12589 "pipeline_instructions[%u].jmp.b_val)",
12596 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12597 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12602 case INSTR_JMP_LT_MH:
12604 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12605 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12610 case INSTR_JMP_LT_HM:
12612 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12613 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12618 case INSTR_JMP_LT_HH:
12620 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12621 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12626 case INSTR_JMP_LT_MI:
12628 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12629 "pipeline_instructions[%u].jmp.b_val)",
12634 case INSTR_JMP_LT_HI:
12636 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12637 "pipeline_instructions[%u].jmp.b_val)",
12644 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12645 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12650 case INSTR_JMP_GT_MH:
12652 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12653 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12658 case INSTR_JMP_GT_HM:
12660 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12661 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12666 case INSTR_JMP_GT_HH:
12668 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12669 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12674 case INSTR_JMP_GT_MI:
12676 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12677 "pipeline_instructions[%u].jmp.b_val)",
12682 case INSTR_JMP_GT_HI:
12684 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12685 "pipeline_instructions[%u].jmp.b_val)",
12694 /* Find the instruction group of the jump instruction. */
12695 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12699 /* Find the instruction group of the jump destination instruction. */
12700 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12704 instr_id = data - p->instruction_data;
12706 g = instruction_group_list_group_find(igl, instr_id);
12710 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12711 * instruction group).
12713 if (g->group_id == jmp_g->group_id)
12715 "\n\t\tgoto %s;\n",
12716 jmp_data->jmp_label);
12720 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12729 instruction_group_list_codegen(struct instruction_group_list *igl,
12730 struct rte_swx_pipeline *p,
12733 struct instruction_group *g;
12735 int is_required = 0;
12737 /* Check if code generation is required. */
12738 TAILQ_FOREACH(g, igl, node)
12739 if (g->first_instr_id < g->last_instr_id)
12745 /* Generate the code for the pipeline instruction array. */
12747 "static const struct instruction pipeline_instructions[] = {\n");
12749 for (i = 0; i < p->n_instructions; i++) {
12750 struct instruction *instr = &p->instructions[i];
12751 instruction_export_t func = export_table[instr->type];
12756 fprintf(f, "};\n\n");
12758 /* Generate the code for the pipeline functions: one function for each instruction group
12759 * that contains more than one instruction.
12761 TAILQ_FOREACH(g, igl, node) {
12762 struct instruction *last_instr;
12765 /* Skip if group contains a single instruction. */
12766 if (g->last_instr_id == g->first_instr_id)
12769 /* Generate new pipeline function. */
12772 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12774 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12778 /* Generate the code for each pipeline instruction. */
12779 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12780 struct instruction *instr = &p->instructions[j];
12781 struct instruction_data *data = &p->instruction_data[j];
12783 /* Label, if present. */
12784 if (data->label[0])
12785 fprintf(f, "\n%s : ", data->label);
12787 fprintf(f, "\n\t");
12789 /* TX instruction type. */
12790 if (instruction_does_tx(instr)) {
12791 pipeline_instr_does_tx_codegen(p, j, instr, f);
12795 /* Jump instruction type. */
12796 if (instruction_is_jmp(instr)) {
12797 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12801 /* Any other instruction type. */
12803 "%s(p, t, &pipeline_instructions[%u]);\n",
12804 instr_type_to_func(instr),
12808 /* Finalize the generated pipeline function. For some instructions such as TX,
12809 * emit-many-and-TX and unconditional jump, the next instruction has been already
12810 * decided unconditionally and the instruction pointer of the current thread set
12811 * accordingly; for all the other instructions, the instruction pointer must be
12814 last_instr = &p->instructions[g->last_instr_id];
12816 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12818 "thread_ip_inc(p);\n");
12827 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12829 struct instruction_group *g;
12830 uint32_t n_custom_instr = 0;
12832 /* Groups with a single instruction: no function is generated for this group, the group
12833 * keeps its current instruction. Groups with more than two instructions: one function and
12834 * the associated custom instruction get generated for each such group.
12836 TAILQ_FOREACH(g, igl, node) {
12837 if (g->first_instr_id == g->last_instr_id)
12843 return n_custom_instr;
12847 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12852 /* Create the .c file. */
12853 f = fopen("/tmp/pipeline.c", "w");
12857 /* Include the .h file. */
12858 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12860 /* Add the code for each action. */
12861 TAILQ_FOREACH(a, &p->actions, node) {
12862 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12864 action_data_codegen(a, f);
12868 action_instr_codegen(a, f);
12873 /* Add the pipeline code. */
12874 instruction_group_list_codegen(igl, p, f);
12876 /* Close the .c file. */
12882 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12883 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12887 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12890 struct instruction_group *g;
12891 char *dir_in, *buffer = NULL;
12892 const char *dir_out;
12895 /* Get the environment variables. */
12896 dir_in = getenv("RTE_INSTALL_DIR");
12904 /* Memory allocation for the command buffer. */
12905 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12912 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12913 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12914 "-I %s/lib/pipeline "
12915 "-I %s/lib/eal/include "
12916 "-I %s/lib/eal/x86/include "
12917 "-I %s/lib/eal/include/generic "
12921 "-I %s/lib/pipeline "
12924 "-I %s/lib/eal/linux/include "
12925 ">%s/pipeline.log 2>&1 "
12927 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12928 ">>%s/pipeline.log 2>&1",
12947 /* Build the shared object library. */
12948 status = system(buffer);
12952 /* Open library. */
12954 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12955 "%s/libpipeline.so",
12958 p->lib = dlopen(buffer, RTLD_LAZY);
12964 /* Get the action function symbols. */
12965 TAILQ_FOREACH(a, &p->actions, node) {
12966 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12968 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12969 if (!p->action_funcs[a->id]) {
12975 /* Get the pipeline function symbols. */
12976 TAILQ_FOREACH(g, igl, node) {
12977 if (g->first_instr_id == g->last_instr_id)
12980 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12982 g->func = dlsym(p->lib, buffer);
12990 if (status && p->lib) {
13001 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
13002 struct instruction_group_list *igl)
13004 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
13006 /* Check that enough space is available within the pipeline instruction table to store all
13007 * the custom instructions.
13009 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
13016 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
13018 struct instruction_group *g;
13021 /* Pipeline table instructions. */
13022 for (i = 0; i < p->n_instructions; i++) {
13023 struct instruction *instr = &p->instructions[i];
13025 if (instr->type == INSTR_TABLE)
13026 instr->type = INSTR_TABLE_AF;
13028 if (instr->type == INSTR_LEARNER)
13029 instr->type = INSTR_LEARNER_AF;
13032 /* Pipeline custom instructions. */
13034 TAILQ_FOREACH(g, igl, node) {
13035 struct instruction *instr = &p->instructions[g->first_instr_id];
13038 if (g->first_instr_id == g->last_instr_id)
13041 /* Install a new custom instruction. */
13042 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
13044 /* First instruction of the group: change its type to the new custom instruction. */
13045 instr->type = INSTR_CUSTOM_0 + i;
13047 /* All the subsequent instructions of the group: invalidate. */
13048 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
13049 struct instruction_data *data = &p->instruction_data[j];
13057 /* Remove the invalidated instructions. */
13058 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
13060 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
13061 * instructions that are the only instruction within their group, so they were left
13064 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
13068 pipeline_compile(struct rte_swx_pipeline *p)
13070 struct instruction_group_list *igl = NULL;
13073 igl = instruction_group_list_create(p);
13079 /* Code generation. */
13080 status = pipeline_codegen(p, igl);
13084 /* Build and load the shared object library. */
13085 status = pipeline_libload(p, igl);
13089 /* Adjust instructions. */
13090 status = pipeline_adjust_check(p, igl);
13094 pipeline_adjust(p, igl);
13097 instruction_group_list_free(igl);