1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <rte_swx_port_ethdev.h>
11 #include <rte_swx_port_fd.h>
12 #include <rte_swx_port_ring.h>
13 #include "rte_swx_port_source_sink.h"
15 #include <rte_swx_table_em.h>
16 #include <rte_swx_table_wm.h>
18 #include "rte_swx_pipeline_internal.h"
20 #define CHECK(condition, err_code) \
26 #define CHECK_NAME(name, err_code) \
29 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
32 #define CHECK_INSTRUCTION(instr, err_code) \
35 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
36 RTE_SWX_INSTRUCTION_SIZE), \
42 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
44 #include <rte_malloc.h>
47 env_malloc(size_t size, size_t alignment, int numa_node)
49 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
53 env_free(void *start, size_t size __rte_unused)
63 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
67 if (numa_available() == -1)
70 start = numa_alloc_onnode(size, numa_node);
74 memset(start, 0, size);
79 env_free(void *start, size_t size)
81 if (numa_available() == -1)
84 numa_free(start, size);
92 static struct struct_type *
93 struct_type_find(struct rte_swx_pipeline *p, const char *name)
95 struct struct_type *elem;
97 TAILQ_FOREACH(elem, &p->struct_types, node)
98 if (strcmp(elem->name, name) == 0)
104 static struct field *
105 struct_type_field_find(struct struct_type *st, const char *name)
109 for (i = 0; i < st->n_fields; i++) {
110 struct field *f = &st->fields[i];
112 if (strcmp(f->name, name) == 0)
120 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
122 struct rte_swx_field_params *fields,
124 int last_field_has_variable_size)
126 struct struct_type *st;
130 CHECK_NAME(name, EINVAL);
131 CHECK(fields, EINVAL);
132 CHECK(n_fields, EINVAL);
134 for (i = 0; i < n_fields; i++) {
135 struct rte_swx_field_params *f = &fields[i];
136 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
139 CHECK_NAME(f->name, EINVAL);
140 CHECK(f->n_bits, EINVAL);
141 CHECK((f->n_bits <= 64) || var_size, EINVAL);
142 CHECK((f->n_bits & 7) == 0, EINVAL);
144 for (j = 0; j < i; j++) {
145 struct rte_swx_field_params *f_prev = &fields[j];
147 CHECK(strcmp(f->name, f_prev->name), EINVAL);
151 CHECK(!struct_type_find(p, name), EEXIST);
153 /* Node allocation. */
154 st = calloc(1, sizeof(struct struct_type));
157 st->fields = calloc(n_fields, sizeof(struct field));
163 /* Node initialization. */
164 strcpy(st->name, name);
165 for (i = 0; i < n_fields; i++) {
166 struct field *dst = &st->fields[i];
167 struct rte_swx_field_params *src = &fields[i];
168 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
170 strcpy(dst->name, src->name);
171 dst->n_bits = src->n_bits;
172 dst->offset = st->n_bits;
173 dst->var_size = var_size;
175 st->n_bits += src->n_bits;
176 st->n_bits_min += var_size ? 0 : src->n_bits;
178 st->n_fields = n_fields;
179 st->var_size = last_field_has_variable_size;
181 /* Node add to tailq. */
182 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
188 struct_build(struct rte_swx_pipeline *p)
192 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
193 struct thread *t = &p->threads[i];
195 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
196 CHECK(t->structs, ENOMEM);
203 struct_build_free(struct rte_swx_pipeline *p)
207 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
208 struct thread *t = &p->threads[i];
216 struct_free(struct rte_swx_pipeline *p)
218 struct_build_free(p);
222 struct struct_type *elem;
224 elem = TAILQ_FIRST(&p->struct_types);
228 TAILQ_REMOVE(&p->struct_types, elem, node);
237 static struct port_in_type *
238 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
240 struct port_in_type *elem;
245 TAILQ_FOREACH(elem, &p->port_in_types, node)
246 if (strcmp(elem->name, name) == 0)
253 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
255 struct rte_swx_port_in_ops *ops)
257 struct port_in_type *elem;
260 CHECK_NAME(name, EINVAL);
262 CHECK(ops->create, EINVAL);
263 CHECK(ops->free, EINVAL);
264 CHECK(ops->pkt_rx, EINVAL);
265 CHECK(ops->stats_read, EINVAL);
267 CHECK(!port_in_type_find(p, name), EEXIST);
269 /* Node allocation. */
270 elem = calloc(1, sizeof(struct port_in_type));
273 /* Node initialization. */
274 strcpy(elem->name, name);
275 memcpy(&elem->ops, ops, sizeof(*ops));
277 /* Node add to tailq. */
278 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
283 static struct port_in *
284 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
286 struct port_in *port;
288 TAILQ_FOREACH(port, &p->ports_in, node)
289 if (port->id == port_id)
296 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
298 const char *port_type_name,
301 struct port_in_type *type = NULL;
302 struct port_in *port = NULL;
307 CHECK(!port_in_find(p, port_id), EINVAL);
309 CHECK_NAME(port_type_name, EINVAL);
310 type = port_in_type_find(p, port_type_name);
313 obj = type->ops.create(args);
316 /* Node allocation. */
317 port = calloc(1, sizeof(struct port_in));
320 /* Node initialization. */
325 /* Node add to tailq. */
326 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
327 if (p->n_ports_in < port_id + 1)
328 p->n_ports_in = port_id + 1;
334 port_in_build(struct rte_swx_pipeline *p)
336 struct port_in *port;
339 CHECK(p->n_ports_in, EINVAL);
340 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
342 for (i = 0; i < p->n_ports_in; i++)
343 CHECK(port_in_find(p, i), EINVAL);
345 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
346 CHECK(p->in, ENOMEM);
348 TAILQ_FOREACH(port, &p->ports_in, node) {
349 struct port_in_runtime *in = &p->in[port->id];
351 in->pkt_rx = port->type->ops.pkt_rx;
359 port_in_build_free(struct rte_swx_pipeline *p)
366 port_in_free(struct rte_swx_pipeline *p)
368 port_in_build_free(p);
372 struct port_in *port;
374 port = TAILQ_FIRST(&p->ports_in);
378 TAILQ_REMOVE(&p->ports_in, port, node);
379 port->type->ops.free(port->obj);
383 /* Input port types. */
385 struct port_in_type *elem;
387 elem = TAILQ_FIRST(&p->port_in_types);
391 TAILQ_REMOVE(&p->port_in_types, elem, node);
399 static struct port_out_type *
400 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
402 struct port_out_type *elem;
407 TAILQ_FOREACH(elem, &p->port_out_types, node)
408 if (!strcmp(elem->name, name))
415 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
417 struct rte_swx_port_out_ops *ops)
419 struct port_out_type *elem;
422 CHECK_NAME(name, EINVAL);
424 CHECK(ops->create, EINVAL);
425 CHECK(ops->free, EINVAL);
426 CHECK(ops->pkt_tx, EINVAL);
427 CHECK(ops->stats_read, EINVAL);
429 CHECK(!port_out_type_find(p, name), EEXIST);
431 /* Node allocation. */
432 elem = calloc(1, sizeof(struct port_out_type));
435 /* Node initialization. */
436 strcpy(elem->name, name);
437 memcpy(&elem->ops, ops, sizeof(*ops));
439 /* Node add to tailq. */
440 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
445 static struct port_out *
446 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
448 struct port_out *port;
450 TAILQ_FOREACH(port, &p->ports_out, node)
451 if (port->id == port_id)
458 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
460 const char *port_type_name,
463 struct port_out_type *type = NULL;
464 struct port_out *port = NULL;
469 CHECK(!port_out_find(p, port_id), EINVAL);
471 CHECK_NAME(port_type_name, EINVAL);
472 type = port_out_type_find(p, port_type_name);
475 obj = type->ops.create(args);
478 /* Node allocation. */
479 port = calloc(1, sizeof(struct port_out));
482 /* Node initialization. */
487 /* Node add to tailq. */
488 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
489 if (p->n_ports_out < port_id + 1)
490 p->n_ports_out = port_id + 1;
496 port_out_build(struct rte_swx_pipeline *p)
498 struct port_out *port;
501 CHECK(p->n_ports_out, EINVAL);
503 for (i = 0; i < p->n_ports_out; i++)
504 CHECK(port_out_find(p, i), EINVAL);
506 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
507 CHECK(p->out, ENOMEM);
509 TAILQ_FOREACH(port, &p->ports_out, node) {
510 struct port_out_runtime *out = &p->out[port->id];
512 out->pkt_tx = port->type->ops.pkt_tx;
513 out->flush = port->type->ops.flush;
514 out->obj = port->obj;
521 port_out_build_free(struct rte_swx_pipeline *p)
528 port_out_free(struct rte_swx_pipeline *p)
530 port_out_build_free(p);
534 struct port_out *port;
536 port = TAILQ_FIRST(&p->ports_out);
540 TAILQ_REMOVE(&p->ports_out, port, node);
541 port->type->ops.free(port->obj);
545 /* Output port types. */
547 struct port_out_type *elem;
549 elem = TAILQ_FIRST(&p->port_out_types);
553 TAILQ_REMOVE(&p->port_out_types, elem, node);
561 static struct extern_type *
562 extern_type_find(struct rte_swx_pipeline *p, const char *name)
564 struct extern_type *elem;
566 TAILQ_FOREACH(elem, &p->extern_types, node)
567 if (strcmp(elem->name, name) == 0)
573 static struct extern_type_member_func *
574 extern_type_member_func_find(struct extern_type *type, const char *name)
576 struct extern_type_member_func *elem;
578 TAILQ_FOREACH(elem, &type->funcs, node)
579 if (strcmp(elem->name, name) == 0)
585 static struct extern_obj *
586 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
588 struct extern_obj *elem;
590 TAILQ_FOREACH(elem, &p->extern_objs, node)
591 if (strcmp(elem->name, name) == 0)
597 static struct extern_type_member_func *
598 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
600 struct extern_obj **obj)
602 struct extern_obj *object;
603 struct extern_type_member_func *func;
604 char *object_name, *func_name;
606 if (name[0] != 'e' || name[1] != '.')
609 object_name = strdup(&name[2]);
613 func_name = strchr(object_name, '.');
622 object = extern_obj_find(p, object_name);
628 func = extern_type_member_func_find(object->type, func_name);
641 static struct field *
642 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
644 struct extern_obj **object)
646 struct extern_obj *obj;
648 char *obj_name, *field_name;
650 if ((name[0] != 'e') || (name[1] != '.'))
653 obj_name = strdup(&name[2]);
657 field_name = strchr(obj_name, '.');
666 obj = extern_obj_find(p, obj_name);
672 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
686 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
688 const char *mailbox_struct_type_name,
689 rte_swx_extern_type_constructor_t constructor,
690 rte_swx_extern_type_destructor_t destructor)
692 struct extern_type *elem;
693 struct struct_type *mailbox_struct_type;
697 CHECK_NAME(name, EINVAL);
698 CHECK(!extern_type_find(p, name), EEXIST);
700 CHECK_NAME(mailbox_struct_type_name, EINVAL);
701 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
702 CHECK(mailbox_struct_type, EINVAL);
703 CHECK(!mailbox_struct_type->var_size, EINVAL);
705 CHECK(constructor, EINVAL);
706 CHECK(destructor, EINVAL);
708 /* Node allocation. */
709 elem = calloc(1, sizeof(struct extern_type));
712 /* Node initialization. */
713 strcpy(elem->name, name);
714 elem->mailbox_struct_type = mailbox_struct_type;
715 elem->constructor = constructor;
716 elem->destructor = destructor;
717 TAILQ_INIT(&elem->funcs);
719 /* Node add to tailq. */
720 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
726 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
727 const char *extern_type_name,
729 rte_swx_extern_type_member_func_t member_func)
731 struct extern_type *type;
732 struct extern_type_member_func *type_member;
736 CHECK_NAME(extern_type_name, EINVAL);
737 type = extern_type_find(p, extern_type_name);
739 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
741 CHECK_NAME(name, EINVAL);
742 CHECK(!extern_type_member_func_find(type, name), EEXIST);
744 CHECK(member_func, EINVAL);
746 /* Node allocation. */
747 type_member = calloc(1, sizeof(struct extern_type_member_func));
748 CHECK(type_member, ENOMEM);
750 /* Node initialization. */
751 strcpy(type_member->name, name);
752 type_member->func = member_func;
753 type_member->id = type->n_funcs;
755 /* Node add to tailq. */
756 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
763 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
764 const char *extern_type_name,
768 struct extern_type *type;
769 struct extern_obj *obj;
774 CHECK_NAME(extern_type_name, EINVAL);
775 type = extern_type_find(p, extern_type_name);
778 CHECK_NAME(name, EINVAL);
779 CHECK(!extern_obj_find(p, name), EEXIST);
781 /* Node allocation. */
782 obj = calloc(1, sizeof(struct extern_obj));
785 /* Object construction. */
786 obj_handle = type->constructor(args);
792 /* Node initialization. */
793 strcpy(obj->name, name);
795 obj->obj = obj_handle;
796 obj->struct_id = p->n_structs;
797 obj->id = p->n_extern_objs;
799 /* Node add to tailq. */
800 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
808 extern_obj_build(struct rte_swx_pipeline *p)
812 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
813 struct thread *t = &p->threads[i];
814 struct extern_obj *obj;
816 t->extern_objs = calloc(p->n_extern_objs,
817 sizeof(struct extern_obj_runtime));
818 CHECK(t->extern_objs, ENOMEM);
820 TAILQ_FOREACH(obj, &p->extern_objs, node) {
821 struct extern_obj_runtime *r =
822 &t->extern_objs[obj->id];
823 struct extern_type_member_func *func;
824 uint32_t mailbox_size =
825 obj->type->mailbox_struct_type->n_bits / 8;
829 r->mailbox = calloc(1, mailbox_size);
830 CHECK(r->mailbox, ENOMEM);
832 TAILQ_FOREACH(func, &obj->type->funcs, node)
833 r->funcs[func->id] = func->func;
835 t->structs[obj->struct_id] = r->mailbox;
843 extern_obj_build_free(struct rte_swx_pipeline *p)
847 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
848 struct thread *t = &p->threads[i];
854 for (j = 0; j < p->n_extern_objs; j++) {
855 struct extern_obj_runtime *r = &t->extern_objs[j];
860 free(t->extern_objs);
861 t->extern_objs = NULL;
866 extern_obj_free(struct rte_swx_pipeline *p)
868 extern_obj_build_free(p);
870 /* Extern objects. */
872 struct extern_obj *elem;
874 elem = TAILQ_FIRST(&p->extern_objs);
878 TAILQ_REMOVE(&p->extern_objs, elem, node);
880 elem->type->destructor(elem->obj);
886 struct extern_type *elem;
888 elem = TAILQ_FIRST(&p->extern_types);
892 TAILQ_REMOVE(&p->extern_types, elem, node);
895 struct extern_type_member_func *func;
897 func = TAILQ_FIRST(&elem->funcs);
901 TAILQ_REMOVE(&elem->funcs, func, node);
912 static struct extern_func *
913 extern_func_find(struct rte_swx_pipeline *p, const char *name)
915 struct extern_func *elem;
917 TAILQ_FOREACH(elem, &p->extern_funcs, node)
918 if (strcmp(elem->name, name) == 0)
924 static struct extern_func *
925 extern_func_parse(struct rte_swx_pipeline *p,
928 if (name[0] != 'f' || name[1] != '.')
931 return extern_func_find(p, &name[2]);
934 static struct field *
935 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
937 struct extern_func **function)
939 struct extern_func *func;
941 char *func_name, *field_name;
943 if ((name[0] != 'f') || (name[1] != '.'))
946 func_name = strdup(&name[2]);
950 field_name = strchr(func_name, '.');
959 func = extern_func_find(p, func_name);
965 f = struct_type_field_find(func->mailbox_struct_type, field_name);
979 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
981 const char *mailbox_struct_type_name,
982 rte_swx_extern_func_t func)
984 struct extern_func *f;
985 struct struct_type *mailbox_struct_type;
989 CHECK_NAME(name, EINVAL);
990 CHECK(!extern_func_find(p, name), EEXIST);
992 CHECK_NAME(mailbox_struct_type_name, EINVAL);
993 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
994 CHECK(mailbox_struct_type, EINVAL);
995 CHECK(!mailbox_struct_type->var_size, EINVAL);
999 /* Node allocation. */
1000 f = calloc(1, sizeof(struct extern_func));
1001 CHECK(func, ENOMEM);
1003 /* Node initialization. */
1004 strcpy(f->name, name);
1005 f->mailbox_struct_type = mailbox_struct_type;
1007 f->struct_id = p->n_structs;
1008 f->id = p->n_extern_funcs;
1010 /* Node add to tailq. */
1011 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1012 p->n_extern_funcs++;
1019 extern_func_build(struct rte_swx_pipeline *p)
1023 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1024 struct thread *t = &p->threads[i];
1025 struct extern_func *func;
1027 /* Memory allocation. */
1028 t->extern_funcs = calloc(p->n_extern_funcs,
1029 sizeof(struct extern_func_runtime));
1030 CHECK(t->extern_funcs, ENOMEM);
1032 /* Extern function. */
1033 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1034 struct extern_func_runtime *r =
1035 &t->extern_funcs[func->id];
1036 uint32_t mailbox_size =
1037 func->mailbox_struct_type->n_bits / 8;
1039 r->func = func->func;
1041 r->mailbox = calloc(1, mailbox_size);
1042 CHECK(r->mailbox, ENOMEM);
1044 t->structs[func->struct_id] = r->mailbox;
1052 extern_func_build_free(struct rte_swx_pipeline *p)
1056 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1057 struct thread *t = &p->threads[i];
1060 if (!t->extern_funcs)
1063 for (j = 0; j < p->n_extern_funcs; j++) {
1064 struct extern_func_runtime *r = &t->extern_funcs[j];
1069 free(t->extern_funcs);
1070 t->extern_funcs = NULL;
1075 extern_func_free(struct rte_swx_pipeline *p)
1077 extern_func_build_free(p);
1080 struct extern_func *elem;
1082 elem = TAILQ_FIRST(&p->extern_funcs);
1086 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1094 static struct header *
1095 header_find(struct rte_swx_pipeline *p, const char *name)
1097 struct header *elem;
1099 TAILQ_FOREACH(elem, &p->headers, node)
1100 if (strcmp(elem->name, name) == 0)
1106 static struct header *
1107 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1109 struct header *elem;
1111 TAILQ_FOREACH(elem, &p->headers, node)
1112 if (elem->struct_id == struct_id)
1118 static struct header *
1119 header_parse(struct rte_swx_pipeline *p,
1122 if (name[0] != 'h' || name[1] != '.')
1125 return header_find(p, &name[2]);
1128 static struct field *
1129 header_field_parse(struct rte_swx_pipeline *p,
1131 struct header **header)
1135 char *header_name, *field_name;
1137 if ((name[0] != 'h') || (name[1] != '.'))
1140 header_name = strdup(&name[2]);
1144 field_name = strchr(header_name, '.');
1153 h = header_find(p, header_name);
1159 f = struct_type_field_find(h->st, field_name);
1173 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1175 const char *struct_type_name)
1177 struct struct_type *st;
1179 size_t n_headers_max;
1182 CHECK_NAME(name, EINVAL);
1183 CHECK_NAME(struct_type_name, EINVAL);
1185 CHECK(!header_find(p, name), EEXIST);
1187 st = struct_type_find(p, struct_type_name);
1190 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1191 CHECK(p->n_headers < n_headers_max, ENOSPC);
1193 /* Node allocation. */
1194 h = calloc(1, sizeof(struct header));
1197 /* Node initialization. */
1198 strcpy(h->name, name);
1200 h->struct_id = p->n_structs;
1201 h->id = p->n_headers;
1203 /* Node add to tailq. */
1204 TAILQ_INSERT_TAIL(&p->headers, h, node);
1212 header_build(struct rte_swx_pipeline *p)
1215 uint32_t n_bytes = 0, i;
1217 TAILQ_FOREACH(h, &p->headers, node) {
1218 n_bytes += h->st->n_bits / 8;
1221 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1222 struct thread *t = &p->threads[i];
1223 uint32_t offset = 0;
1225 t->headers = calloc(p->n_headers,
1226 sizeof(struct header_runtime));
1227 CHECK(t->headers, ENOMEM);
1229 t->headers_out = calloc(p->n_headers,
1230 sizeof(struct header_out_runtime));
1231 CHECK(t->headers_out, ENOMEM);
1233 t->header_storage = calloc(1, n_bytes);
1234 CHECK(t->header_storage, ENOMEM);
1236 t->header_out_storage = calloc(1, n_bytes);
1237 CHECK(t->header_out_storage, ENOMEM);
1239 TAILQ_FOREACH(h, &p->headers, node) {
1240 uint8_t *header_storage;
1241 uint32_t n_bytes = h->st->n_bits / 8;
1243 header_storage = &t->header_storage[offset];
1246 t->headers[h->id].ptr0 = header_storage;
1247 t->headers[h->id].n_bytes = n_bytes;
1249 t->structs[h->struct_id] = header_storage;
1257 header_build_free(struct rte_swx_pipeline *p)
1261 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1262 struct thread *t = &p->threads[i];
1264 free(t->headers_out);
1265 t->headers_out = NULL;
1270 free(t->header_out_storage);
1271 t->header_out_storage = NULL;
1273 free(t->header_storage);
1274 t->header_storage = NULL;
1279 header_free(struct rte_swx_pipeline *p)
1281 header_build_free(p);
1284 struct header *elem;
1286 elem = TAILQ_FIRST(&p->headers);
1290 TAILQ_REMOVE(&p->headers, elem, node);
1298 static struct field *
1299 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1301 if (!p->metadata_st)
1304 if (name[0] != 'm' || name[1] != '.')
1307 return struct_type_field_find(p->metadata_st, &name[2]);
1311 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1312 const char *struct_type_name)
1314 struct struct_type *st = NULL;
1318 CHECK_NAME(struct_type_name, EINVAL);
1319 st = struct_type_find(p, struct_type_name);
1321 CHECK(!st->var_size, EINVAL);
1322 CHECK(!p->metadata_st, EINVAL);
1324 p->metadata_st = st;
1325 p->metadata_struct_id = p->n_structs;
1333 metadata_build(struct rte_swx_pipeline *p)
1335 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1338 /* Thread-level initialization. */
1339 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1340 struct thread *t = &p->threads[i];
1343 metadata = calloc(1, n_bytes);
1344 CHECK(metadata, ENOMEM);
1346 t->metadata = metadata;
1347 t->structs[p->metadata_struct_id] = metadata;
1354 metadata_build_free(struct rte_swx_pipeline *p)
1358 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1359 struct thread *t = &p->threads[i];
1367 metadata_free(struct rte_swx_pipeline *p)
1369 metadata_build_free(p);
1376 instruction_is_tx(enum instruction_type type)
1390 instruction_does_tx(struct instruction *instr)
1392 switch (instr->type) {
1396 case INSTR_HDR_EMIT_TX:
1397 case INSTR_HDR_EMIT2_TX:
1398 case INSTR_HDR_EMIT3_TX:
1399 case INSTR_HDR_EMIT4_TX:
1400 case INSTR_HDR_EMIT5_TX:
1401 case INSTR_HDR_EMIT6_TX:
1402 case INSTR_HDR_EMIT7_TX:
1403 case INSTR_HDR_EMIT8_TX:
1411 instruction_is_jmp(struct instruction *instr)
1413 switch (instr->type) {
1415 case INSTR_JMP_VALID:
1416 case INSTR_JMP_INVALID:
1418 case INSTR_JMP_MISS:
1419 case INSTR_JMP_ACTION_HIT:
1420 case INSTR_JMP_ACTION_MISS:
1422 case INSTR_JMP_EQ_MH:
1423 case INSTR_JMP_EQ_HM:
1424 case INSTR_JMP_EQ_HH:
1425 case INSTR_JMP_EQ_I:
1427 case INSTR_JMP_NEQ_MH:
1428 case INSTR_JMP_NEQ_HM:
1429 case INSTR_JMP_NEQ_HH:
1430 case INSTR_JMP_NEQ_I:
1432 case INSTR_JMP_LT_MH:
1433 case INSTR_JMP_LT_HM:
1434 case INSTR_JMP_LT_HH:
1435 case INSTR_JMP_LT_MI:
1436 case INSTR_JMP_LT_HI:
1438 case INSTR_JMP_GT_MH:
1439 case INSTR_JMP_GT_HM:
1440 case INSTR_JMP_GT_HH:
1441 case INSTR_JMP_GT_MI:
1442 case INSTR_JMP_GT_HI:
1451 instruction_does_thread_yield(struct instruction *instr)
1453 switch (instr->type) {
1456 case INSTR_TABLE_AF:
1457 case INSTR_SELECTOR:
1459 case INSTR_LEARNER_AF:
1460 case INSTR_EXTERN_OBJ:
1461 case INSTR_EXTERN_FUNC:
1468 static struct field *
1469 action_field_parse(struct action *action, const char *name);
1471 static struct field *
1472 struct_field_parse(struct rte_swx_pipeline *p,
1473 struct action *action,
1475 uint32_t *struct_id)
1482 struct header *header;
1484 f = header_field_parse(p, name, &header);
1488 *struct_id = header->struct_id;
1494 f = metadata_field_parse(p, name);
1498 *struct_id = p->metadata_struct_id;
1507 f = action_field_parse(action, name);
1517 struct extern_obj *obj;
1519 f = extern_obj_mailbox_field_parse(p, name, &obj);
1523 *struct_id = obj->struct_id;
1529 struct extern_func *func;
1531 f = extern_func_mailbox_field_parse(p, name, &func);
1535 *struct_id = func->struct_id;
1548 instr_rx_translate(struct rte_swx_pipeline *p,
1549 struct action *action,
1552 struct instruction *instr,
1553 struct instruction_data *data __rte_unused)
1557 CHECK(!action, EINVAL);
1558 CHECK(n_tokens == 2, EINVAL);
1560 f = metadata_field_parse(p, tokens[1]);
1563 instr->type = INSTR_RX;
1564 instr->io.io.offset = f->offset / 8;
1565 instr->io.io.n_bits = f->n_bits;
1573 instr_tx_translate(struct rte_swx_pipeline *p,
1574 struct action *action __rte_unused,
1577 struct instruction *instr,
1578 struct instruction_data *data __rte_unused)
1580 char *port = tokens[1];
1584 CHECK(n_tokens == 2, EINVAL);
1586 f = metadata_field_parse(p, port);
1588 instr->type = INSTR_TX;
1589 instr->io.io.offset = f->offset / 8;
1590 instr->io.io.n_bits = f->n_bits;
1595 port_val = strtoul(port, &port, 0);
1596 CHECK(!port[0], EINVAL);
1598 instr->type = INSTR_TX_I;
1599 instr->io.io.val = port_val;
1604 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1605 struct action *action __rte_unused,
1606 char **tokens __rte_unused,
1608 struct instruction *instr,
1609 struct instruction_data *data __rte_unused)
1611 CHECK(n_tokens == 1, EINVAL);
1614 instr->type = INSTR_DROP;
1619 instr_tx_exec(struct rte_swx_pipeline *p)
1621 struct thread *t = &p->threads[p->thread_id];
1622 struct instruction *ip = t->ip;
1624 __instr_tx_exec(p, t, ip);
1627 thread_ip_reset(p, t);
1632 instr_tx_i_exec(struct rte_swx_pipeline *p)
1634 struct thread *t = &p->threads[p->thread_id];
1635 struct instruction *ip = t->ip;
1637 __instr_tx_i_exec(p, t, ip);
1640 thread_ip_reset(p, t);
1645 instr_drop_exec(struct rte_swx_pipeline *p)
1647 struct thread *t = &p->threads[p->thread_id];
1648 struct instruction *ip = t->ip;
1650 __instr_drop_exec(p, t, ip);
1653 thread_ip_reset(p, t);
1661 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1662 struct action *action,
1665 struct instruction *instr,
1666 struct instruction_data *data __rte_unused)
1670 CHECK(!action, EINVAL);
1671 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1673 h = header_parse(p, tokens[1]);
1676 if (n_tokens == 2) {
1677 CHECK(!h->st->var_size, EINVAL);
1679 instr->type = INSTR_HDR_EXTRACT;
1680 instr->io.hdr.header_id[0] = h->id;
1681 instr->io.hdr.struct_id[0] = h->struct_id;
1682 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1686 CHECK(h->st->var_size, EINVAL);
1688 mf = metadata_field_parse(p, tokens[2]);
1690 CHECK(!mf->var_size, EINVAL);
1692 instr->type = INSTR_HDR_EXTRACT_M;
1693 instr->io.io.offset = mf->offset / 8;
1694 instr->io.io.n_bits = mf->n_bits;
1695 instr->io.hdr.header_id[0] = h->id;
1696 instr->io.hdr.struct_id[0] = h->struct_id;
1697 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1704 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1705 struct action *action,
1708 struct instruction *instr,
1709 struct instruction_data *data __rte_unused)
1713 CHECK(!action, EINVAL);
1714 CHECK(n_tokens == 2, EINVAL);
1716 h = header_parse(p, tokens[1]);
1718 CHECK(!h->st->var_size, EINVAL);
1720 instr->type = INSTR_HDR_LOOKAHEAD;
1721 instr->io.hdr.header_id[0] = h->id;
1722 instr->io.hdr.struct_id[0] = h->struct_id;
1723 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1729 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1731 struct thread *t = &p->threads[p->thread_id];
1732 struct instruction *ip = t->ip;
1734 __instr_hdr_extract_exec(p, t, ip);
1741 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1743 struct thread *t = &p->threads[p->thread_id];
1744 struct instruction *ip = t->ip;
1746 __instr_hdr_extract2_exec(p, t, ip);
1753 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1755 struct thread *t = &p->threads[p->thread_id];
1756 struct instruction *ip = t->ip;
1758 __instr_hdr_extract3_exec(p, t, ip);
1765 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1767 struct thread *t = &p->threads[p->thread_id];
1768 struct instruction *ip = t->ip;
1770 __instr_hdr_extract4_exec(p, t, ip);
1777 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1779 struct thread *t = &p->threads[p->thread_id];
1780 struct instruction *ip = t->ip;
1782 __instr_hdr_extract5_exec(p, t, ip);
1789 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1791 struct thread *t = &p->threads[p->thread_id];
1792 struct instruction *ip = t->ip;
1794 __instr_hdr_extract6_exec(p, t, ip);
1801 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1803 struct thread *t = &p->threads[p->thread_id];
1804 struct instruction *ip = t->ip;
1806 __instr_hdr_extract7_exec(p, t, ip);
1813 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1815 struct thread *t = &p->threads[p->thread_id];
1816 struct instruction *ip = t->ip;
1818 __instr_hdr_extract8_exec(p, t, ip);
1825 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1827 struct thread *t = &p->threads[p->thread_id];
1828 struct instruction *ip = t->ip;
1830 __instr_hdr_extract_m_exec(p, t, ip);
1837 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1839 struct thread *t = &p->threads[p->thread_id];
1840 struct instruction *ip = t->ip;
1842 __instr_hdr_lookahead_exec(p, t, ip);
1852 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1853 struct action *action __rte_unused,
1856 struct instruction *instr,
1857 struct instruction_data *data __rte_unused)
1861 CHECK(n_tokens == 2, EINVAL);
1863 h = header_parse(p, tokens[1]);
1866 instr->type = INSTR_HDR_EMIT;
1867 instr->io.hdr.header_id[0] = h->id;
1868 instr->io.hdr.struct_id[0] = h->struct_id;
1869 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1874 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1876 struct thread *t = &p->threads[p->thread_id];
1877 struct instruction *ip = t->ip;
1879 __instr_hdr_emit_exec(p, t, ip);
1886 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1888 struct thread *t = &p->threads[p->thread_id];
1889 struct instruction *ip = t->ip;
1891 __instr_hdr_emit_tx_exec(p, t, ip);
1894 thread_ip_reset(p, t);
1899 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1901 struct thread *t = &p->threads[p->thread_id];
1902 struct instruction *ip = t->ip;
1904 __instr_hdr_emit2_tx_exec(p, t, ip);
1907 thread_ip_reset(p, t);
1912 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1914 struct thread *t = &p->threads[p->thread_id];
1915 struct instruction *ip = t->ip;
1917 __instr_hdr_emit3_tx_exec(p, t, ip);
1920 thread_ip_reset(p, t);
1925 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1927 struct thread *t = &p->threads[p->thread_id];
1928 struct instruction *ip = t->ip;
1930 __instr_hdr_emit4_tx_exec(p, t, ip);
1933 thread_ip_reset(p, t);
1938 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1940 struct thread *t = &p->threads[p->thread_id];
1941 struct instruction *ip = t->ip;
1943 __instr_hdr_emit5_tx_exec(p, t, ip);
1946 thread_ip_reset(p, t);
1951 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1953 struct thread *t = &p->threads[p->thread_id];
1954 struct instruction *ip = t->ip;
1956 __instr_hdr_emit6_tx_exec(p, t, ip);
1959 thread_ip_reset(p, t);
1964 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1966 struct thread *t = &p->threads[p->thread_id];
1967 struct instruction *ip = t->ip;
1969 __instr_hdr_emit7_tx_exec(p, t, ip);
1972 thread_ip_reset(p, t);
1977 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1979 struct thread *t = &p->threads[p->thread_id];
1980 struct instruction *ip = t->ip;
1982 __instr_hdr_emit8_tx_exec(p, t, ip);
1985 thread_ip_reset(p, t);
1993 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1994 struct action *action __rte_unused,
1997 struct instruction *instr,
1998 struct instruction_data *data __rte_unused)
2002 CHECK(n_tokens == 2, EINVAL);
2004 h = header_parse(p, tokens[1]);
2007 instr->type = INSTR_HDR_VALIDATE;
2008 instr->valid.header_id = h->id;
2013 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2015 struct thread *t = &p->threads[p->thread_id];
2016 struct instruction *ip = t->ip;
2018 __instr_hdr_validate_exec(p, t, ip);
2028 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2029 struct action *action __rte_unused,
2032 struct instruction *instr,
2033 struct instruction_data *data __rte_unused)
2037 CHECK(n_tokens == 2, EINVAL);
2039 h = header_parse(p, tokens[1]);
2042 instr->type = INSTR_HDR_INVALIDATE;
2043 instr->valid.header_id = h->id;
2048 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2050 struct thread *t = &p->threads[p->thread_id];
2051 struct instruction *ip = t->ip;
2053 __instr_hdr_invalidate_exec(p, t, ip);
2062 static struct table *
2063 table_find(struct rte_swx_pipeline *p, const char *name);
2065 static struct selector *
2066 selector_find(struct rte_swx_pipeline *p, const char *name);
2068 static struct learner *
2069 learner_find(struct rte_swx_pipeline *p, const char *name);
2072 instr_table_translate(struct rte_swx_pipeline *p,
2073 struct action *action,
2076 struct instruction *instr,
2077 struct instruction_data *data __rte_unused)
2083 CHECK(!action, EINVAL);
2084 CHECK(n_tokens == 2, EINVAL);
2086 t = table_find(p, tokens[1]);
2088 instr->type = INSTR_TABLE;
2089 instr->table.table_id = t->id;
2093 s = selector_find(p, tokens[1]);
2095 instr->type = INSTR_SELECTOR;
2096 instr->table.table_id = s->id;
2100 l = learner_find(p, tokens[1]);
2102 instr->type = INSTR_LEARNER;
2103 instr->table.table_id = l->id;
2111 instr_table_exec(struct rte_swx_pipeline *p)
2113 struct thread *t = &p->threads[p->thread_id];
2114 struct instruction *ip = t->ip;
2115 uint32_t table_id = ip->table.table_id;
2116 struct rte_swx_table_state *ts = &t->table_state[table_id];
2117 struct table_runtime *table = &t->tables[table_id];
2118 struct table_statistics *stats = &p->table_stats[table_id];
2119 uint64_t action_id, n_pkts_hit, n_pkts_action;
2120 uint8_t *action_data;
2124 done = table->func(ts->obj,
2132 TRACE("[Thread %2u] table %u (not finalized)\n",
2140 action_id = hit ? action_id : ts->default_action_id;
2141 action_data = hit ? action_data : ts->default_action_data;
2142 n_pkts_hit = stats->n_pkts_hit[hit];
2143 n_pkts_action = stats->n_pkts_action[action_id];
2145 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2148 hit ? "hit" : "miss",
2149 (uint32_t)action_id);
2151 t->action_id = action_id;
2152 t->structs[0] = action_data;
2154 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2155 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2158 thread_ip_action_call(p, t, action_id);
2162 instr_table_af_exec(struct rte_swx_pipeline *p)
2164 struct thread *t = &p->threads[p->thread_id];
2165 struct instruction *ip = t->ip;
2166 uint32_t table_id = ip->table.table_id;
2167 struct rte_swx_table_state *ts = &t->table_state[table_id];
2168 struct table_runtime *table = &t->tables[table_id];
2169 struct table_statistics *stats = &p->table_stats[table_id];
2170 uint64_t action_id, n_pkts_hit, n_pkts_action;
2171 uint8_t *action_data;
2172 action_func_t action_func;
2176 done = table->func(ts->obj,
2184 TRACE("[Thread %2u] table %u (not finalized)\n",
2192 action_id = hit ? action_id : ts->default_action_id;
2193 action_data = hit ? action_data : ts->default_action_data;
2194 action_func = p->action_funcs[action_id];
2195 n_pkts_hit = stats->n_pkts_hit[hit];
2196 n_pkts_action = stats->n_pkts_action[action_id];
2198 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2201 hit ? "hit" : "miss",
2202 (uint32_t)action_id);
2204 t->action_id = action_id;
2205 t->structs[0] = action_data;
2207 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2208 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2218 instr_selector_exec(struct rte_swx_pipeline *p)
2220 struct thread *t = &p->threads[p->thread_id];
2221 struct instruction *ip = t->ip;
2222 uint32_t selector_id = ip->table.table_id;
2223 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2224 struct selector_runtime *selector = &t->selectors[selector_id];
2225 struct selector_statistics *stats = &p->selector_stats[selector_id];
2226 uint64_t n_pkts = stats->n_pkts;
2230 done = rte_swx_table_selector_select(ts->obj,
2232 selector->group_id_buffer,
2233 selector->selector_buffer,
2234 selector->member_id_buffer);
2237 TRACE("[Thread %2u] selector %u (not finalized)\n",
2246 TRACE("[Thread %2u] selector %u\n",
2250 stats->n_pkts = n_pkts + 1;
2257 instr_learner_exec(struct rte_swx_pipeline *p)
2259 struct thread *t = &p->threads[p->thread_id];
2260 struct instruction *ip = t->ip;
2261 uint32_t learner_id = ip->table.table_id;
2262 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2263 p->n_selectors + learner_id];
2264 struct learner_runtime *l = &t->learners[learner_id];
2265 struct learner_statistics *stats = &p->learner_stats[learner_id];
2266 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2267 uint8_t *action_data;
2271 time = rte_get_tsc_cycles();
2273 done = rte_swx_table_learner_lookup(ts->obj,
2282 TRACE("[Thread %2u] learner %u (not finalized)\n",
2290 action_id = hit ? action_id : ts->default_action_id;
2291 action_data = hit ? action_data : ts->default_action_data;
2292 n_pkts_hit = stats->n_pkts_hit[hit];
2293 n_pkts_action = stats->n_pkts_action[action_id];
2295 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2298 hit ? "hit" : "miss",
2299 (uint32_t)action_id);
2301 t->action_id = action_id;
2302 t->structs[0] = action_data;
2304 t->learner_id = learner_id;
2306 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2307 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2310 thread_ip_action_call(p, t, action_id);
2314 instr_learner_af_exec(struct rte_swx_pipeline *p)
2316 struct thread *t = &p->threads[p->thread_id];
2317 struct instruction *ip = t->ip;
2318 uint32_t learner_id = ip->table.table_id;
2319 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2320 p->n_selectors + learner_id];
2321 struct learner_runtime *l = &t->learners[learner_id];
2322 struct learner_statistics *stats = &p->learner_stats[learner_id];
2323 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2324 uint8_t *action_data;
2325 action_func_t action_func;
2329 time = rte_get_tsc_cycles();
2331 done = rte_swx_table_learner_lookup(ts->obj,
2340 TRACE("[Thread %2u] learner %u (not finalized)\n",
2348 action_id = hit ? action_id : ts->default_action_id;
2349 action_data = hit ? action_data : ts->default_action_data;
2350 action_func = p->action_funcs[action_id];
2351 n_pkts_hit = stats->n_pkts_hit[hit];
2352 n_pkts_action = stats->n_pkts_action[action_id];
2354 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2357 hit ? "hit" : "miss",
2358 (uint32_t)action_id);
2360 t->action_id = action_id;
2361 t->structs[0] = action_data;
2363 t->learner_id = learner_id;
2365 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2366 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2369 thread_ip_action_call(p, t, action_id);
2378 static struct action *
2379 action_find(struct rte_swx_pipeline *p, const char *name);
2382 action_has_nbo_args(struct action *a);
2385 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2388 instr_learn_translate(struct rte_swx_pipeline *p,
2389 struct action *action,
2392 struct instruction *instr,
2393 struct instruction_data *data __rte_unused)
2396 const char *mf_name;
2397 uint32_t mf_offset = 0;
2399 CHECK(action, EINVAL);
2400 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2402 a = action_find(p, tokens[1]);
2404 CHECK(!action_has_nbo_args(a), EINVAL);
2406 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2407 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2412 mf = metadata_field_parse(p, mf_name);
2415 mf_offset = mf->offset / 8;
2418 instr->type = INSTR_LEARNER_LEARN;
2419 instr->learn.action_id = a->id;
2420 instr->learn.mf_offset = mf_offset;
2426 instr_learn_exec(struct rte_swx_pipeline *p)
2428 struct thread *t = &p->threads[p->thread_id];
2429 struct instruction *ip = t->ip;
2431 __instr_learn_exec(p, t, ip);
2441 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2442 struct action *action,
2443 char **tokens __rte_unused,
2445 struct instruction *instr,
2446 struct instruction_data *data __rte_unused)
2448 CHECK(action, EINVAL);
2449 CHECK(n_tokens == 1, EINVAL);
2451 instr->type = INSTR_LEARNER_FORGET;
2457 instr_forget_exec(struct rte_swx_pipeline *p)
2459 struct thread *t = &p->threads[p->thread_id];
2460 struct instruction *ip = t->ip;
2462 __instr_forget_exec(p, t, ip);
2472 instr_extern_translate(struct rte_swx_pipeline *p,
2473 struct action *action __rte_unused,
2476 struct instruction *instr,
2477 struct instruction_data *data __rte_unused)
2479 char *token = tokens[1];
2481 CHECK(n_tokens == 2, EINVAL);
2483 if (token[0] == 'e') {
2484 struct extern_obj *obj;
2485 struct extern_type_member_func *func;
2487 func = extern_obj_member_func_parse(p, token, &obj);
2488 CHECK(func, EINVAL);
2490 instr->type = INSTR_EXTERN_OBJ;
2491 instr->ext_obj.ext_obj_id = obj->id;
2492 instr->ext_obj.func_id = func->id;
2497 if (token[0] == 'f') {
2498 struct extern_func *func;
2500 func = extern_func_parse(p, token);
2501 CHECK(func, EINVAL);
2503 instr->type = INSTR_EXTERN_FUNC;
2504 instr->ext_func.ext_func_id = func->id;
2513 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2515 struct thread *t = &p->threads[p->thread_id];
2516 struct instruction *ip = t->ip;
2519 /* Extern object member function execute. */
2520 done = __instr_extern_obj_exec(p, t, ip);
2523 thread_ip_inc_cond(t, done);
2524 thread_yield_cond(p, done ^ 1);
2528 instr_extern_func_exec(struct rte_swx_pipeline *p)
2530 struct thread *t = &p->threads[p->thread_id];
2531 struct instruction *ip = t->ip;
2534 /* Extern function execute. */
2535 done = __instr_extern_func_exec(p, t, ip);
2538 thread_ip_inc_cond(t, done);
2539 thread_yield_cond(p, done ^ 1);
2546 instr_mov_translate(struct rte_swx_pipeline *p,
2547 struct action *action,
2550 struct instruction *instr,
2551 struct instruction_data *data __rte_unused)
2553 char *dst = tokens[1], *src = tokens[2];
2554 struct field *fdst, *fsrc;
2556 uint32_t dst_struct_id = 0, src_struct_id = 0;
2558 CHECK(n_tokens == 3, EINVAL);
2560 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2561 CHECK(fdst, EINVAL);
2562 CHECK(!fdst->var_size, EINVAL);
2564 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2565 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2567 CHECK(!fsrc->var_size, EINVAL);
2569 instr->type = INSTR_MOV;
2570 if (dst[0] != 'h' && src[0] == 'h')
2571 instr->type = INSTR_MOV_MH;
2572 if (dst[0] == 'h' && src[0] != 'h')
2573 instr->type = INSTR_MOV_HM;
2574 if (dst[0] == 'h' && src[0] == 'h')
2575 instr->type = INSTR_MOV_HH;
2577 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2578 instr->mov.dst.n_bits = fdst->n_bits;
2579 instr->mov.dst.offset = fdst->offset / 8;
2580 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2581 instr->mov.src.n_bits = fsrc->n_bits;
2582 instr->mov.src.offset = fsrc->offset / 8;
2587 src_val = strtoull(src, &src, 0);
2588 CHECK(!src[0], EINVAL);
2591 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2593 instr->type = INSTR_MOV_I;
2594 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2595 instr->mov.dst.n_bits = fdst->n_bits;
2596 instr->mov.dst.offset = fdst->offset / 8;
2597 instr->mov.src_val = src_val;
2602 instr_mov_exec(struct rte_swx_pipeline *p)
2604 struct thread *t = &p->threads[p->thread_id];
2605 struct instruction *ip = t->ip;
2607 __instr_mov_exec(p, t, ip);
2614 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2616 struct thread *t = &p->threads[p->thread_id];
2617 struct instruction *ip = t->ip;
2619 __instr_mov_mh_exec(p, t, ip);
2626 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2628 struct thread *t = &p->threads[p->thread_id];
2629 struct instruction *ip = t->ip;
2631 __instr_mov_hm_exec(p, t, ip);
2638 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2640 struct thread *t = &p->threads[p->thread_id];
2641 struct instruction *ip = t->ip;
2643 __instr_mov_hh_exec(p, t, ip);
2650 instr_mov_i_exec(struct rte_swx_pipeline *p)
2652 struct thread *t = &p->threads[p->thread_id];
2653 struct instruction *ip = t->ip;
2655 __instr_mov_i_exec(p, t, ip);
2665 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2667 struct thread *t = &p->threads[p->thread_id];
2668 struct instruction *ip = t->ip;
2670 __instr_dma_ht_exec(p, t, ip);
2677 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2679 struct thread *t = &p->threads[p->thread_id];
2680 struct instruction *ip = t->ip;
2682 __instr_dma_ht2_exec(p, t, ip);
2689 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2691 struct thread *t = &p->threads[p->thread_id];
2692 struct instruction *ip = t->ip;
2694 __instr_dma_ht3_exec(p, t, ip);
2701 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2703 struct thread *t = &p->threads[p->thread_id];
2704 struct instruction *ip = t->ip;
2706 __instr_dma_ht4_exec(p, t, ip);
2713 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2715 struct thread *t = &p->threads[p->thread_id];
2716 struct instruction *ip = t->ip;
2718 __instr_dma_ht5_exec(p, t, ip);
2725 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2727 struct thread *t = &p->threads[p->thread_id];
2728 struct instruction *ip = t->ip;
2730 __instr_dma_ht6_exec(p, t, ip);
2737 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2739 struct thread *t = &p->threads[p->thread_id];
2740 struct instruction *ip = t->ip;
2742 __instr_dma_ht7_exec(p, t, ip);
2749 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2751 struct thread *t = &p->threads[p->thread_id];
2752 struct instruction *ip = t->ip;
2754 __instr_dma_ht8_exec(p, t, ip);
2764 instr_alu_add_translate(struct rte_swx_pipeline *p,
2765 struct action *action,
2768 struct instruction *instr,
2769 struct instruction_data *data __rte_unused)
2771 char *dst = tokens[1], *src = tokens[2];
2772 struct field *fdst, *fsrc;
2774 uint32_t dst_struct_id = 0, src_struct_id = 0;
2776 CHECK(n_tokens == 3, EINVAL);
2778 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2779 CHECK(fdst, EINVAL);
2780 CHECK(!fdst->var_size, EINVAL);
2782 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2783 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2785 CHECK(!fsrc->var_size, EINVAL);
2787 instr->type = INSTR_ALU_ADD;
2788 if (dst[0] == 'h' && src[0] != 'h')
2789 instr->type = INSTR_ALU_ADD_HM;
2790 if (dst[0] != 'h' && src[0] == 'h')
2791 instr->type = INSTR_ALU_ADD_MH;
2792 if (dst[0] == 'h' && src[0] == 'h')
2793 instr->type = INSTR_ALU_ADD_HH;
2795 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2796 instr->alu.dst.n_bits = fdst->n_bits;
2797 instr->alu.dst.offset = fdst->offset / 8;
2798 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2799 instr->alu.src.n_bits = fsrc->n_bits;
2800 instr->alu.src.offset = fsrc->offset / 8;
2804 /* ADD_MI, ADD_HI. */
2805 src_val = strtoull(src, &src, 0);
2806 CHECK(!src[0], EINVAL);
2808 instr->type = INSTR_ALU_ADD_MI;
2810 instr->type = INSTR_ALU_ADD_HI;
2812 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2813 instr->alu.dst.n_bits = fdst->n_bits;
2814 instr->alu.dst.offset = fdst->offset / 8;
2815 instr->alu.src_val = src_val;
2820 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2821 struct action *action,
2824 struct instruction *instr,
2825 struct instruction_data *data __rte_unused)
2827 char *dst = tokens[1], *src = tokens[2];
2828 struct field *fdst, *fsrc;
2830 uint32_t dst_struct_id = 0, src_struct_id = 0;
2832 CHECK(n_tokens == 3, EINVAL);
2834 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2835 CHECK(fdst, EINVAL);
2836 CHECK(!fdst->var_size, EINVAL);
2838 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2839 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2841 CHECK(!fsrc->var_size, EINVAL);
2843 instr->type = INSTR_ALU_SUB;
2844 if (dst[0] == 'h' && src[0] != 'h')
2845 instr->type = INSTR_ALU_SUB_HM;
2846 if (dst[0] != 'h' && src[0] == 'h')
2847 instr->type = INSTR_ALU_SUB_MH;
2848 if (dst[0] == 'h' && src[0] == 'h')
2849 instr->type = INSTR_ALU_SUB_HH;
2851 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2852 instr->alu.dst.n_bits = fdst->n_bits;
2853 instr->alu.dst.offset = fdst->offset / 8;
2854 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2855 instr->alu.src.n_bits = fsrc->n_bits;
2856 instr->alu.src.offset = fsrc->offset / 8;
2860 /* SUB_MI, SUB_HI. */
2861 src_val = strtoull(src, &src, 0);
2862 CHECK(!src[0], EINVAL);
2864 instr->type = INSTR_ALU_SUB_MI;
2866 instr->type = INSTR_ALU_SUB_HI;
2868 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2869 instr->alu.dst.n_bits = fdst->n_bits;
2870 instr->alu.dst.offset = fdst->offset / 8;
2871 instr->alu.src_val = src_val;
2876 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2877 struct action *action __rte_unused,
2880 struct instruction *instr,
2881 struct instruction_data *data __rte_unused)
2883 char *dst = tokens[1], *src = tokens[2];
2884 struct header *hdst, *hsrc;
2885 struct field *fdst, *fsrc;
2887 CHECK(n_tokens == 3, EINVAL);
2889 fdst = header_field_parse(p, dst, &hdst);
2890 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2891 CHECK(!fdst->var_size, EINVAL);
2894 fsrc = header_field_parse(p, src, &hsrc);
2896 CHECK(!fsrc->var_size, EINVAL);
2898 instr->type = INSTR_ALU_CKADD_FIELD;
2899 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2900 instr->alu.dst.n_bits = fdst->n_bits;
2901 instr->alu.dst.offset = fdst->offset / 8;
2902 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2903 instr->alu.src.n_bits = fsrc->n_bits;
2904 instr->alu.src.offset = fsrc->offset / 8;
2908 /* CKADD_STRUCT, CKADD_STRUCT20. */
2909 hsrc = header_parse(p, src);
2910 CHECK(hsrc, EINVAL);
2911 CHECK(!hsrc->st->var_size, EINVAL);
2913 instr->type = INSTR_ALU_CKADD_STRUCT;
2914 if ((hsrc->st->n_bits / 8) == 20)
2915 instr->type = INSTR_ALU_CKADD_STRUCT20;
2917 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2918 instr->alu.dst.n_bits = fdst->n_bits;
2919 instr->alu.dst.offset = fdst->offset / 8;
2920 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2921 instr->alu.src.n_bits = hsrc->st->n_bits;
2922 instr->alu.src.offset = 0; /* Unused. */
2927 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2928 struct action *action __rte_unused,
2931 struct instruction *instr,
2932 struct instruction_data *data __rte_unused)
2934 char *dst = tokens[1], *src = tokens[2];
2935 struct header *hdst, *hsrc;
2936 struct field *fdst, *fsrc;
2938 CHECK(n_tokens == 3, EINVAL);
2940 fdst = header_field_parse(p, dst, &hdst);
2941 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2942 CHECK(!fdst->var_size, EINVAL);
2944 fsrc = header_field_parse(p, src, &hsrc);
2945 CHECK(fsrc, EINVAL);
2946 CHECK(!fsrc->var_size, EINVAL);
2948 instr->type = INSTR_ALU_CKSUB_FIELD;
2949 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2950 instr->alu.dst.n_bits = fdst->n_bits;
2951 instr->alu.dst.offset = fdst->offset / 8;
2952 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2953 instr->alu.src.n_bits = fsrc->n_bits;
2954 instr->alu.src.offset = fsrc->offset / 8;
2959 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2960 struct action *action,
2963 struct instruction *instr,
2964 struct instruction_data *data __rte_unused)
2966 char *dst = tokens[1], *src = tokens[2];
2967 struct field *fdst, *fsrc;
2969 uint32_t dst_struct_id = 0, src_struct_id = 0;
2971 CHECK(n_tokens == 3, EINVAL);
2973 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2974 CHECK(fdst, EINVAL);
2975 CHECK(!fdst->var_size, EINVAL);
2977 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2978 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2980 CHECK(!fsrc->var_size, EINVAL);
2982 instr->type = INSTR_ALU_SHL;
2983 if (dst[0] == 'h' && src[0] != 'h')
2984 instr->type = INSTR_ALU_SHL_HM;
2985 if (dst[0] != 'h' && src[0] == 'h')
2986 instr->type = INSTR_ALU_SHL_MH;
2987 if (dst[0] == 'h' && src[0] == 'h')
2988 instr->type = INSTR_ALU_SHL_HH;
2990 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2991 instr->alu.dst.n_bits = fdst->n_bits;
2992 instr->alu.dst.offset = fdst->offset / 8;
2993 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2994 instr->alu.src.n_bits = fsrc->n_bits;
2995 instr->alu.src.offset = fsrc->offset / 8;
2999 /* SHL_MI, SHL_HI. */
3000 src_val = strtoull(src, &src, 0);
3001 CHECK(!src[0], EINVAL);
3003 instr->type = INSTR_ALU_SHL_MI;
3005 instr->type = INSTR_ALU_SHL_HI;
3007 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3008 instr->alu.dst.n_bits = fdst->n_bits;
3009 instr->alu.dst.offset = fdst->offset / 8;
3010 instr->alu.src_val = src_val;
3015 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3016 struct action *action,
3019 struct instruction *instr,
3020 struct instruction_data *data __rte_unused)
3022 char *dst = tokens[1], *src = tokens[2];
3023 struct field *fdst, *fsrc;
3025 uint32_t dst_struct_id = 0, src_struct_id = 0;
3027 CHECK(n_tokens == 3, EINVAL);
3029 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3030 CHECK(fdst, EINVAL);
3031 CHECK(!fdst->var_size, EINVAL);
3033 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3034 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3036 CHECK(!fsrc->var_size, EINVAL);
3038 instr->type = INSTR_ALU_SHR;
3039 if (dst[0] == 'h' && src[0] != 'h')
3040 instr->type = INSTR_ALU_SHR_HM;
3041 if (dst[0] != 'h' && src[0] == 'h')
3042 instr->type = INSTR_ALU_SHR_MH;
3043 if (dst[0] == 'h' && src[0] == 'h')
3044 instr->type = INSTR_ALU_SHR_HH;
3046 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3047 instr->alu.dst.n_bits = fdst->n_bits;
3048 instr->alu.dst.offset = fdst->offset / 8;
3049 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3050 instr->alu.src.n_bits = fsrc->n_bits;
3051 instr->alu.src.offset = fsrc->offset / 8;
3055 /* SHR_MI, SHR_HI. */
3056 src_val = strtoull(src, &src, 0);
3057 CHECK(!src[0], EINVAL);
3059 instr->type = INSTR_ALU_SHR_MI;
3061 instr->type = INSTR_ALU_SHR_HI;
3063 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3064 instr->alu.dst.n_bits = fdst->n_bits;
3065 instr->alu.dst.offset = fdst->offset / 8;
3066 instr->alu.src_val = src_val;
3071 instr_alu_and_translate(struct rte_swx_pipeline *p,
3072 struct action *action,
3075 struct instruction *instr,
3076 struct instruction_data *data __rte_unused)
3078 char *dst = tokens[1], *src = tokens[2];
3079 struct field *fdst, *fsrc;
3081 uint32_t dst_struct_id = 0, src_struct_id = 0;
3083 CHECK(n_tokens == 3, EINVAL);
3085 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3086 CHECK(fdst, EINVAL);
3087 CHECK(!fdst->var_size, EINVAL);
3089 /* AND, AND_MH, AND_HM, AND_HH. */
3090 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3092 CHECK(!fsrc->var_size, EINVAL);
3094 instr->type = INSTR_ALU_AND;
3095 if (dst[0] != 'h' && src[0] == 'h')
3096 instr->type = INSTR_ALU_AND_MH;
3097 if (dst[0] == 'h' && src[0] != 'h')
3098 instr->type = INSTR_ALU_AND_HM;
3099 if (dst[0] == 'h' && src[0] == 'h')
3100 instr->type = INSTR_ALU_AND_HH;
3102 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3103 instr->alu.dst.n_bits = fdst->n_bits;
3104 instr->alu.dst.offset = fdst->offset / 8;
3105 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3106 instr->alu.src.n_bits = fsrc->n_bits;
3107 instr->alu.src.offset = fsrc->offset / 8;
3112 src_val = strtoull(src, &src, 0);
3113 CHECK(!src[0], EINVAL);
3116 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3118 instr->type = INSTR_ALU_AND_I;
3119 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3120 instr->alu.dst.n_bits = fdst->n_bits;
3121 instr->alu.dst.offset = fdst->offset / 8;
3122 instr->alu.src_val = src_val;
3127 instr_alu_or_translate(struct rte_swx_pipeline *p,
3128 struct action *action,
3131 struct instruction *instr,
3132 struct instruction_data *data __rte_unused)
3134 char *dst = tokens[1], *src = tokens[2];
3135 struct field *fdst, *fsrc;
3137 uint32_t dst_struct_id = 0, src_struct_id = 0;
3139 CHECK(n_tokens == 3, EINVAL);
3141 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3142 CHECK(fdst, EINVAL);
3143 CHECK(!fdst->var_size, EINVAL);
3145 /* OR, OR_MH, OR_HM, OR_HH. */
3146 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3148 CHECK(!fsrc->var_size, EINVAL);
3150 instr->type = INSTR_ALU_OR;
3151 if (dst[0] != 'h' && src[0] == 'h')
3152 instr->type = INSTR_ALU_OR_MH;
3153 if (dst[0] == 'h' && src[0] != 'h')
3154 instr->type = INSTR_ALU_OR_HM;
3155 if (dst[0] == 'h' && src[0] == 'h')
3156 instr->type = INSTR_ALU_OR_HH;
3158 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3159 instr->alu.dst.n_bits = fdst->n_bits;
3160 instr->alu.dst.offset = fdst->offset / 8;
3161 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3162 instr->alu.src.n_bits = fsrc->n_bits;
3163 instr->alu.src.offset = fsrc->offset / 8;
3168 src_val = strtoull(src, &src, 0);
3169 CHECK(!src[0], EINVAL);
3172 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3174 instr->type = INSTR_ALU_OR_I;
3175 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3176 instr->alu.dst.n_bits = fdst->n_bits;
3177 instr->alu.dst.offset = fdst->offset / 8;
3178 instr->alu.src_val = src_val;
3183 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3184 struct action *action,
3187 struct instruction *instr,
3188 struct instruction_data *data __rte_unused)
3190 char *dst = tokens[1], *src = tokens[2];
3191 struct field *fdst, *fsrc;
3193 uint32_t dst_struct_id = 0, src_struct_id = 0;
3195 CHECK(n_tokens == 3, EINVAL);
3197 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3198 CHECK(fdst, EINVAL);
3199 CHECK(!fdst->var_size, EINVAL);
3201 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3202 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3204 CHECK(!fsrc->var_size, EINVAL);
3206 instr->type = INSTR_ALU_XOR;
3207 if (dst[0] != 'h' && src[0] == 'h')
3208 instr->type = INSTR_ALU_XOR_MH;
3209 if (dst[0] == 'h' && src[0] != 'h')
3210 instr->type = INSTR_ALU_XOR_HM;
3211 if (dst[0] == 'h' && src[0] == 'h')
3212 instr->type = INSTR_ALU_XOR_HH;
3214 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3215 instr->alu.dst.n_bits = fdst->n_bits;
3216 instr->alu.dst.offset = fdst->offset / 8;
3217 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3218 instr->alu.src.n_bits = fsrc->n_bits;
3219 instr->alu.src.offset = fsrc->offset / 8;
3224 src_val = strtoull(src, &src, 0);
3225 CHECK(!src[0], EINVAL);
3228 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3230 instr->type = INSTR_ALU_XOR_I;
3231 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3232 instr->alu.dst.n_bits = fdst->n_bits;
3233 instr->alu.dst.offset = fdst->offset / 8;
3234 instr->alu.src_val = src_val;
3239 instr_alu_add_exec(struct rte_swx_pipeline *p)
3241 struct thread *t = &p->threads[p->thread_id];
3242 struct instruction *ip = t->ip;
3245 __instr_alu_add_exec(p, t, ip);
3252 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3254 struct thread *t = &p->threads[p->thread_id];
3255 struct instruction *ip = t->ip;
3258 __instr_alu_add_mh_exec(p, t, ip);
3265 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3267 struct thread *t = &p->threads[p->thread_id];
3268 struct instruction *ip = t->ip;
3271 __instr_alu_add_hm_exec(p, t, ip);
3278 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3280 struct thread *t = &p->threads[p->thread_id];
3281 struct instruction *ip = t->ip;
3284 __instr_alu_add_hh_exec(p, t, ip);
3291 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3293 struct thread *t = &p->threads[p->thread_id];
3294 struct instruction *ip = t->ip;
3297 __instr_alu_add_mi_exec(p, t, ip);
3304 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3306 struct thread *t = &p->threads[p->thread_id];
3307 struct instruction *ip = t->ip;
3310 __instr_alu_add_hi_exec(p, t, ip);
3317 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3319 struct thread *t = &p->threads[p->thread_id];
3320 struct instruction *ip = t->ip;
3323 __instr_alu_sub_exec(p, t, ip);
3330 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3332 struct thread *t = &p->threads[p->thread_id];
3333 struct instruction *ip = t->ip;
3336 __instr_alu_sub_mh_exec(p, t, ip);
3343 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3345 struct thread *t = &p->threads[p->thread_id];
3346 struct instruction *ip = t->ip;
3349 __instr_alu_sub_hm_exec(p, t, ip);
3356 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3358 struct thread *t = &p->threads[p->thread_id];
3359 struct instruction *ip = t->ip;
3362 __instr_alu_sub_hh_exec(p, t, ip);
3369 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3371 struct thread *t = &p->threads[p->thread_id];
3372 struct instruction *ip = t->ip;
3375 __instr_alu_sub_mi_exec(p, t, ip);
3382 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3384 struct thread *t = &p->threads[p->thread_id];
3385 struct instruction *ip = t->ip;
3388 __instr_alu_sub_hi_exec(p, t, ip);
3395 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3397 struct thread *t = &p->threads[p->thread_id];
3398 struct instruction *ip = t->ip;
3401 __instr_alu_shl_exec(p, t, ip);
3408 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3410 struct thread *t = &p->threads[p->thread_id];
3411 struct instruction *ip = t->ip;
3414 __instr_alu_shl_mh_exec(p, t, ip);
3421 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3423 struct thread *t = &p->threads[p->thread_id];
3424 struct instruction *ip = t->ip;
3427 __instr_alu_shl_hm_exec(p, t, ip);
3434 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3436 struct thread *t = &p->threads[p->thread_id];
3437 struct instruction *ip = t->ip;
3440 __instr_alu_shl_hh_exec(p, t, ip);
3447 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3449 struct thread *t = &p->threads[p->thread_id];
3450 struct instruction *ip = t->ip;
3453 __instr_alu_shl_mi_exec(p, t, ip);
3460 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3462 struct thread *t = &p->threads[p->thread_id];
3463 struct instruction *ip = t->ip;
3466 __instr_alu_shl_hi_exec(p, t, ip);
3473 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3475 struct thread *t = &p->threads[p->thread_id];
3476 struct instruction *ip = t->ip;
3479 __instr_alu_shr_exec(p, t, ip);
3486 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3488 struct thread *t = &p->threads[p->thread_id];
3489 struct instruction *ip = t->ip;
3492 __instr_alu_shr_mh_exec(p, t, ip);
3499 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3501 struct thread *t = &p->threads[p->thread_id];
3502 struct instruction *ip = t->ip;
3505 __instr_alu_shr_hm_exec(p, t, ip);
3512 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3514 struct thread *t = &p->threads[p->thread_id];
3515 struct instruction *ip = t->ip;
3518 __instr_alu_shr_hh_exec(p, t, ip);
3525 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3527 struct thread *t = &p->threads[p->thread_id];
3528 struct instruction *ip = t->ip;
3531 __instr_alu_shr_mi_exec(p, t, ip);
3538 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3540 struct thread *t = &p->threads[p->thread_id];
3541 struct instruction *ip = t->ip;
3544 __instr_alu_shr_hi_exec(p, t, ip);
3551 instr_alu_and_exec(struct rte_swx_pipeline *p)
3553 struct thread *t = &p->threads[p->thread_id];
3554 struct instruction *ip = t->ip;
3557 __instr_alu_and_exec(p, t, ip);
3564 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3566 struct thread *t = &p->threads[p->thread_id];
3567 struct instruction *ip = t->ip;
3570 __instr_alu_and_mh_exec(p, t, ip);
3577 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3579 struct thread *t = &p->threads[p->thread_id];
3580 struct instruction *ip = t->ip;
3583 __instr_alu_and_hm_exec(p, t, ip);
3590 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3592 struct thread *t = &p->threads[p->thread_id];
3593 struct instruction *ip = t->ip;
3596 __instr_alu_and_hh_exec(p, t, ip);
3603 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3605 struct thread *t = &p->threads[p->thread_id];
3606 struct instruction *ip = t->ip;
3609 __instr_alu_and_i_exec(p, t, ip);
3616 instr_alu_or_exec(struct rte_swx_pipeline *p)
3618 struct thread *t = &p->threads[p->thread_id];
3619 struct instruction *ip = t->ip;
3622 __instr_alu_or_exec(p, t, ip);
3629 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3631 struct thread *t = &p->threads[p->thread_id];
3632 struct instruction *ip = t->ip;
3635 __instr_alu_or_mh_exec(p, t, ip);
3642 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3644 struct thread *t = &p->threads[p->thread_id];
3645 struct instruction *ip = t->ip;
3648 __instr_alu_or_hm_exec(p, t, ip);
3655 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3657 struct thread *t = &p->threads[p->thread_id];
3658 struct instruction *ip = t->ip;
3661 __instr_alu_or_hh_exec(p, t, ip);
3668 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3670 struct thread *t = &p->threads[p->thread_id];
3671 struct instruction *ip = t->ip;
3674 __instr_alu_or_i_exec(p, t, ip);
3681 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3683 struct thread *t = &p->threads[p->thread_id];
3684 struct instruction *ip = t->ip;
3687 __instr_alu_xor_exec(p, t, ip);
3694 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3696 struct thread *t = &p->threads[p->thread_id];
3697 struct instruction *ip = t->ip;
3700 __instr_alu_xor_mh_exec(p, t, ip);
3707 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3709 struct thread *t = &p->threads[p->thread_id];
3710 struct instruction *ip = t->ip;
3713 __instr_alu_xor_hm_exec(p, t, ip);
3720 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3722 struct thread *t = &p->threads[p->thread_id];
3723 struct instruction *ip = t->ip;
3726 __instr_alu_xor_hh_exec(p, t, ip);
3733 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3735 struct thread *t = &p->threads[p->thread_id];
3736 struct instruction *ip = t->ip;
3739 __instr_alu_xor_i_exec(p, t, ip);
3746 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3748 struct thread *t = &p->threads[p->thread_id];
3749 struct instruction *ip = t->ip;
3752 __instr_alu_ckadd_field_exec(p, t, ip);
3759 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3761 struct thread *t = &p->threads[p->thread_id];
3762 struct instruction *ip = t->ip;
3765 __instr_alu_cksub_field_exec(p, t, ip);
3772 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3774 struct thread *t = &p->threads[p->thread_id];
3775 struct instruction *ip = t->ip;
3778 __instr_alu_ckadd_struct20_exec(p, t, ip);
3785 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3787 struct thread *t = &p->threads[p->thread_id];
3788 struct instruction *ip = t->ip;
3791 __instr_alu_ckadd_struct_exec(p, t, ip);
3800 static struct regarray *
3801 regarray_find(struct rte_swx_pipeline *p, const char *name);
3804 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3805 struct action *action,
3808 struct instruction *instr,
3809 struct instruction_data *data __rte_unused)
3811 char *regarray = tokens[1], *idx = tokens[2];
3814 uint32_t idx_struct_id, idx_val;
3816 CHECK(n_tokens == 3, EINVAL);
3818 r = regarray_find(p, regarray);
3821 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3822 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3824 CHECK(!fidx->var_size, EINVAL);
3826 instr->type = INSTR_REGPREFETCH_RM;
3828 instr->type = INSTR_REGPREFETCH_RH;
3830 instr->regarray.regarray_id = r->id;
3831 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3832 instr->regarray.idx.n_bits = fidx->n_bits;
3833 instr->regarray.idx.offset = fidx->offset / 8;
3834 instr->regarray.dstsrc_val = 0; /* Unused. */
3838 /* REGPREFETCH_RI. */
3839 idx_val = strtoul(idx, &idx, 0);
3840 CHECK(!idx[0], EINVAL);
3842 instr->type = INSTR_REGPREFETCH_RI;
3843 instr->regarray.regarray_id = r->id;
3844 instr->regarray.idx_val = idx_val;
3845 instr->regarray.dstsrc_val = 0; /* Unused. */
3850 instr_regrd_translate(struct rte_swx_pipeline *p,
3851 struct action *action,
3854 struct instruction *instr,
3855 struct instruction_data *data __rte_unused)
3857 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3859 struct field *fdst, *fidx;
3860 uint32_t dst_struct_id, idx_struct_id, idx_val;
3862 CHECK(n_tokens == 4, EINVAL);
3864 r = regarray_find(p, regarray);
3867 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3868 CHECK(fdst, EINVAL);
3869 CHECK(!fdst->var_size, EINVAL);
3871 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3872 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3874 CHECK(!fidx->var_size, EINVAL);
3876 instr->type = INSTR_REGRD_MRM;
3877 if (dst[0] == 'h' && idx[0] != 'h')
3878 instr->type = INSTR_REGRD_HRM;
3879 if (dst[0] != 'h' && idx[0] == 'h')
3880 instr->type = INSTR_REGRD_MRH;
3881 if (dst[0] == 'h' && idx[0] == 'h')
3882 instr->type = INSTR_REGRD_HRH;
3884 instr->regarray.regarray_id = r->id;
3885 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3886 instr->regarray.idx.n_bits = fidx->n_bits;
3887 instr->regarray.idx.offset = fidx->offset / 8;
3888 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3889 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3890 instr->regarray.dstsrc.offset = fdst->offset / 8;
3894 /* REGRD_MRI, REGRD_HRI. */
3895 idx_val = strtoul(idx, &idx, 0);
3896 CHECK(!idx[0], EINVAL);
3898 instr->type = INSTR_REGRD_MRI;
3900 instr->type = INSTR_REGRD_HRI;
3902 instr->regarray.regarray_id = r->id;
3903 instr->regarray.idx_val = idx_val;
3904 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3905 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3906 instr->regarray.dstsrc.offset = fdst->offset / 8;
3911 instr_regwr_translate(struct rte_swx_pipeline *p,
3912 struct action *action,
3915 struct instruction *instr,
3916 struct instruction_data *data __rte_unused)
3918 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3920 struct field *fidx, *fsrc;
3922 uint32_t idx_struct_id, idx_val, src_struct_id;
3924 CHECK(n_tokens == 4, EINVAL);
3926 r = regarray_find(p, regarray);
3929 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3930 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3931 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3933 CHECK(!fidx->var_size, EINVAL);
3934 CHECK(!fsrc->var_size, EINVAL);
3936 instr->type = INSTR_REGWR_RMM;
3937 if (idx[0] == 'h' && src[0] != 'h')
3938 instr->type = INSTR_REGWR_RHM;
3939 if (idx[0] != 'h' && src[0] == 'h')
3940 instr->type = INSTR_REGWR_RMH;
3941 if (idx[0] == 'h' && src[0] == 'h')
3942 instr->type = INSTR_REGWR_RHH;
3944 instr->regarray.regarray_id = r->id;
3945 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3946 instr->regarray.idx.n_bits = fidx->n_bits;
3947 instr->regarray.idx.offset = fidx->offset / 8;
3948 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3949 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3950 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3954 /* REGWR_RHI, REGWR_RMI. */
3955 if (fidx && !fsrc) {
3956 CHECK(!fidx->var_size, EINVAL);
3958 src_val = strtoull(src, &src, 0);
3959 CHECK(!src[0], EINVAL);
3961 instr->type = INSTR_REGWR_RMI;
3963 instr->type = INSTR_REGWR_RHI;
3965 instr->regarray.regarray_id = r->id;
3966 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3967 instr->regarray.idx.n_bits = fidx->n_bits;
3968 instr->regarray.idx.offset = fidx->offset / 8;
3969 instr->regarray.dstsrc_val = src_val;
3973 /* REGWR_RIH, REGWR_RIM. */
3974 if (!fidx && fsrc) {
3975 idx_val = strtoul(idx, &idx, 0);
3976 CHECK(!idx[0], EINVAL);
3978 CHECK(!fsrc->var_size, EINVAL);
3980 instr->type = INSTR_REGWR_RIM;
3982 instr->type = INSTR_REGWR_RIH;
3984 instr->regarray.regarray_id = r->id;
3985 instr->regarray.idx_val = idx_val;
3986 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3987 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3988 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3993 src_val = strtoull(src, &src, 0);
3994 CHECK(!src[0], EINVAL);
3996 idx_val = strtoul(idx, &idx, 0);
3997 CHECK(!idx[0], EINVAL);
3999 instr->type = INSTR_REGWR_RII;
4000 instr->regarray.idx_val = idx_val;
4001 instr->regarray.dstsrc_val = src_val;
4007 instr_regadd_translate(struct rte_swx_pipeline *p,
4008 struct action *action,
4011 struct instruction *instr,
4012 struct instruction_data *data __rte_unused)
4014 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4016 struct field *fidx, *fsrc;
4018 uint32_t idx_struct_id, idx_val, src_struct_id;
4020 CHECK(n_tokens == 4, EINVAL);
4022 r = regarray_find(p, regarray);
4025 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4026 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4027 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4029 CHECK(!fidx->var_size, EINVAL);
4030 CHECK(!fsrc->var_size, EINVAL);
4032 instr->type = INSTR_REGADD_RMM;
4033 if (idx[0] == 'h' && src[0] != 'h')
4034 instr->type = INSTR_REGADD_RHM;
4035 if (idx[0] != 'h' && src[0] == 'h')
4036 instr->type = INSTR_REGADD_RMH;
4037 if (idx[0] == 'h' && src[0] == 'h')
4038 instr->type = INSTR_REGADD_RHH;
4040 instr->regarray.regarray_id = r->id;
4041 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4042 instr->regarray.idx.n_bits = fidx->n_bits;
4043 instr->regarray.idx.offset = fidx->offset / 8;
4044 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4045 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4046 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4050 /* REGADD_RHI, REGADD_RMI. */
4051 if (fidx && !fsrc) {
4052 CHECK(!fidx->var_size, EINVAL);
4054 src_val = strtoull(src, &src, 0);
4055 CHECK(!src[0], EINVAL);
4057 instr->type = INSTR_REGADD_RMI;
4059 instr->type = INSTR_REGADD_RHI;
4061 instr->regarray.regarray_id = r->id;
4062 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4063 instr->regarray.idx.n_bits = fidx->n_bits;
4064 instr->regarray.idx.offset = fidx->offset / 8;
4065 instr->regarray.dstsrc_val = src_val;
4069 /* REGADD_RIH, REGADD_RIM. */
4070 if (!fidx && fsrc) {
4071 idx_val = strtoul(idx, &idx, 0);
4072 CHECK(!idx[0], EINVAL);
4074 CHECK(!fsrc->var_size, EINVAL);
4076 instr->type = INSTR_REGADD_RIM;
4078 instr->type = INSTR_REGADD_RIH;
4080 instr->regarray.regarray_id = r->id;
4081 instr->regarray.idx_val = idx_val;
4082 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4083 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4084 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4089 src_val = strtoull(src, &src, 0);
4090 CHECK(!src[0], EINVAL);
4092 idx_val = strtoul(idx, &idx, 0);
4093 CHECK(!idx[0], EINVAL);
4095 instr->type = INSTR_REGADD_RII;
4096 instr->regarray.idx_val = idx_val;
4097 instr->regarray.dstsrc_val = src_val;
4102 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4104 struct thread *t = &p->threads[p->thread_id];
4105 struct instruction *ip = t->ip;
4108 __instr_regprefetch_rh_exec(p, t, ip);
4115 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4117 struct thread *t = &p->threads[p->thread_id];
4118 struct instruction *ip = t->ip;
4121 __instr_regprefetch_rm_exec(p, t, ip);
4128 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4130 struct thread *t = &p->threads[p->thread_id];
4131 struct instruction *ip = t->ip;
4134 __instr_regprefetch_ri_exec(p, t, ip);
4141 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4143 struct thread *t = &p->threads[p->thread_id];
4144 struct instruction *ip = t->ip;
4147 __instr_regrd_hrh_exec(p, t, ip);
4154 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4156 struct thread *t = &p->threads[p->thread_id];
4157 struct instruction *ip = t->ip;
4160 __instr_regrd_hrm_exec(p, t, ip);
4167 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4169 struct thread *t = &p->threads[p->thread_id];
4170 struct instruction *ip = t->ip;
4173 __instr_regrd_mrh_exec(p, t, ip);
4180 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4182 struct thread *t = &p->threads[p->thread_id];
4183 struct instruction *ip = t->ip;
4186 __instr_regrd_mrm_exec(p, t, ip);
4193 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4195 struct thread *t = &p->threads[p->thread_id];
4196 struct instruction *ip = t->ip;
4199 __instr_regrd_hri_exec(p, t, ip);
4206 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4208 struct thread *t = &p->threads[p->thread_id];
4209 struct instruction *ip = t->ip;
4212 __instr_regrd_mri_exec(p, t, ip);
4219 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4221 struct thread *t = &p->threads[p->thread_id];
4222 struct instruction *ip = t->ip;
4225 __instr_regwr_rhh_exec(p, t, ip);
4232 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4234 struct thread *t = &p->threads[p->thread_id];
4235 struct instruction *ip = t->ip;
4238 __instr_regwr_rhm_exec(p, t, ip);
4245 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4247 struct thread *t = &p->threads[p->thread_id];
4248 struct instruction *ip = t->ip;
4251 __instr_regwr_rmh_exec(p, t, ip);
4258 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4260 struct thread *t = &p->threads[p->thread_id];
4261 struct instruction *ip = t->ip;
4264 __instr_regwr_rmm_exec(p, t, ip);
4271 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4273 struct thread *t = &p->threads[p->thread_id];
4274 struct instruction *ip = t->ip;
4277 __instr_regwr_rhi_exec(p, t, ip);
4284 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4286 struct thread *t = &p->threads[p->thread_id];
4287 struct instruction *ip = t->ip;
4290 __instr_regwr_rmi_exec(p, t, ip);
4297 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4299 struct thread *t = &p->threads[p->thread_id];
4300 struct instruction *ip = t->ip;
4303 __instr_regwr_rih_exec(p, t, ip);
4310 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4312 struct thread *t = &p->threads[p->thread_id];
4313 struct instruction *ip = t->ip;
4316 __instr_regwr_rim_exec(p, t, ip);
4323 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4325 struct thread *t = &p->threads[p->thread_id];
4326 struct instruction *ip = t->ip;
4329 __instr_regwr_rii_exec(p, t, ip);
4336 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4338 struct thread *t = &p->threads[p->thread_id];
4339 struct instruction *ip = t->ip;
4342 __instr_regadd_rhh_exec(p, t, ip);
4349 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4351 struct thread *t = &p->threads[p->thread_id];
4352 struct instruction *ip = t->ip;
4355 __instr_regadd_rhm_exec(p, t, ip);
4362 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4364 struct thread *t = &p->threads[p->thread_id];
4365 struct instruction *ip = t->ip;
4368 __instr_regadd_rmh_exec(p, t, ip);
4375 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4377 struct thread *t = &p->threads[p->thread_id];
4378 struct instruction *ip = t->ip;
4381 __instr_regadd_rmm_exec(p, t, ip);
4388 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4390 struct thread *t = &p->threads[p->thread_id];
4391 struct instruction *ip = t->ip;
4394 __instr_regadd_rhi_exec(p, t, ip);
4401 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4403 struct thread *t = &p->threads[p->thread_id];
4404 struct instruction *ip = t->ip;
4407 __instr_regadd_rmi_exec(p, t, ip);
4414 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4416 struct thread *t = &p->threads[p->thread_id];
4417 struct instruction *ip = t->ip;
4420 __instr_regadd_rih_exec(p, t, ip);
4427 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4429 struct thread *t = &p->threads[p->thread_id];
4430 struct instruction *ip = t->ip;
4433 __instr_regadd_rim_exec(p, t, ip);
4440 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4442 struct thread *t = &p->threads[p->thread_id];
4443 struct instruction *ip = t->ip;
4446 __instr_regadd_rii_exec(p, t, ip);
4455 static struct metarray *
4456 metarray_find(struct rte_swx_pipeline *p, const char *name);
4459 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4460 struct action *action,
4463 struct instruction *instr,
4464 struct instruction_data *data __rte_unused)
4466 char *metarray = tokens[1], *idx = tokens[2];
4469 uint32_t idx_struct_id, idx_val;
4471 CHECK(n_tokens == 3, EINVAL);
4473 m = metarray_find(p, metarray);
4476 /* METPREFETCH_H, METPREFETCH_M. */
4477 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4479 CHECK(!fidx->var_size, EINVAL);
4481 instr->type = INSTR_METPREFETCH_M;
4483 instr->type = INSTR_METPREFETCH_H;
4485 instr->meter.metarray_id = m->id;
4486 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4487 instr->meter.idx.n_bits = fidx->n_bits;
4488 instr->meter.idx.offset = fidx->offset / 8;
4492 /* METPREFETCH_I. */
4493 idx_val = strtoul(idx, &idx, 0);
4494 CHECK(!idx[0], EINVAL);
4496 instr->type = INSTR_METPREFETCH_I;
4497 instr->meter.metarray_id = m->id;
4498 instr->meter.idx_val = idx_val;
4503 instr_meter_translate(struct rte_swx_pipeline *p,
4504 struct action *action,
4507 struct instruction *instr,
4508 struct instruction_data *data __rte_unused)
4510 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4511 char *color_in = tokens[4], *color_out = tokens[5];
4513 struct field *fidx, *flength, *fcin, *fcout;
4514 uint32_t idx_struct_id, length_struct_id;
4515 uint32_t color_in_struct_id, color_out_struct_id;
4517 CHECK(n_tokens == 6, EINVAL);
4519 m = metarray_find(p, metarray);
4522 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4524 flength = struct_field_parse(p, action, length, &length_struct_id);
4525 CHECK(flength, EINVAL);
4526 CHECK(!flength->var_size, EINVAL);
4528 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4530 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4531 CHECK(fcout, EINVAL);
4532 CHECK(!fcout->var_size, EINVAL);
4534 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4536 CHECK(!fidx->var_size, EINVAL);
4537 CHECK(!fcin->var_size, EINVAL);
4539 instr->type = INSTR_METER_MMM;
4540 if (idx[0] == 'h' && length[0] == 'h')
4541 instr->type = INSTR_METER_HHM;
4542 if (idx[0] == 'h' && length[0] != 'h')
4543 instr->type = INSTR_METER_HMM;
4544 if (idx[0] != 'h' && length[0] == 'h')
4545 instr->type = INSTR_METER_MHM;
4547 instr->meter.metarray_id = m->id;
4549 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4550 instr->meter.idx.n_bits = fidx->n_bits;
4551 instr->meter.idx.offset = fidx->offset / 8;
4553 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4554 instr->meter.length.n_bits = flength->n_bits;
4555 instr->meter.length.offset = flength->offset / 8;
4557 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4558 instr->meter.color_in.n_bits = fcin->n_bits;
4559 instr->meter.color_in.offset = fcin->offset / 8;
4561 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4562 instr->meter.color_out.n_bits = fcout->n_bits;
4563 instr->meter.color_out.offset = fcout->offset / 8;
4566 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4567 if (fidx && !fcin) {
4568 uint32_t color_in_val;
4570 CHECK(!fidx->var_size, EINVAL);
4572 color_in_val = strtoul(color_in, &color_in, 0);
4573 CHECK(!color_in[0], EINVAL);
4575 instr->type = INSTR_METER_MMI;
4576 if (idx[0] == 'h' && length[0] == 'h')
4577 instr->type = INSTR_METER_HHI;
4578 if (idx[0] == 'h' && length[0] != 'h')
4579 instr->type = INSTR_METER_HMI;
4580 if (idx[0] != 'h' && length[0] == 'h')
4581 instr->type = INSTR_METER_MHI;
4583 instr->meter.metarray_id = m->id;
4585 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4586 instr->meter.idx.n_bits = fidx->n_bits;
4587 instr->meter.idx.offset = fidx->offset / 8;
4589 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4590 instr->meter.length.n_bits = flength->n_bits;
4591 instr->meter.length.offset = flength->offset / 8;
4593 instr->meter.color_in_val = color_in_val;
4595 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4596 instr->meter.color_out.n_bits = fcout->n_bits;
4597 instr->meter.color_out.offset = fcout->offset / 8;
4600 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4601 if (!fidx && fcin) {
4604 idx_val = strtoul(idx, &idx, 0);
4605 CHECK(!idx[0], EINVAL);
4607 CHECK(!fcin->var_size, EINVAL);
4609 instr->type = INSTR_METER_IMM;
4610 if (length[0] == 'h')
4611 instr->type = INSTR_METER_IHM;
4613 instr->meter.metarray_id = m->id;
4615 instr->meter.idx_val = idx_val;
4617 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4618 instr->meter.length.n_bits = flength->n_bits;
4619 instr->meter.length.offset = flength->offset / 8;
4621 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4622 instr->meter.color_in.n_bits = fcin->n_bits;
4623 instr->meter.color_in.offset = fcin->offset / 8;
4625 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4626 instr->meter.color_out.n_bits = fcout->n_bits;
4627 instr->meter.color_out.offset = fcout->offset / 8;
4630 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4631 if (!fidx && !fcin) {
4632 uint32_t idx_val, color_in_val;
4634 idx_val = strtoul(idx, &idx, 0);
4635 CHECK(!idx[0], EINVAL);
4637 color_in_val = strtoul(color_in, &color_in, 0);
4638 CHECK(!color_in[0], EINVAL);
4640 instr->type = INSTR_METER_IMI;
4641 if (length[0] == 'h')
4642 instr->type = INSTR_METER_IHI;
4644 instr->meter.metarray_id = m->id;
4646 instr->meter.idx_val = idx_val;
4648 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4649 instr->meter.length.n_bits = flength->n_bits;
4650 instr->meter.length.offset = flength->offset / 8;
4652 instr->meter.color_in_val = color_in_val;
4654 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4655 instr->meter.color_out.n_bits = fcout->n_bits;
4656 instr->meter.color_out.offset = fcout->offset / 8;
4663 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4665 struct thread *t = &p->threads[p->thread_id];
4666 struct instruction *ip = t->ip;
4669 __instr_metprefetch_h_exec(p, t, ip);
4676 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4678 struct thread *t = &p->threads[p->thread_id];
4679 struct instruction *ip = t->ip;
4682 __instr_metprefetch_m_exec(p, t, ip);
4689 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4691 struct thread *t = &p->threads[p->thread_id];
4692 struct instruction *ip = t->ip;
4695 __instr_metprefetch_i_exec(p, t, ip);
4702 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4704 struct thread *t = &p->threads[p->thread_id];
4705 struct instruction *ip = t->ip;
4708 __instr_meter_hhm_exec(p, t, ip);
4715 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4717 struct thread *t = &p->threads[p->thread_id];
4718 struct instruction *ip = t->ip;
4721 __instr_meter_hhi_exec(p, t, ip);
4728 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4730 struct thread *t = &p->threads[p->thread_id];
4731 struct instruction *ip = t->ip;
4734 __instr_meter_hmm_exec(p, t, ip);
4741 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4743 struct thread *t = &p->threads[p->thread_id];
4744 struct instruction *ip = t->ip;
4747 __instr_meter_hmi_exec(p, t, ip);
4754 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4756 struct thread *t = &p->threads[p->thread_id];
4757 struct instruction *ip = t->ip;
4760 __instr_meter_mhm_exec(p, t, ip);
4767 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4769 struct thread *t = &p->threads[p->thread_id];
4770 struct instruction *ip = t->ip;
4773 __instr_meter_mhi_exec(p, t, ip);
4780 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4782 struct thread *t = &p->threads[p->thread_id];
4783 struct instruction *ip = t->ip;
4786 __instr_meter_mmm_exec(p, t, ip);
4793 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4795 struct thread *t = &p->threads[p->thread_id];
4796 struct instruction *ip = t->ip;
4799 __instr_meter_mmi_exec(p, t, ip);
4806 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4808 struct thread *t = &p->threads[p->thread_id];
4809 struct instruction *ip = t->ip;
4812 __instr_meter_ihm_exec(p, t, ip);
4819 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4821 struct thread *t = &p->threads[p->thread_id];
4822 struct instruction *ip = t->ip;
4825 __instr_meter_ihi_exec(p, t, ip);
4832 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4834 struct thread *t = &p->threads[p->thread_id];
4835 struct instruction *ip = t->ip;
4838 __instr_meter_imm_exec(p, t, ip);
4845 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4847 struct thread *t = &p->threads[p->thread_id];
4848 struct instruction *ip = t->ip;
4851 __instr_meter_imi_exec(p, t, ip);
4861 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4862 struct action *action __rte_unused,
4865 struct instruction *instr,
4866 struct instruction_data *data)
4868 CHECK(n_tokens == 2, EINVAL);
4870 strcpy(data->jmp_label, tokens[1]);
4872 instr->type = INSTR_JMP;
4873 instr->jmp.ip = NULL; /* Resolved later. */
4878 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4879 struct action *action __rte_unused,
4882 struct instruction *instr,
4883 struct instruction_data *data)
4887 CHECK(n_tokens == 3, EINVAL);
4889 strcpy(data->jmp_label, tokens[1]);
4891 h = header_parse(p, tokens[2]);
4894 instr->type = INSTR_JMP_VALID;
4895 instr->jmp.ip = NULL; /* Resolved later. */
4896 instr->jmp.header_id = h->id;
4901 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4902 struct action *action __rte_unused,
4905 struct instruction *instr,
4906 struct instruction_data *data)
4910 CHECK(n_tokens == 3, EINVAL);
4912 strcpy(data->jmp_label, tokens[1]);
4914 h = header_parse(p, tokens[2]);
4917 instr->type = INSTR_JMP_INVALID;
4918 instr->jmp.ip = NULL; /* Resolved later. */
4919 instr->jmp.header_id = h->id;
4924 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4925 struct action *action,
4928 struct instruction *instr,
4929 struct instruction_data *data)
4931 CHECK(!action, EINVAL);
4932 CHECK(n_tokens == 2, EINVAL);
4934 strcpy(data->jmp_label, tokens[1]);
4936 instr->type = INSTR_JMP_HIT;
4937 instr->jmp.ip = NULL; /* Resolved later. */
4942 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4943 struct action *action,
4946 struct instruction *instr,
4947 struct instruction_data *data)
4949 CHECK(!action, EINVAL);
4950 CHECK(n_tokens == 2, EINVAL);
4952 strcpy(data->jmp_label, tokens[1]);
4954 instr->type = INSTR_JMP_MISS;
4955 instr->jmp.ip = NULL; /* Resolved later. */
4960 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4961 struct action *action,
4964 struct instruction *instr,
4965 struct instruction_data *data)
4969 CHECK(!action, EINVAL);
4970 CHECK(n_tokens == 3, EINVAL);
4972 strcpy(data->jmp_label, tokens[1]);
4974 a = action_find(p, tokens[2]);
4977 instr->type = INSTR_JMP_ACTION_HIT;
4978 instr->jmp.ip = NULL; /* Resolved later. */
4979 instr->jmp.action_id = a->id;
4984 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4985 struct action *action,
4988 struct instruction *instr,
4989 struct instruction_data *data)
4993 CHECK(!action, EINVAL);
4994 CHECK(n_tokens == 3, EINVAL);
4996 strcpy(data->jmp_label, tokens[1]);
4998 a = action_find(p, tokens[2]);
5001 instr->type = INSTR_JMP_ACTION_MISS;
5002 instr->jmp.ip = NULL; /* Resolved later. */
5003 instr->jmp.action_id = a->id;
5008 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5009 struct action *action,
5012 struct instruction *instr,
5013 struct instruction_data *data)
5015 char *a = tokens[2], *b = tokens[3];
5016 struct field *fa, *fb;
5018 uint32_t a_struct_id, b_struct_id;
5020 CHECK(n_tokens == 4, EINVAL);
5022 strcpy(data->jmp_label, tokens[1]);
5024 fa = struct_field_parse(p, action, a, &a_struct_id);
5026 CHECK(!fa->var_size, EINVAL);
5028 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5029 fb = struct_field_parse(p, action, b, &b_struct_id);
5031 CHECK(!fb->var_size, EINVAL);
5033 instr->type = INSTR_JMP_EQ;
5034 if (a[0] != 'h' && b[0] == 'h')
5035 instr->type = INSTR_JMP_EQ_MH;
5036 if (a[0] == 'h' && b[0] != 'h')
5037 instr->type = INSTR_JMP_EQ_HM;
5038 if (a[0] == 'h' && b[0] == 'h')
5039 instr->type = INSTR_JMP_EQ_HH;
5040 instr->jmp.ip = NULL; /* Resolved later. */
5042 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5043 instr->jmp.a.n_bits = fa->n_bits;
5044 instr->jmp.a.offset = fa->offset / 8;
5045 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5046 instr->jmp.b.n_bits = fb->n_bits;
5047 instr->jmp.b.offset = fb->offset / 8;
5052 b_val = strtoull(b, &b, 0);
5053 CHECK(!b[0], EINVAL);
5056 b_val = hton64(b_val) >> (64 - fa->n_bits);
5058 instr->type = INSTR_JMP_EQ_I;
5059 instr->jmp.ip = NULL; /* Resolved later. */
5060 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5061 instr->jmp.a.n_bits = fa->n_bits;
5062 instr->jmp.a.offset = fa->offset / 8;
5063 instr->jmp.b_val = b_val;
5068 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5069 struct action *action,
5072 struct instruction *instr,
5073 struct instruction_data *data)
5075 char *a = tokens[2], *b = tokens[3];
5076 struct field *fa, *fb;
5078 uint32_t a_struct_id, b_struct_id;
5080 CHECK(n_tokens == 4, EINVAL);
5082 strcpy(data->jmp_label, tokens[1]);
5084 fa = struct_field_parse(p, action, a, &a_struct_id);
5086 CHECK(!fa->var_size, EINVAL);
5088 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5089 fb = struct_field_parse(p, action, b, &b_struct_id);
5091 CHECK(!fb->var_size, EINVAL);
5093 instr->type = INSTR_JMP_NEQ;
5094 if (a[0] != 'h' && b[0] == 'h')
5095 instr->type = INSTR_JMP_NEQ_MH;
5096 if (a[0] == 'h' && b[0] != 'h')
5097 instr->type = INSTR_JMP_NEQ_HM;
5098 if (a[0] == 'h' && b[0] == 'h')
5099 instr->type = INSTR_JMP_NEQ_HH;
5100 instr->jmp.ip = NULL; /* Resolved later. */
5102 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5103 instr->jmp.a.n_bits = fa->n_bits;
5104 instr->jmp.a.offset = fa->offset / 8;
5105 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5106 instr->jmp.b.n_bits = fb->n_bits;
5107 instr->jmp.b.offset = fb->offset / 8;
5112 b_val = strtoull(b, &b, 0);
5113 CHECK(!b[0], EINVAL);
5116 b_val = hton64(b_val) >> (64 - fa->n_bits);
5118 instr->type = INSTR_JMP_NEQ_I;
5119 instr->jmp.ip = NULL; /* Resolved later. */
5120 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5121 instr->jmp.a.n_bits = fa->n_bits;
5122 instr->jmp.a.offset = fa->offset / 8;
5123 instr->jmp.b_val = b_val;
5128 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5129 struct action *action,
5132 struct instruction *instr,
5133 struct instruction_data *data)
5135 char *a = tokens[2], *b = tokens[3];
5136 struct field *fa, *fb;
5138 uint32_t a_struct_id, b_struct_id;
5140 CHECK(n_tokens == 4, EINVAL);
5142 strcpy(data->jmp_label, tokens[1]);
5144 fa = struct_field_parse(p, action, a, &a_struct_id);
5146 CHECK(!fa->var_size, EINVAL);
5148 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5149 fb = struct_field_parse(p, action, b, &b_struct_id);
5151 CHECK(!fb->var_size, EINVAL);
5153 instr->type = INSTR_JMP_LT;
5154 if (a[0] == 'h' && b[0] != 'h')
5155 instr->type = INSTR_JMP_LT_HM;
5156 if (a[0] != 'h' && b[0] == 'h')
5157 instr->type = INSTR_JMP_LT_MH;
5158 if (a[0] == 'h' && b[0] == 'h')
5159 instr->type = INSTR_JMP_LT_HH;
5160 instr->jmp.ip = NULL; /* Resolved later. */
5162 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5163 instr->jmp.a.n_bits = fa->n_bits;
5164 instr->jmp.a.offset = fa->offset / 8;
5165 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5166 instr->jmp.b.n_bits = fb->n_bits;
5167 instr->jmp.b.offset = fb->offset / 8;
5171 /* JMP_LT_MI, JMP_LT_HI. */
5172 b_val = strtoull(b, &b, 0);
5173 CHECK(!b[0], EINVAL);
5175 instr->type = INSTR_JMP_LT_MI;
5177 instr->type = INSTR_JMP_LT_HI;
5178 instr->jmp.ip = NULL; /* Resolved later. */
5180 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5181 instr->jmp.a.n_bits = fa->n_bits;
5182 instr->jmp.a.offset = fa->offset / 8;
5183 instr->jmp.b_val = b_val;
5188 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5189 struct action *action,
5192 struct instruction *instr,
5193 struct instruction_data *data)
5195 char *a = tokens[2], *b = tokens[3];
5196 struct field *fa, *fb;
5198 uint32_t a_struct_id, b_struct_id;
5200 CHECK(n_tokens == 4, EINVAL);
5202 strcpy(data->jmp_label, tokens[1]);
5204 fa = struct_field_parse(p, action, a, &a_struct_id);
5206 CHECK(!fa->var_size, EINVAL);
5208 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5209 fb = struct_field_parse(p, action, b, &b_struct_id);
5211 CHECK(!fb->var_size, EINVAL);
5213 instr->type = INSTR_JMP_GT;
5214 if (a[0] == 'h' && b[0] != 'h')
5215 instr->type = INSTR_JMP_GT_HM;
5216 if (a[0] != 'h' && b[0] == 'h')
5217 instr->type = INSTR_JMP_GT_MH;
5218 if (a[0] == 'h' && b[0] == 'h')
5219 instr->type = INSTR_JMP_GT_HH;
5220 instr->jmp.ip = NULL; /* Resolved later. */
5222 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5223 instr->jmp.a.n_bits = fa->n_bits;
5224 instr->jmp.a.offset = fa->offset / 8;
5225 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5226 instr->jmp.b.n_bits = fb->n_bits;
5227 instr->jmp.b.offset = fb->offset / 8;
5231 /* JMP_GT_MI, JMP_GT_HI. */
5232 b_val = strtoull(b, &b, 0);
5233 CHECK(!b[0], EINVAL);
5235 instr->type = INSTR_JMP_GT_MI;
5237 instr->type = INSTR_JMP_GT_HI;
5238 instr->jmp.ip = NULL; /* Resolved later. */
5240 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5241 instr->jmp.a.n_bits = fa->n_bits;
5242 instr->jmp.a.offset = fa->offset / 8;
5243 instr->jmp.b_val = b_val;
5248 instr_jmp_exec(struct rte_swx_pipeline *p)
5250 struct thread *t = &p->threads[p->thread_id];
5251 struct instruction *ip = t->ip;
5253 TRACE("[Thread %2u] jmp\n", p->thread_id);
5255 thread_ip_set(t, ip->jmp.ip);
5259 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5261 struct thread *t = &p->threads[p->thread_id];
5262 struct instruction *ip = t->ip;
5263 uint32_t header_id = ip->jmp.header_id;
5265 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5267 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5271 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5273 struct thread *t = &p->threads[p->thread_id];
5274 struct instruction *ip = t->ip;
5275 uint32_t header_id = ip->jmp.header_id;
5277 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5279 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5283 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5285 struct thread *t = &p->threads[p->thread_id];
5286 struct instruction *ip = t->ip;
5287 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5289 TRACE("[Thread %2u] jmph\n", p->thread_id);
5291 t->ip = ip_next[t->hit];
5295 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5297 struct thread *t = &p->threads[p->thread_id];
5298 struct instruction *ip = t->ip;
5299 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5301 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5303 t->ip = ip_next[t->hit];
5307 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5309 struct thread *t = &p->threads[p->thread_id];
5310 struct instruction *ip = t->ip;
5312 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5314 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5318 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5320 struct thread *t = &p->threads[p->thread_id];
5321 struct instruction *ip = t->ip;
5323 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5325 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5329 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5331 struct thread *t = &p->threads[p->thread_id];
5332 struct instruction *ip = t->ip;
5334 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5340 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5342 struct thread *t = &p->threads[p->thread_id];
5343 struct instruction *ip = t->ip;
5345 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5347 JMP_CMP_MH(t, ip, ==);
5351 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5353 struct thread *t = &p->threads[p->thread_id];
5354 struct instruction *ip = t->ip;
5356 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5358 JMP_CMP_HM(t, ip, ==);
5362 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5364 struct thread *t = &p->threads[p->thread_id];
5365 struct instruction *ip = t->ip;
5367 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5369 JMP_CMP_HH_FAST(t, ip, ==);
5373 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5375 struct thread *t = &p->threads[p->thread_id];
5376 struct instruction *ip = t->ip;
5378 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5380 JMP_CMP_I(t, ip, ==);
5384 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5386 struct thread *t = &p->threads[p->thread_id];
5387 struct instruction *ip = t->ip;
5389 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5395 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5397 struct thread *t = &p->threads[p->thread_id];
5398 struct instruction *ip = t->ip;
5400 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5402 JMP_CMP_MH(t, ip, !=);
5406 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5408 struct thread *t = &p->threads[p->thread_id];
5409 struct instruction *ip = t->ip;
5411 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5413 JMP_CMP_HM(t, ip, !=);
5417 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5419 struct thread *t = &p->threads[p->thread_id];
5420 struct instruction *ip = t->ip;
5422 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5424 JMP_CMP_HH_FAST(t, ip, !=);
5428 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5430 struct thread *t = &p->threads[p->thread_id];
5431 struct instruction *ip = t->ip;
5433 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5435 JMP_CMP_I(t, ip, !=);
5439 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5441 struct thread *t = &p->threads[p->thread_id];
5442 struct instruction *ip = t->ip;
5444 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5450 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5452 struct thread *t = &p->threads[p->thread_id];
5453 struct instruction *ip = t->ip;
5455 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5457 JMP_CMP_MH(t, ip, <);
5461 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5463 struct thread *t = &p->threads[p->thread_id];
5464 struct instruction *ip = t->ip;
5466 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5468 JMP_CMP_HM(t, ip, <);
5472 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5474 struct thread *t = &p->threads[p->thread_id];
5475 struct instruction *ip = t->ip;
5477 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5479 JMP_CMP_HH(t, ip, <);
5483 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5485 struct thread *t = &p->threads[p->thread_id];
5486 struct instruction *ip = t->ip;
5488 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5490 JMP_CMP_MI(t, ip, <);
5494 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5496 struct thread *t = &p->threads[p->thread_id];
5497 struct instruction *ip = t->ip;
5499 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5501 JMP_CMP_HI(t, ip, <);
5505 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5507 struct thread *t = &p->threads[p->thread_id];
5508 struct instruction *ip = t->ip;
5510 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5516 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5518 struct thread *t = &p->threads[p->thread_id];
5519 struct instruction *ip = t->ip;
5521 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5523 JMP_CMP_MH(t, ip, >);
5527 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5529 struct thread *t = &p->threads[p->thread_id];
5530 struct instruction *ip = t->ip;
5532 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5534 JMP_CMP_HM(t, ip, >);
5538 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5540 struct thread *t = &p->threads[p->thread_id];
5541 struct instruction *ip = t->ip;
5543 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5545 JMP_CMP_HH(t, ip, >);
5549 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5551 struct thread *t = &p->threads[p->thread_id];
5552 struct instruction *ip = t->ip;
5554 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5556 JMP_CMP_MI(t, ip, >);
5560 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5562 struct thread *t = &p->threads[p->thread_id];
5563 struct instruction *ip = t->ip;
5565 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5567 JMP_CMP_HI(t, ip, >);
5574 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5575 struct action *action,
5576 char **tokens __rte_unused,
5578 struct instruction *instr,
5579 struct instruction_data *data __rte_unused)
5581 CHECK(action, EINVAL);
5582 CHECK(n_tokens == 1, EINVAL);
5584 instr->type = INSTR_RETURN;
5589 instr_return_exec(struct rte_swx_pipeline *p)
5591 struct thread *t = &p->threads[p->thread_id];
5593 TRACE("[Thread %2u] return\n", p->thread_id);
5599 instr_translate(struct rte_swx_pipeline *p,
5600 struct action *action,
5602 struct instruction *instr,
5603 struct instruction_data *data)
5605 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5606 int n_tokens = 0, tpos = 0;
5608 /* Parse the instruction string into tokens. */
5612 token = strtok_r(string, " \t\v", &string);
5616 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5617 CHECK_NAME(token, EINVAL);
5619 tokens[n_tokens] = token;
5623 CHECK(n_tokens, EINVAL);
5625 /* Handle the optional instruction label. */
5626 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5627 strcpy(data->label, tokens[0]);
5630 CHECK(n_tokens - tpos, EINVAL);
5633 /* Identify the instruction type. */
5634 if (!strcmp(tokens[tpos], "rx"))
5635 return instr_rx_translate(p,
5642 if (!strcmp(tokens[tpos], "tx"))
5643 return instr_tx_translate(p,
5650 if (!strcmp(tokens[tpos], "drop"))
5651 return instr_drop_translate(p,
5658 if (!strcmp(tokens[tpos], "extract"))
5659 return instr_hdr_extract_translate(p,
5666 if (!strcmp(tokens[tpos], "lookahead"))
5667 return instr_hdr_lookahead_translate(p,
5674 if (!strcmp(tokens[tpos], "emit"))
5675 return instr_hdr_emit_translate(p,
5682 if (!strcmp(tokens[tpos], "validate"))
5683 return instr_hdr_validate_translate(p,
5690 if (!strcmp(tokens[tpos], "invalidate"))
5691 return instr_hdr_invalidate_translate(p,
5698 if (!strcmp(tokens[tpos], "mov"))
5699 return instr_mov_translate(p,
5706 if (!strcmp(tokens[tpos], "add"))
5707 return instr_alu_add_translate(p,
5714 if (!strcmp(tokens[tpos], "sub"))
5715 return instr_alu_sub_translate(p,
5722 if (!strcmp(tokens[tpos], "ckadd"))
5723 return instr_alu_ckadd_translate(p,
5730 if (!strcmp(tokens[tpos], "cksub"))
5731 return instr_alu_cksub_translate(p,
5738 if (!strcmp(tokens[tpos], "and"))
5739 return instr_alu_and_translate(p,
5746 if (!strcmp(tokens[tpos], "or"))
5747 return instr_alu_or_translate(p,
5754 if (!strcmp(tokens[tpos], "xor"))
5755 return instr_alu_xor_translate(p,
5762 if (!strcmp(tokens[tpos], "shl"))
5763 return instr_alu_shl_translate(p,
5770 if (!strcmp(tokens[tpos], "shr"))
5771 return instr_alu_shr_translate(p,
5778 if (!strcmp(tokens[tpos], "regprefetch"))
5779 return instr_regprefetch_translate(p,
5786 if (!strcmp(tokens[tpos], "regrd"))
5787 return instr_regrd_translate(p,
5794 if (!strcmp(tokens[tpos], "regwr"))
5795 return instr_regwr_translate(p,
5802 if (!strcmp(tokens[tpos], "regadd"))
5803 return instr_regadd_translate(p,
5810 if (!strcmp(tokens[tpos], "metprefetch"))
5811 return instr_metprefetch_translate(p,
5818 if (!strcmp(tokens[tpos], "meter"))
5819 return instr_meter_translate(p,
5826 if (!strcmp(tokens[tpos], "table"))
5827 return instr_table_translate(p,
5834 if (!strcmp(tokens[tpos], "learn"))
5835 return instr_learn_translate(p,
5842 if (!strcmp(tokens[tpos], "forget"))
5843 return instr_forget_translate(p,
5850 if (!strcmp(tokens[tpos], "extern"))
5851 return instr_extern_translate(p,
5858 if (!strcmp(tokens[tpos], "jmp"))
5859 return instr_jmp_translate(p,
5866 if (!strcmp(tokens[tpos], "jmpv"))
5867 return instr_jmp_valid_translate(p,
5874 if (!strcmp(tokens[tpos], "jmpnv"))
5875 return instr_jmp_invalid_translate(p,
5882 if (!strcmp(tokens[tpos], "jmph"))
5883 return instr_jmp_hit_translate(p,
5890 if (!strcmp(tokens[tpos], "jmpnh"))
5891 return instr_jmp_miss_translate(p,
5898 if (!strcmp(tokens[tpos], "jmpa"))
5899 return instr_jmp_action_hit_translate(p,
5906 if (!strcmp(tokens[tpos], "jmpna"))
5907 return instr_jmp_action_miss_translate(p,
5914 if (!strcmp(tokens[tpos], "jmpeq"))
5915 return instr_jmp_eq_translate(p,
5922 if (!strcmp(tokens[tpos], "jmpneq"))
5923 return instr_jmp_neq_translate(p,
5930 if (!strcmp(tokens[tpos], "jmplt"))
5931 return instr_jmp_lt_translate(p,
5938 if (!strcmp(tokens[tpos], "jmpgt"))
5939 return instr_jmp_gt_translate(p,
5946 if (!strcmp(tokens[tpos], "return"))
5947 return instr_return_translate(p,
5957 static struct instruction_data *
5958 label_find(struct instruction_data *data, uint32_t n, const char *label)
5962 for (i = 0; i < n; i++)
5963 if (!strcmp(label, data[i].label))
5970 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5972 uint32_t count = 0, i;
5977 for (i = 0; i < n; i++)
5978 if (!strcmp(label, data[i].jmp_label))
5985 instr_label_check(struct instruction_data *instruction_data,
5986 uint32_t n_instructions)
5990 /* Check that all instruction labels are unique. */
5991 for (i = 0; i < n_instructions; i++) {
5992 struct instruction_data *data = &instruction_data[i];
5993 char *label = data->label;
5999 for (j = i + 1; j < n_instructions; j++)
6000 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6003 /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6004 for (i = 0; i < n_instructions; i++) {
6005 struct instruction_data *data = &instruction_data[i];
6006 char *label = data->label;
6007 char *jmp_label = data->jmp_label;
6009 /* Continue if this instruction does not have a label or it is not a jump. */
6010 if (!label[0] || !jmp_label[0])
6013 CHECK(strcmp(label, jmp_label), EINVAL);
6016 /* Get users for each instruction label. */
6017 for (i = 0; i < n_instructions; i++) {
6018 struct instruction_data *data = &instruction_data[i];
6019 char *label = data->label;
6021 data->n_users = label_is_used(instruction_data,
6030 instr_jmp_resolve(struct instruction *instructions,
6031 struct instruction_data *instruction_data,
6032 uint32_t n_instructions)
6036 for (i = 0; i < n_instructions; i++) {
6037 struct instruction *instr = &instructions[i];
6038 struct instruction_data *data = &instruction_data[i];
6039 struct instruction_data *found;
6041 if (!instruction_is_jmp(instr))
6044 found = label_find(instruction_data,
6047 CHECK(found, EINVAL);
6049 instr->jmp.ip = &instructions[found - instruction_data];
6056 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6058 struct instruction *instr,
6059 struct instruction_data *data __rte_unused,
6060 uint32_t n_instructions)
6063 enum instruction_type type;
6066 /* Check that the first instruction is rx. */
6067 CHECK(instr[0].type == INSTR_RX, EINVAL);
6069 /* Check that there is at least one tx instruction. */
6070 for (i = 0; i < n_instructions; i++) {
6071 type = instr[i].type;
6073 if (instruction_is_tx(type))
6076 CHECK(i < n_instructions, EINVAL);
6078 /* Check that the last instruction is either tx or unconditional
6081 type = instr[n_instructions - 1].type;
6082 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6086 enum instruction_type type;
6089 /* Check that there is at least one return or tx instruction. */
6090 for (i = 0; i < n_instructions; i++) {
6091 type = instr[i].type;
6093 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6096 CHECK(i < n_instructions, EINVAL);
6103 instr_compact(struct instruction *instructions,
6104 struct instruction_data *instruction_data,
6105 uint32_t n_instructions)
6107 uint32_t i, pos = 0;
6109 /* Eliminate the invalid instructions that have been optimized out. */
6110 for (i = 0; i < n_instructions; i++) {
6111 struct instruction *instr = &instructions[i];
6112 struct instruction_data *data = &instruction_data[i];
6118 memcpy(&instructions[pos], instr, sizeof(*instr));
6119 memcpy(&instruction_data[pos], data, sizeof(*data));
6129 instr_pattern_extract_many_search(struct instruction *instr,
6130 struct instruction_data *data,
6132 uint32_t *n_pattern_instr)
6136 for (i = 0; i < n_instr; i++) {
6137 if (data[i].invalid)
6140 if (instr[i].type != INSTR_HDR_EXTRACT)
6143 if (i == RTE_DIM(instr->io.hdr.header_id))
6146 if (i && data[i].n_users)
6153 *n_pattern_instr = i;
6158 instr_pattern_extract_many_replace(struct instruction *instr,
6159 struct instruction_data *data,
6164 for (i = 1; i < n_instr; i++) {
6166 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6167 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6168 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6170 data[i].invalid = 1;
6175 instr_pattern_extract_many_optimize(struct instruction *instructions,
6176 struct instruction_data *instruction_data,
6177 uint32_t n_instructions)
6181 for (i = 0; i < n_instructions; ) {
6182 struct instruction *instr = &instructions[i];
6183 struct instruction_data *data = &instruction_data[i];
6184 uint32_t n_instr = 0;
6188 detected = instr_pattern_extract_many_search(instr,
6193 instr_pattern_extract_many_replace(instr,
6200 /* No pattern starting at the current instruction. */
6204 /* Eliminate the invalid instructions that have been optimized out. */
6205 n_instructions = instr_compact(instructions,
6209 return n_instructions;
6213 instr_pattern_emit_many_tx_search(struct instruction *instr,
6214 struct instruction_data *data,
6216 uint32_t *n_pattern_instr)
6220 for (i = 0; i < n_instr; i++) {
6221 if (data[i].invalid)
6224 if (instr[i].type != INSTR_HDR_EMIT)
6227 if (i == RTE_DIM(instr->io.hdr.header_id))
6230 if (i && data[i].n_users)
6237 if (instr[i].type != INSTR_TX)
6240 if (data[i].n_users)
6245 *n_pattern_instr = i;
6250 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6251 struct instruction_data *data,
6256 /* Any emit instruction in addition to the first one. */
6257 for (i = 1; i < n_instr - 1; i++) {
6259 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6260 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6261 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6263 data[i].invalid = 1;
6266 /* The TX instruction is the last one in the pattern. */
6268 instr[0].io.io.offset = instr[i].io.io.offset;
6269 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6270 data[i].invalid = 1;
6274 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6275 struct instruction_data *instruction_data,
6276 uint32_t n_instructions)
6280 for (i = 0; i < n_instructions; ) {
6281 struct instruction *instr = &instructions[i];
6282 struct instruction_data *data = &instruction_data[i];
6283 uint32_t n_instr = 0;
6286 /* Emit many + TX. */
6287 detected = instr_pattern_emit_many_tx_search(instr,
6292 instr_pattern_emit_many_tx_replace(instr,
6299 /* No pattern starting at the current instruction. */
6303 /* Eliminate the invalid instructions that have been optimized out. */
6304 n_instructions = instr_compact(instructions,
6308 return n_instructions;
6312 action_arg_src_mov_count(struct action *a,
6314 struct instruction *instructions,
6315 struct instruction_data *instruction_data,
6316 uint32_t n_instructions);
6319 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6321 struct instruction *instr,
6322 struct instruction_data *data,
6324 struct instruction *instructions,
6325 struct instruction_data *instruction_data,
6326 uint32_t n_instructions,
6327 uint32_t *n_pattern_instr)
6330 uint32_t src_field_id, i, j;
6332 /* Prerequisites. */
6336 /* First instruction: MOV_HM. */
6337 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6340 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6341 if (!h || h->st->var_size)
6344 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6345 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6348 if (src_field_id == a->st->n_fields)
6351 if (instr[0].mov.dst.offset ||
6352 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6353 instr[0].mov.src.struct_id ||
6354 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6355 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6358 if ((n_instr < h->st->n_fields + 1) ||
6359 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6362 /* Subsequent instructions: MOV_HM. */
6363 for (i = 1; i < h->st->n_fields; i++)
6364 if (data[i].invalid ||
6366 (instr[i].type != INSTR_MOV_HM) ||
6367 (instr[i].mov.dst.struct_id != h->struct_id) ||
6368 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6369 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6370 instr[i].mov.src.struct_id ||
6371 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6372 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6373 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6376 /* Last instruction: HDR_VALIDATE. */
6377 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6378 (instr[i].valid.header_id != h->id))
6381 /* Check that none of the action args that are used as source for this
6382 * DMA transfer are not used as source in any other mov instruction.
6384 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6387 n_users = action_arg_src_mov_count(a,
6396 *n_pattern_instr = 1 + i;
6401 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6403 struct instruction *instr,
6404 struct instruction_data *data,
6408 uint32_t src_field_id, src_offset, i;
6410 /* Read from the instructions before they are modified. */
6411 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6415 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6416 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6419 if (src_field_id == a->st->n_fields)
6422 src_offset = instr[0].mov.src.offset;
6424 /* Modify the instructions. */
6425 instr[0].type = INSTR_DMA_HT;
6426 instr[0].dma.dst.header_id[0] = h->id;
6427 instr[0].dma.dst.struct_id[0] = h->struct_id;
6428 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6429 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6431 for (i = 1; i < n_instr; i++)
6432 data[i].invalid = 1;
6434 /* Update the endianness of the action arguments to header endianness. */
6435 for (i = 0; i < h->st->n_fields; i++)
6436 a->args_endianness[src_field_id + i] = 1;
6440 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6442 struct instruction *instructions,
6443 struct instruction_data *instruction_data,
6444 uint32_t n_instructions)
6449 return n_instructions;
6451 for (i = 0; i < n_instructions; ) {
6452 struct instruction *instr = &instructions[i];
6453 struct instruction_data *data = &instruction_data[i];
6454 uint32_t n_instr = 0;
6457 /* Mov all + validate. */
6458 detected = instr_pattern_mov_all_validate_search(p,
6468 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6473 /* No pattern starting at the current instruction. */
6477 /* Eliminate the invalid instructions that have been optimized out. */
6478 n_instructions = instr_compact(instructions,
6482 return n_instructions;
6486 instr_pattern_dma_many_search(struct instruction *instr,
6487 struct instruction_data *data,
6489 uint32_t *n_pattern_instr)
6493 for (i = 0; i < n_instr; i++) {
6494 if (data[i].invalid)
6497 if (instr[i].type != INSTR_DMA_HT)
6500 if (i == RTE_DIM(instr->dma.dst.header_id))
6503 if (i && data[i].n_users)
6510 *n_pattern_instr = i;
6515 instr_pattern_dma_many_replace(struct instruction *instr,
6516 struct instruction_data *data,
6521 for (i = 1; i < n_instr; i++) {
6523 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6524 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6525 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6526 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6528 data[i].invalid = 1;
6533 instr_pattern_dma_many_optimize(struct instruction *instructions,
6534 struct instruction_data *instruction_data,
6535 uint32_t n_instructions)
6539 for (i = 0; i < n_instructions; ) {
6540 struct instruction *instr = &instructions[i];
6541 struct instruction_data *data = &instruction_data[i];
6542 uint32_t n_instr = 0;
6546 detected = instr_pattern_dma_many_search(instr,
6551 instr_pattern_dma_many_replace(instr, data, n_instr);
6556 /* No pattern starting at the current instruction. */
6560 /* Eliminate the invalid instructions that have been optimized out. */
6561 n_instructions = instr_compact(instructions,
6565 return n_instructions;
6569 instr_optimize(struct rte_swx_pipeline *p,
6571 struct instruction *instructions,
6572 struct instruction_data *instruction_data,
6573 uint32_t n_instructions)
6576 n_instructions = instr_pattern_extract_many_optimize(instructions,
6580 /* Emit many + TX. */
6581 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6585 /* Mov all + validate. */
6586 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6593 n_instructions = instr_pattern_dma_many_optimize(instructions,
6597 return n_instructions;
6601 instruction_config(struct rte_swx_pipeline *p,
6603 const char **instructions,
6604 uint32_t n_instructions)
6606 struct instruction *instr = NULL;
6607 struct instruction_data *data = NULL;
6611 CHECK(n_instructions, EINVAL);
6612 CHECK(instructions, EINVAL);
6613 for (i = 0; i < n_instructions; i++)
6614 CHECK_INSTRUCTION(instructions[i], EINVAL);
6616 /* Memory allocation. */
6617 instr = calloc(n_instructions, sizeof(struct instruction));
6623 data = calloc(n_instructions, sizeof(struct instruction_data));
6629 for (i = 0; i < n_instructions; i++) {
6630 char *string = strdup(instructions[i]);
6636 err = instr_translate(p, a, string, &instr[i], &data[i]);
6645 err = instr_label_check(data, n_instructions);
6649 err = instr_verify(p, a, instr, data, n_instructions);
6653 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6655 err = instr_jmp_resolve(instr, data, n_instructions);
6660 a->instructions = instr;
6661 a->instruction_data = data;
6662 a->n_instructions = n_instructions;
6664 p->instructions = instr;
6665 p->instruction_data = data;
6666 p->n_instructions = n_instructions;
6677 static instr_exec_t instruction_table[] = {
6678 [INSTR_RX] = instr_rx_exec,
6679 [INSTR_TX] = instr_tx_exec,
6680 [INSTR_TX_I] = instr_tx_i_exec,
6681 [INSTR_DROP] = instr_drop_exec,
6683 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6684 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6685 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6686 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6687 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6688 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6689 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6690 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6691 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6692 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6694 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6695 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6696 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6697 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6698 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6699 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6700 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6701 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6702 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6704 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6705 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6707 [INSTR_MOV] = instr_mov_exec,
6708 [INSTR_MOV_MH] = instr_mov_mh_exec,
6709 [INSTR_MOV_HM] = instr_mov_hm_exec,
6710 [INSTR_MOV_HH] = instr_mov_hh_exec,
6711 [INSTR_MOV_I] = instr_mov_i_exec,
6713 [INSTR_DMA_HT] = instr_dma_ht_exec,
6714 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6715 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6716 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6717 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6718 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6719 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6720 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6722 [INSTR_ALU_ADD] = instr_alu_add_exec,
6723 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6724 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6725 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6726 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6727 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6729 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6730 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6731 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6732 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6733 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6734 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6736 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6737 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6738 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6739 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6741 [INSTR_ALU_AND] = instr_alu_and_exec,
6742 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6743 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6744 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6745 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6747 [INSTR_ALU_OR] = instr_alu_or_exec,
6748 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6749 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6750 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6751 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6753 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6754 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6755 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6756 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6757 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6759 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6760 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6761 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6762 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6763 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6764 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6766 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6767 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6768 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6769 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6770 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6771 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6773 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6774 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6775 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6777 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6778 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6779 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6780 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6781 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6782 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6784 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6785 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6786 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6787 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6788 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6789 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6790 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6791 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6792 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6794 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6795 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6796 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6797 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6798 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6799 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6800 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6801 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6802 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6804 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6805 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6806 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6808 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6809 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6810 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6811 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6812 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6813 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6814 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6815 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6816 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6817 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6818 [INSTR_METER_IMM] = instr_meter_imm_exec,
6819 [INSTR_METER_IMI] = instr_meter_imi_exec,
6821 [INSTR_TABLE] = instr_table_exec,
6822 [INSTR_TABLE_AF] = instr_table_af_exec,
6823 [INSTR_SELECTOR] = instr_selector_exec,
6824 [INSTR_LEARNER] = instr_learner_exec,
6825 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6826 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6827 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6828 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6829 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6831 [INSTR_JMP] = instr_jmp_exec,
6832 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6833 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6834 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6835 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6836 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6837 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6839 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6840 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6841 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6842 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6843 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6845 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6846 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6847 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6848 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6849 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6851 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6852 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6853 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6854 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6855 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6856 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6858 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6859 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6860 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6861 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6862 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6863 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6865 [INSTR_RETURN] = instr_return_exec,
6869 instruction_table_build(struct rte_swx_pipeline *p)
6871 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6872 sizeof(struct instr_exec_t *));
6873 if (!p->instruction_table)
6876 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6882 instruction_table_build_free(struct rte_swx_pipeline *p)
6884 if (!p->instruction_table)
6887 free(p->instruction_table);
6888 p->instruction_table = NULL;
6892 instruction_table_free(struct rte_swx_pipeline *p)
6894 instruction_table_build_free(p);
6898 instr_exec(struct rte_swx_pipeline *p)
6900 struct thread *t = &p->threads[p->thread_id];
6901 struct instruction *ip = t->ip;
6902 instr_exec_t instr = p->instruction_table[ip->type];
6910 static struct action *
6911 action_find(struct rte_swx_pipeline *p, const char *name)
6913 struct action *elem;
6918 TAILQ_FOREACH(elem, &p->actions, node)
6919 if (strcmp(elem->name, name) == 0)
6925 static struct action *
6926 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6928 struct action *action = NULL;
6930 TAILQ_FOREACH(action, &p->actions, node)
6931 if (action->id == id)
6937 static struct field *
6938 action_field_find(struct action *a, const char *name)
6940 return a->st ? struct_type_field_find(a->st, name) : NULL;
6943 static struct field *
6944 action_field_parse(struct action *action, const char *name)
6946 if (name[0] != 't' || name[1] != '.')
6949 return action_field_find(action, &name[2]);
6953 action_has_nbo_args(struct action *a)
6957 /* Return if the action does not have any args. */
6959 return 0; /* FALSE */
6961 for (i = 0; i < a->st->n_fields; i++)
6962 if (a->args_endianness[i])
6963 return 1; /* TRUE */
6965 return 0; /* FALSE */
6969 action_does_learning(struct action *a)
6973 for (i = 0; i < a->n_instructions; i++)
6974 switch (a->instructions[i].type) {
6975 case INSTR_LEARNER_LEARN:
6976 return 1; /* TRUE */
6978 case INSTR_LEARNER_FORGET:
6979 return 1; /* TRUE */
6985 return 0; /* FALSE */
6989 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6991 const char *args_struct_type_name,
6992 const char **instructions,
6993 uint32_t n_instructions)
6995 struct struct_type *args_struct_type = NULL;
7001 CHECK_NAME(name, EINVAL);
7002 CHECK(!action_find(p, name), EEXIST);
7004 if (args_struct_type_name) {
7005 CHECK_NAME(args_struct_type_name, EINVAL);
7006 args_struct_type = struct_type_find(p, args_struct_type_name);
7007 CHECK(args_struct_type, EINVAL);
7008 CHECK(!args_struct_type->var_size, EINVAL);
7011 /* Node allocation. */
7012 a = calloc(1, sizeof(struct action));
7014 if (args_struct_type) {
7015 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7016 if (!a->args_endianness) {
7022 /* Node initialization. */
7023 strcpy(a->name, name);
7024 a->st = args_struct_type;
7025 a->id = p->n_actions;
7027 /* Instruction translation. */
7028 err = instruction_config(p, a, instructions, n_instructions);
7030 free(a->args_endianness);
7035 /* Node add to tailq. */
7036 TAILQ_INSERT_TAIL(&p->actions, a, node);
7043 action_build(struct rte_swx_pipeline *p)
7045 struct action *action;
7047 /* p->action_instructions. */
7048 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7049 CHECK(p->action_instructions, ENOMEM);
7051 TAILQ_FOREACH(action, &p->actions, node)
7052 p->action_instructions[action->id] = action->instructions;
7054 /* p->action_funcs. */
7055 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7056 CHECK(p->action_funcs, ENOMEM);
7062 action_build_free(struct rte_swx_pipeline *p)
7064 free(p->action_funcs);
7065 p->action_funcs = NULL;
7067 free(p->action_instructions);
7068 p->action_instructions = NULL;
7072 action_free(struct rte_swx_pipeline *p)
7074 action_build_free(p);
7077 struct action *action;
7079 action = TAILQ_FIRST(&p->actions);
7083 TAILQ_REMOVE(&p->actions, action, node);
7084 free(action->instruction_data);
7085 free(action->instructions);
7091 action_arg_src_mov_count(struct action *a,
7093 struct instruction *instructions,
7094 struct instruction_data *instruction_data,
7095 uint32_t n_instructions)
7097 uint32_t offset, n_users = 0, i;
7100 (arg_id >= a->st->n_fields) ||
7102 !instruction_data ||
7106 offset = a->st->fields[arg_id].offset / 8;
7108 for (i = 0; i < n_instructions; i++) {
7109 struct instruction *instr = &instructions[i];
7110 struct instruction_data *data = &instruction_data[i];
7112 if (data->invalid ||
7113 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7114 instr->mov.src.struct_id ||
7115 (instr->mov.src.offset != offset))
7127 static struct table_type *
7128 table_type_find(struct rte_swx_pipeline *p, const char *name)
7130 struct table_type *elem;
7132 TAILQ_FOREACH(elem, &p->table_types, node)
7133 if (strcmp(elem->name, name) == 0)
7139 static struct table_type *
7140 table_type_resolve(struct rte_swx_pipeline *p,
7141 const char *recommended_type_name,
7142 enum rte_swx_table_match_type match_type)
7144 struct table_type *elem;
7146 /* Only consider the recommended type if the match type is correct. */
7147 if (recommended_type_name)
7148 TAILQ_FOREACH(elem, &p->table_types, node)
7149 if (!strcmp(elem->name, recommended_type_name) &&
7150 (elem->match_type == match_type))
7153 /* Ignore the recommended type and get the first element with this match
7156 TAILQ_FOREACH(elem, &p->table_types, node)
7157 if (elem->match_type == match_type)
7163 static struct table *
7164 table_find(struct rte_swx_pipeline *p, const char *name)
7168 TAILQ_FOREACH(elem, &p->tables, node)
7169 if (strcmp(elem->name, name) == 0)
7175 static struct table *
7176 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7178 struct table *table = NULL;
7180 TAILQ_FOREACH(table, &p->tables, node)
7181 if (table->id == id)
7188 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7190 enum rte_swx_table_match_type match_type,
7191 struct rte_swx_table_ops *ops)
7193 struct table_type *elem;
7197 CHECK_NAME(name, EINVAL);
7198 CHECK(!table_type_find(p, name), EEXIST);
7201 CHECK(ops->create, EINVAL);
7202 CHECK(ops->lkp, EINVAL);
7203 CHECK(ops->free, EINVAL);
7205 /* Node allocation. */
7206 elem = calloc(1, sizeof(struct table_type));
7207 CHECK(elem, ENOMEM);
7209 /* Node initialization. */
7210 strcpy(elem->name, name);
7211 elem->match_type = match_type;
7212 memcpy(&elem->ops, ops, sizeof(*ops));
7214 /* Node add to tailq. */
7215 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7221 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7223 enum rte_swx_table_match_type *match_type)
7225 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7227 for (i = 0; i < n_fields; i++) {
7228 struct rte_swx_match_field_params *f = &fields[i];
7230 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7233 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7237 if ((n_fields_lpm > 1) ||
7238 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7241 *match_type = (n_fields_em == n_fields) ?
7242 RTE_SWX_TABLE_MATCH_EXACT :
7243 RTE_SWX_TABLE_MATCH_WILDCARD;
7249 table_match_fields_check(struct rte_swx_pipeline *p,
7250 struct rte_swx_pipeline_table_params *params,
7251 struct header **header)
7253 struct header *h0 = NULL;
7254 struct field *hf, *mf;
7255 uint32_t *offset = NULL, i;
7258 /* Return if no match fields. */
7259 if (!params->n_fields) {
7260 if (params->fields) {
7271 /* Memory allocation. */
7272 offset = calloc(params->n_fields, sizeof(uint32_t));
7278 /* Check that all the match fields belong to either the same header or
7281 hf = header_field_parse(p, params->fields[0].name, &h0);
7282 mf = metadata_field_parse(p, params->fields[0].name);
7283 if ((!hf && !mf) || (hf && hf->var_size)) {
7288 offset[0] = h0 ? hf->offset : mf->offset;
7290 for (i = 1; i < params->n_fields; i++)
7294 hf = header_field_parse(p, params->fields[i].name, &h);
7295 if (!hf || (h->id != h0->id) || hf->var_size) {
7300 offset[i] = hf->offset;
7302 mf = metadata_field_parse(p, params->fields[i].name);
7308 offset[i] = mf->offset;
7311 /* Check that there are no duplicated match fields. */
7312 for (i = 0; i < params->n_fields; i++) {
7315 for (j = 0; j < i; j++)
7316 if (offset[j] == offset[i]) {
7332 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7334 struct rte_swx_pipeline_table_params *params,
7335 const char *recommended_table_type_name,
7339 struct table_type *type;
7340 struct table *t = NULL;
7341 struct action *default_action;
7342 struct header *header = NULL;
7343 uint32_t action_data_size_max = 0, i;
7348 CHECK_NAME(name, EINVAL);
7349 CHECK(!table_find(p, name), EEXIST);
7350 CHECK(!selector_find(p, name), EEXIST);
7351 CHECK(!learner_find(p, name), EEXIST);
7353 CHECK(params, EINVAL);
7356 status = table_match_fields_check(p, params, &header);
7360 /* Action checks. */
7361 CHECK(params->n_actions, EINVAL);
7362 CHECK(params->action_names, EINVAL);
7363 for (i = 0; i < params->n_actions; i++) {
7364 const char *action_name = params->action_names[i];
7366 uint32_t action_data_size;
7367 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7369 CHECK_NAME(action_name, EINVAL);
7371 a = action_find(p, action_name);
7373 CHECK(!action_does_learning(a), EINVAL);
7375 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7376 if (action_data_size > action_data_size_max)
7377 action_data_size_max = action_data_size;
7379 if (params->action_is_for_table_entries)
7380 action_is_for_table_entries = params->action_is_for_table_entries[i];
7381 if (params->action_is_for_default_entry)
7382 action_is_for_default_entry = params->action_is_for_default_entry[i];
7383 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7386 CHECK_NAME(params->default_action_name, EINVAL);
7387 for (i = 0; i < p->n_actions; i++)
7388 if (!strcmp(params->action_names[i],
7389 params->default_action_name))
7391 CHECK(i < params->n_actions, EINVAL);
7392 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7395 default_action = action_find(p, params->default_action_name);
7396 CHECK((default_action->st && params->default_action_data) ||
7397 !params->default_action_data, EINVAL);
7399 /* Table type checks. */
7400 if (recommended_table_type_name)
7401 CHECK_NAME(recommended_table_type_name, EINVAL);
7403 if (params->n_fields) {
7404 enum rte_swx_table_match_type match_type;
7406 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7410 type = table_type_resolve(p, recommended_table_type_name, match_type);
7411 CHECK(type, EINVAL);
7416 /* Memory allocation. */
7417 t = calloc(1, sizeof(struct table));
7421 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7425 t->actions = calloc(params->n_actions, sizeof(struct action *));
7429 if (action_data_size_max) {
7430 t->default_action_data = calloc(1, action_data_size_max);
7431 if (!t->default_action_data)
7435 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7436 if (!t->action_is_for_table_entries)
7439 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7440 if (!t->action_is_for_default_entry)
7443 /* Node initialization. */
7444 strcpy(t->name, name);
7445 if (args && args[0])
7446 strcpy(t->args, args);
7449 for (i = 0; i < params->n_fields; i++) {
7450 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7451 struct match_field *f = &t->fields[i];
7453 f->match_type = field->match_type;
7455 header_field_parse(p, field->name, NULL) :
7456 metadata_field_parse(p, field->name);
7458 t->n_fields = params->n_fields;
7461 for (i = 0; i < params->n_actions; i++) {
7462 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7464 if (params->action_is_for_table_entries)
7465 action_is_for_table_entries = params->action_is_for_table_entries[i];
7466 if (params->action_is_for_default_entry)
7467 action_is_for_default_entry = params->action_is_for_default_entry[i];
7469 t->actions[i] = action_find(p, params->action_names[i]);
7470 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7471 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7473 t->default_action = default_action;
7474 if (default_action->st)
7475 memcpy(t->default_action_data,
7476 params->default_action_data,
7477 default_action->st->n_bits / 8);
7478 t->n_actions = params->n_actions;
7479 t->default_action_is_const = params->default_action_is_const;
7480 t->action_data_size_max = action_data_size_max;
7483 t->id = p->n_tables;
7485 /* Node add to tailq. */
7486 TAILQ_INSERT_TAIL(&p->tables, t, node);
7495 free(t->action_is_for_default_entry);
7496 free(t->action_is_for_table_entries);
7497 free(t->default_action_data);
7505 static struct rte_swx_table_params *
7506 table_params_get(struct table *table)
7508 struct rte_swx_table_params *params;
7509 struct field *first, *last;
7511 uint32_t key_size, key_offset, action_data_size, i;
7513 /* Memory allocation. */
7514 params = calloc(1, sizeof(struct rte_swx_table_params));
7518 /* Find first (smallest offset) and last (biggest offset) match fields. */
7519 first = table->fields[0].field;
7520 last = table->fields[0].field;
7522 for (i = 0; i < table->n_fields; i++) {
7523 struct field *f = table->fields[i].field;
7525 if (f->offset < first->offset)
7528 if (f->offset > last->offset)
7532 /* Key offset and size. */
7533 key_offset = first->offset / 8;
7534 key_size = (last->offset + last->n_bits - first->offset) / 8;
7536 /* Memory allocation. */
7537 key_mask = calloc(1, key_size);
7544 for (i = 0; i < table->n_fields; i++) {
7545 struct field *f = table->fields[i].field;
7546 uint32_t start = (f->offset - first->offset) / 8;
7547 size_t size = f->n_bits / 8;
7549 memset(&key_mask[start], 0xFF, size);
7552 /* Action data size. */
7553 action_data_size = 0;
7554 for (i = 0; i < table->n_actions; i++) {
7555 struct action *action = table->actions[i];
7556 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7558 if (ads > action_data_size)
7559 action_data_size = ads;
7563 params->match_type = table->type->match_type;
7564 params->key_size = key_size;
7565 params->key_offset = key_offset;
7566 params->key_mask0 = key_mask;
7567 params->action_data_size = action_data_size;
7568 params->n_keys_max = table->size;
7574 table_params_free(struct rte_swx_table_params *params)
7579 free(params->key_mask0);
7584 table_stub_lkp(void *table __rte_unused,
7585 void *mailbox __rte_unused,
7586 uint8_t **key __rte_unused,
7587 uint64_t *action_id __rte_unused,
7588 uint8_t **action_data __rte_unused,
7592 return 1; /* DONE. */
7596 table_build(struct rte_swx_pipeline *p)
7600 /* Per pipeline: table statistics. */
7601 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7602 CHECK(p->table_stats, ENOMEM);
7604 for (i = 0; i < p->n_tables; i++) {
7605 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7606 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7609 /* Per thread: table runt-time. */
7610 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7611 struct thread *t = &p->threads[i];
7612 struct table *table;
7614 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7615 CHECK(t->tables, ENOMEM);
7617 TAILQ_FOREACH(table, &p->tables, node) {
7618 struct table_runtime *r = &t->tables[table->id];
7623 size = table->type->ops.mailbox_size_get();
7626 r->func = table->type->ops.lkp;
7630 r->mailbox = calloc(1, size);
7631 CHECK(r->mailbox, ENOMEM);
7635 r->key = table->header ?
7636 &t->structs[table->header->struct_id] :
7637 &t->structs[p->metadata_struct_id];
7639 r->func = table_stub_lkp;
7648 table_build_free(struct rte_swx_pipeline *p)
7652 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7653 struct thread *t = &p->threads[i];
7659 for (j = 0; j < p->n_tables; j++) {
7660 struct table_runtime *r = &t->tables[j];
7669 if (p->table_stats) {
7670 for (i = 0; i < p->n_tables; i++)
7671 free(p->table_stats[i].n_pkts_action);
7673 free(p->table_stats);
7678 table_free(struct rte_swx_pipeline *p)
7680 table_build_free(p);
7686 elem = TAILQ_FIRST(&p->tables);
7690 TAILQ_REMOVE(&p->tables, elem, node);
7692 free(elem->actions);
7693 free(elem->default_action_data);
7699 struct table_type *elem;
7701 elem = TAILQ_FIRST(&p->table_types);
7705 TAILQ_REMOVE(&p->table_types, elem, node);
7713 static struct selector *
7714 selector_find(struct rte_swx_pipeline *p, const char *name)
7718 TAILQ_FOREACH(s, &p->selectors, node)
7719 if (strcmp(s->name, name) == 0)
7725 static struct selector *
7726 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7728 struct selector *s = NULL;
7730 TAILQ_FOREACH(s, &p->selectors, node)
7738 selector_fields_check(struct rte_swx_pipeline *p,
7739 struct rte_swx_pipeline_selector_params *params,
7740 struct header **header)
7742 struct header *h0 = NULL;
7743 struct field *hf, *mf;
7746 /* Return if no selector fields. */
7747 if (!params->n_selector_fields || !params->selector_field_names)
7750 /* Check that all the selector fields either belong to the same header
7751 * or are all meta-data fields.
7753 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7754 mf = metadata_field_parse(p, params->selector_field_names[0]);
7758 for (i = 1; i < params->n_selector_fields; i++)
7762 hf = header_field_parse(p, params->selector_field_names[i], &h);
7763 if (!hf || (h->id != h0->id))
7766 mf = metadata_field_parse(p, params->selector_field_names[i]);
7771 /* Check that there are no duplicated match fields. */
7772 for (i = 0; i < params->n_selector_fields; i++) {
7773 const char *field_name = params->selector_field_names[i];
7776 for (j = i + 1; j < params->n_selector_fields; j++)
7777 if (!strcmp(params->selector_field_names[j], field_name))
7789 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7791 struct rte_swx_pipeline_selector_params *params)
7794 struct header *selector_header = NULL;
7795 struct field *group_id_field, *member_id_field;
7801 CHECK_NAME(name, EINVAL);
7802 CHECK(!table_find(p, name), EEXIST);
7803 CHECK(!selector_find(p, name), EEXIST);
7804 CHECK(!learner_find(p, name), EEXIST);
7806 CHECK(params, EINVAL);
7808 CHECK_NAME(params->group_id_field_name, EINVAL);
7809 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7810 CHECK(group_id_field, EINVAL);
7812 for (i = 0; i < params->n_selector_fields; i++) {
7813 const char *field_name = params->selector_field_names[i];
7815 CHECK_NAME(field_name, EINVAL);
7817 status = selector_fields_check(p, params, &selector_header);
7821 CHECK_NAME(params->member_id_field_name, EINVAL);
7822 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7823 CHECK(member_id_field, EINVAL);
7825 CHECK(params->n_groups_max, EINVAL);
7827 CHECK(params->n_members_per_group_max, EINVAL);
7829 /* Memory allocation. */
7830 s = calloc(1, sizeof(struct selector));
7836 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7837 if (!s->selector_fields) {
7842 /* Node initialization. */
7843 strcpy(s->name, name);
7845 s->group_id_field = group_id_field;
7847 for (i = 0; i < params->n_selector_fields; i++) {
7848 const char *field_name = params->selector_field_names[i];
7850 s->selector_fields[i] = selector_header ?
7851 header_field_parse(p, field_name, NULL) :
7852 metadata_field_parse(p, field_name);
7855 s->n_selector_fields = params->n_selector_fields;
7857 s->selector_header = selector_header;
7859 s->member_id_field = member_id_field;
7861 s->n_groups_max = params->n_groups_max;
7863 s->n_members_per_group_max = params->n_members_per_group_max;
7865 s->id = p->n_selectors;
7867 /* Node add to tailq. */
7868 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7877 free(s->selector_fields);
7885 selector_params_free(struct rte_swx_table_selector_params *params)
7890 free(params->selector_mask);
7895 static struct rte_swx_table_selector_params *
7896 selector_table_params_get(struct selector *s)
7898 struct rte_swx_table_selector_params *params = NULL;
7899 struct field *first, *last;
7902 /* Memory allocation. */
7903 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7908 params->group_id_offset = s->group_id_field->offset / 8;
7910 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7911 first = s->selector_fields[0];
7912 last = s->selector_fields[0];
7914 for (i = 0; i < s->n_selector_fields; i++) {
7915 struct field *f = s->selector_fields[i];
7917 if (f->offset < first->offset)
7920 if (f->offset > last->offset)
7924 /* Selector offset and size. */
7925 params->selector_offset = first->offset / 8;
7926 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7928 /* Memory allocation. */
7929 params->selector_mask = calloc(1, params->selector_size);
7930 if (!params->selector_mask)
7933 /* Selector mask. */
7934 for (i = 0; i < s->n_selector_fields; i++) {
7935 struct field *f = s->selector_fields[i];
7936 uint32_t start = (f->offset - first->offset) / 8;
7937 size_t size = f->n_bits / 8;
7939 memset(¶ms->selector_mask[start], 0xFF, size);
7943 params->member_id_offset = s->member_id_field->offset / 8;
7945 /* Maximum number of groups. */
7946 params->n_groups_max = s->n_groups_max;
7948 /* Maximum number of members per group. */
7949 params->n_members_per_group_max = s->n_members_per_group_max;
7954 selector_params_free(params);
7959 selector_build_free(struct rte_swx_pipeline *p)
7963 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7964 struct thread *t = &p->threads[i];
7970 for (j = 0; j < p->n_selectors; j++) {
7971 struct selector_runtime *r = &t->selectors[j];
7977 t->selectors = NULL;
7980 free(p->selector_stats);
7981 p->selector_stats = NULL;
7985 selector_build(struct rte_swx_pipeline *p)
7990 /* Per pipeline: selector statistics. */
7991 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7992 if (!p->selector_stats) {
7997 /* Per thread: selector run-time. */
7998 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7999 struct thread *t = &p->threads[i];
8002 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8003 if (!t->selectors) {
8008 TAILQ_FOREACH(s, &p->selectors, node) {
8009 struct selector_runtime *r = &t->selectors[s->id];
8013 size = rte_swx_table_selector_mailbox_size_get();
8015 r->mailbox = calloc(1, size);
8022 /* r->group_id_buffer. */
8023 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8025 /* r->selector_buffer. */
8026 r->selector_buffer = s->selector_header ?
8027 &t->structs[s->selector_header->struct_id] :
8028 &t->structs[p->metadata_struct_id];
8030 /* r->member_id_buffer. */
8031 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8038 selector_build_free(p);
8043 selector_free(struct rte_swx_pipeline *p)
8045 selector_build_free(p);
8047 /* Selector tables. */
8049 struct selector *elem;
8051 elem = TAILQ_FIRST(&p->selectors);
8055 TAILQ_REMOVE(&p->selectors, elem, node);
8056 free(elem->selector_fields);
8064 static struct learner *
8065 learner_find(struct rte_swx_pipeline *p, const char *name)
8069 TAILQ_FOREACH(l, &p->learners, node)
8070 if (!strcmp(l->name, name))
8076 static struct learner *
8077 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8079 struct learner *l = NULL;
8081 TAILQ_FOREACH(l, &p->learners, node)
8089 learner_match_fields_check(struct rte_swx_pipeline *p,
8090 struct rte_swx_pipeline_learner_params *params,
8091 struct header **header)
8093 struct header *h0 = NULL;
8094 struct field *hf, *mf;
8097 /* Return if no match fields. */
8098 if (!params->n_fields || !params->field_names)
8101 /* Check that all the match fields either belong to the same header
8102 * or are all meta-data fields.
8104 hf = header_field_parse(p, params->field_names[0], &h0);
8105 mf = metadata_field_parse(p, params->field_names[0]);
8109 for (i = 1; i < params->n_fields; i++)
8113 hf = header_field_parse(p, params->field_names[i], &h);
8114 if (!hf || (h->id != h0->id))
8117 mf = metadata_field_parse(p, params->field_names[i]);
8122 /* Check that there are no duplicated match fields. */
8123 for (i = 0; i < params->n_fields; i++) {
8124 const char *field_name = params->field_names[i];
8127 for (j = i + 1; j < params->n_fields; j++)
8128 if (!strcmp(params->field_names[j], field_name))
8140 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8142 struct struct_type *mst = p->metadata_st, *ast = a->st;
8143 struct field *mf, *af;
8153 /* Check that mf_name is the name of a valid meta-data field. */
8154 CHECK_NAME(mf_name, EINVAL);
8155 mf = metadata_field_parse(p, mf_name);
8158 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8159 * all the action arguments.
8161 mf_pos = mf - mst->fields;
8162 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8164 /* Check that the size of each of the identified meta-data fields matches exactly the size
8165 * of the corresponding action argument.
8167 for (i = 0; i < ast->n_fields; i++) {
8168 mf = &mst->fields[mf_pos + i];
8169 af = &ast->fields[i];
8171 CHECK(mf->n_bits == af->n_bits, EINVAL);
8178 learner_action_learning_check(struct rte_swx_pipeline *p,
8179 struct action *action,
8180 const char **action_names,
8185 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8186 * the action passed as argument to the "learn" instruction) is also enabled for the
8187 * current learner table.
8189 for (i = 0; i < action->n_instructions; i++) {
8190 struct instruction *instr = &action->instructions[i];
8191 uint32_t found = 0, j;
8193 if (instr->type != INSTR_LEARNER_LEARN)
8196 for (j = 0; j < n_actions; j++) {
8199 a = action_find(p, action_names[j]);
8203 if (a->id == instr->learn.action_id)
8215 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8217 struct rte_swx_pipeline_learner_params *params,
8221 struct learner *l = NULL;
8222 struct action *default_action;
8223 struct header *header = NULL;
8224 uint32_t action_data_size_max = 0, i;
8229 CHECK_NAME(name, EINVAL);
8230 CHECK(!table_find(p, name), EEXIST);
8231 CHECK(!selector_find(p, name), EEXIST);
8232 CHECK(!learner_find(p, name), EEXIST);
8234 CHECK(params, EINVAL);
8237 status = learner_match_fields_check(p, params, &header);
8241 /* Action checks. */
8242 CHECK(params->n_actions, EINVAL);
8243 CHECK(params->action_names, EINVAL);
8244 for (i = 0; i < params->n_actions; i++) {
8245 const char *action_name = params->action_names[i];
8247 uint32_t action_data_size;
8248 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8250 CHECK_NAME(action_name, EINVAL);
8252 a = action_find(p, action_name);
8255 status = learner_action_learning_check(p,
8257 params->action_names,
8262 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8263 if (action_data_size > action_data_size_max)
8264 action_data_size_max = action_data_size;
8266 if (params->action_is_for_table_entries)
8267 action_is_for_table_entries = params->action_is_for_table_entries[i];
8268 if (params->action_is_for_default_entry)
8269 action_is_for_default_entry = params->action_is_for_default_entry[i];
8270 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8273 CHECK_NAME(params->default_action_name, EINVAL);
8274 for (i = 0; i < p->n_actions; i++)
8275 if (!strcmp(params->action_names[i],
8276 params->default_action_name))
8278 CHECK(i < params->n_actions, EINVAL);
8279 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8282 default_action = action_find(p, params->default_action_name);
8283 CHECK((default_action->st && params->default_action_data) ||
8284 !params->default_action_data, EINVAL);
8286 /* Any other checks. */
8287 CHECK(size, EINVAL);
8288 CHECK(timeout, EINVAL);
8290 /* Memory allocation. */
8291 l = calloc(1, sizeof(struct learner));
8295 l->fields = calloc(params->n_fields, sizeof(struct field *));
8299 l->actions = calloc(params->n_actions, sizeof(struct action *));
8303 if (action_data_size_max) {
8304 l->default_action_data = calloc(1, action_data_size_max);
8305 if (!l->default_action_data)
8309 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8310 if (!l->action_is_for_table_entries)
8313 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8314 if (!l->action_is_for_default_entry)
8317 /* Node initialization. */
8318 strcpy(l->name, name);
8320 for (i = 0; i < params->n_fields; i++) {
8321 const char *field_name = params->field_names[i];
8323 l->fields[i] = header ?
8324 header_field_parse(p, field_name, NULL) :
8325 metadata_field_parse(p, field_name);
8328 l->n_fields = params->n_fields;
8332 for (i = 0; i < params->n_actions; i++) {
8333 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8335 if (params->action_is_for_table_entries)
8336 action_is_for_table_entries = params->action_is_for_table_entries[i];
8337 if (params->action_is_for_default_entry)
8338 action_is_for_default_entry = params->action_is_for_default_entry[i];
8340 l->actions[i] = action_find(p, params->action_names[i]);
8341 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8342 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8345 l->default_action = default_action;
8347 if (default_action->st)
8348 memcpy(l->default_action_data,
8349 params->default_action_data,
8350 default_action->st->n_bits / 8);
8352 l->n_actions = params->n_actions;
8354 l->default_action_is_const = params->default_action_is_const;
8356 l->action_data_size_max = action_data_size_max;
8360 l->timeout = timeout;
8362 l->id = p->n_learners;
8364 /* Node add to tailq. */
8365 TAILQ_INSERT_TAIL(&p->learners, l, node);
8374 free(l->action_is_for_default_entry);
8375 free(l->action_is_for_table_entries);
8376 free(l->default_action_data);
8385 learner_params_free(struct rte_swx_table_learner_params *params)
8390 free(params->key_mask0);
8395 static struct rte_swx_table_learner_params *
8396 learner_params_get(struct learner *l)
8398 struct rte_swx_table_learner_params *params = NULL;
8399 struct field *first, *last;
8402 /* Memory allocation. */
8403 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8407 /* Find first (smallest offset) and last (biggest offset) match fields. */
8408 first = l->fields[0];
8409 last = l->fields[0];
8411 for (i = 0; i < l->n_fields; i++) {
8412 struct field *f = l->fields[i];
8414 if (f->offset < first->offset)
8417 if (f->offset > last->offset)
8421 /* Key offset and size. */
8422 params->key_offset = first->offset / 8;
8423 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8425 /* Memory allocation. */
8426 params->key_mask0 = calloc(1, params->key_size);
8427 if (!params->key_mask0)
8431 for (i = 0; i < l->n_fields; i++) {
8432 struct field *f = l->fields[i];
8433 uint32_t start = (f->offset - first->offset) / 8;
8434 size_t size = f->n_bits / 8;
8436 memset(¶ms->key_mask0[start], 0xFF, size);
8439 /* Action data size. */
8440 params->action_data_size = l->action_data_size_max;
8442 /* Maximum number of keys. */
8443 params->n_keys_max = l->size;
8446 params->key_timeout = l->timeout;
8451 learner_params_free(params);
8456 learner_build_free(struct rte_swx_pipeline *p)
8460 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8461 struct thread *t = &p->threads[i];
8467 for (j = 0; j < p->n_learners; j++) {
8468 struct learner_runtime *r = &t->learners[j];
8477 if (p->learner_stats) {
8478 for (i = 0; i < p->n_learners; i++)
8479 free(p->learner_stats[i].n_pkts_action);
8481 free(p->learner_stats);
8486 learner_build(struct rte_swx_pipeline *p)
8491 /* Per pipeline: learner statistics. */
8492 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8493 CHECK(p->learner_stats, ENOMEM);
8495 for (i = 0; i < p->n_learners; i++) {
8496 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8497 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8500 /* Per thread: learner run-time. */
8501 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8502 struct thread *t = &p->threads[i];
8505 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8511 TAILQ_FOREACH(l, &p->learners, node) {
8512 struct learner_runtime *r = &t->learners[l->id];
8516 size = rte_swx_table_learner_mailbox_size_get();
8518 r->mailbox = calloc(1, size);
8526 r->key = l->header ?
8527 &t->structs[l->header->struct_id] :
8528 &t->structs[p->metadata_struct_id];
8535 learner_build_free(p);
8540 learner_free(struct rte_swx_pipeline *p)
8542 learner_build_free(p);
8544 /* Learner tables. */
8548 l = TAILQ_FIRST(&p->learners);
8552 TAILQ_REMOVE(&p->learners, l, node);
8555 free(l->default_action_data);
8564 table_state_build(struct rte_swx_pipeline *p)
8566 struct table *table;
8570 p->table_state = calloc(p->n_tables + p->n_selectors,
8571 sizeof(struct rte_swx_table_state));
8572 CHECK(p->table_state, ENOMEM);
8574 TAILQ_FOREACH(table, &p->tables, node) {
8575 struct rte_swx_table_state *ts = &p->table_state[table->id];
8578 struct rte_swx_table_params *params;
8581 params = table_params_get(table);
8582 CHECK(params, ENOMEM);
8584 ts->obj = table->type->ops.create(params,
8589 table_params_free(params);
8590 CHECK(ts->obj, ENODEV);
8593 /* ts->default_action_data. */
8594 if (table->action_data_size_max) {
8595 ts->default_action_data =
8596 malloc(table->action_data_size_max);
8597 CHECK(ts->default_action_data, ENOMEM);
8599 memcpy(ts->default_action_data,
8600 table->default_action_data,
8601 table->action_data_size_max);
8604 /* ts->default_action_id. */
8605 ts->default_action_id = table->default_action->id;
8608 TAILQ_FOREACH(s, &p->selectors, node) {
8609 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8610 struct rte_swx_table_selector_params *params;
8613 params = selector_table_params_get(s);
8614 CHECK(params, ENOMEM);
8616 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8618 selector_params_free(params);
8619 CHECK(ts->obj, ENODEV);
8622 TAILQ_FOREACH(l, &p->learners, node) {
8623 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8624 p->n_selectors + l->id];
8625 struct rte_swx_table_learner_params *params;
8628 params = learner_params_get(l);
8629 CHECK(params, ENOMEM);
8631 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8632 learner_params_free(params);
8633 CHECK(ts->obj, ENODEV);
8635 /* ts->default_action_data. */
8636 if (l->action_data_size_max) {
8637 ts->default_action_data = malloc(l->action_data_size_max);
8638 CHECK(ts->default_action_data, ENOMEM);
8640 memcpy(ts->default_action_data,
8641 l->default_action_data,
8642 l->action_data_size_max);
8645 /* ts->default_action_id. */
8646 ts->default_action_id = l->default_action->id;
8653 table_state_build_free(struct rte_swx_pipeline *p)
8657 if (!p->table_state)
8660 for (i = 0; i < p->n_tables; i++) {
8661 struct rte_swx_table_state *ts = &p->table_state[i];
8662 struct table *table = table_find_by_id(p, i);
8665 if (table->type && ts->obj)
8666 table->type->ops.free(ts->obj);
8668 /* ts->default_action_data. */
8669 free(ts->default_action_data);
8672 for (i = 0; i < p->n_selectors; i++) {
8673 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8677 rte_swx_table_selector_free(ts->obj);
8680 for (i = 0; i < p->n_learners; i++) {
8681 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8685 rte_swx_table_learner_free(ts->obj);
8687 /* ts->default_action_data. */
8688 free(ts->default_action_data);
8691 free(p->table_state);
8692 p->table_state = NULL;
8696 table_state_free(struct rte_swx_pipeline *p)
8698 table_state_build_free(p);
8704 static struct regarray *
8705 regarray_find(struct rte_swx_pipeline *p, const char *name)
8707 struct regarray *elem;
8709 TAILQ_FOREACH(elem, &p->regarrays, node)
8710 if (!strcmp(elem->name, name))
8716 static struct regarray *
8717 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8719 struct regarray *elem = NULL;
8721 TAILQ_FOREACH(elem, &p->regarrays, node)
8729 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8738 CHECK_NAME(name, EINVAL);
8739 CHECK(!regarray_find(p, name), EEXIST);
8741 CHECK(size, EINVAL);
8742 size = rte_align32pow2(size);
8744 /* Memory allocation. */
8745 r = calloc(1, sizeof(struct regarray));
8748 /* Node initialization. */
8749 strcpy(r->name, name);
8750 r->init_val = init_val;
8752 r->id = p->n_regarrays;
8754 /* Node add to tailq. */
8755 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8762 regarray_build(struct rte_swx_pipeline *p)
8764 struct regarray *regarray;
8766 if (!p->n_regarrays)
8769 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8770 CHECK(p->regarray_runtime, ENOMEM);
8772 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8773 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8776 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8777 RTE_CACHE_LINE_SIZE,
8779 CHECK(r->regarray, ENOMEM);
8781 if (regarray->init_val)
8782 for (i = 0; i < regarray->size; i++)
8783 r->regarray[i] = regarray->init_val;
8785 r->size_mask = regarray->size - 1;
8792 regarray_build_free(struct rte_swx_pipeline *p)
8796 if (!p->regarray_runtime)
8799 for (i = 0; i < p->n_regarrays; i++) {
8800 struct regarray *regarray = regarray_find_by_id(p, i);
8801 struct regarray_runtime *r = &p->regarray_runtime[i];
8803 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8806 free(p->regarray_runtime);
8807 p->regarray_runtime = NULL;
8811 regarray_free(struct rte_swx_pipeline *p)
8813 regarray_build_free(p);
8816 struct regarray *elem;
8818 elem = TAILQ_FIRST(&p->regarrays);
8822 TAILQ_REMOVE(&p->regarrays, elem, node);
8830 static struct meter_profile *
8831 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8833 struct meter_profile *elem;
8835 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8836 if (!strcmp(elem->name, name))
8842 static struct metarray *
8843 metarray_find(struct rte_swx_pipeline *p, const char *name)
8845 struct metarray *elem;
8847 TAILQ_FOREACH(elem, &p->metarrays, node)
8848 if (!strcmp(elem->name, name))
8854 static struct metarray *
8855 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8857 struct metarray *elem = NULL;
8859 TAILQ_FOREACH(elem, &p->metarrays, node)
8867 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8875 CHECK_NAME(name, EINVAL);
8876 CHECK(!metarray_find(p, name), EEXIST);
8878 CHECK(size, EINVAL);
8879 size = rte_align32pow2(size);
8881 /* Memory allocation. */
8882 m = calloc(1, sizeof(struct metarray));
8885 /* Node initialization. */
8886 strcpy(m->name, name);
8888 m->id = p->n_metarrays;
8890 /* Node add to tailq. */
8891 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8897 struct meter_profile meter_profile_default = {
8906 .cir_bytes_per_period = 1,
8908 .pir_bytes_per_period = 1,
8915 meter_init(struct meter *m)
8917 memset(m, 0, sizeof(struct meter));
8918 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8919 m->profile = &meter_profile_default;
8920 m->color_mask = RTE_COLOR_GREEN;
8922 meter_profile_default.n_users++;
8926 metarray_build(struct rte_swx_pipeline *p)
8930 if (!p->n_metarrays)
8933 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8934 CHECK(p->metarray_runtime, ENOMEM);
8936 TAILQ_FOREACH(m, &p->metarrays, node) {
8937 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8940 r->metarray = env_malloc(m->size * sizeof(struct meter),
8941 RTE_CACHE_LINE_SIZE,
8943 CHECK(r->metarray, ENOMEM);
8945 for (i = 0; i < m->size; i++)
8946 meter_init(&r->metarray[i]);
8948 r->size_mask = m->size - 1;
8955 metarray_build_free(struct rte_swx_pipeline *p)
8959 if (!p->metarray_runtime)
8962 for (i = 0; i < p->n_metarrays; i++) {
8963 struct metarray *m = metarray_find_by_id(p, i);
8964 struct metarray_runtime *r = &p->metarray_runtime[i];
8966 env_free(r->metarray, m->size * sizeof(struct meter));
8969 free(p->metarray_runtime);
8970 p->metarray_runtime = NULL;
8974 metarray_free(struct rte_swx_pipeline *p)
8976 metarray_build_free(p);
8980 struct metarray *elem;
8982 elem = TAILQ_FIRST(&p->metarrays);
8986 TAILQ_REMOVE(&p->metarrays, elem, node);
8990 /* Meter profiles. */
8992 struct meter_profile *elem;
8994 elem = TAILQ_FIRST(&p->meter_profiles);
8998 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9007 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9016 free(p->instruction_data);
9017 free(p->instructions);
9021 table_state_free(p);
9026 instruction_table_free(p);
9029 extern_func_free(p);
9042 port_in_types_register(struct rte_swx_pipeline *p)
9046 status = rte_swx_pipeline_port_in_type_register(p,
9048 &rte_swx_port_ethdev_reader_ops);
9052 status = rte_swx_pipeline_port_in_type_register(p,
9054 &rte_swx_port_ring_reader_ops);
9058 #ifdef RTE_PORT_PCAP
9059 status = rte_swx_pipeline_port_in_type_register(p,
9061 &rte_swx_port_source_ops);
9066 status = rte_swx_pipeline_port_in_type_register(p,
9068 &rte_swx_port_fd_reader_ops);
9076 port_out_types_register(struct rte_swx_pipeline *p)
9080 status = rte_swx_pipeline_port_out_type_register(p,
9082 &rte_swx_port_ethdev_writer_ops);
9086 status = rte_swx_pipeline_port_out_type_register(p,
9088 &rte_swx_port_ring_writer_ops);
9092 status = rte_swx_pipeline_port_out_type_register(p,
9094 &rte_swx_port_sink_ops);
9098 status = rte_swx_pipeline_port_out_type_register(p,
9100 &rte_swx_port_fd_writer_ops);
9108 table_types_register(struct rte_swx_pipeline *p)
9112 status = rte_swx_pipeline_table_type_register(p,
9114 RTE_SWX_TABLE_MATCH_EXACT,
9115 &rte_swx_table_exact_match_ops);
9119 status = rte_swx_pipeline_table_type_register(p,
9121 RTE_SWX_TABLE_MATCH_WILDCARD,
9122 &rte_swx_table_wildcard_match_ops);
9130 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9132 struct rte_swx_pipeline *pipeline = NULL;
9135 /* Check input parameters. */
9138 /* Memory allocation. */
9139 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9145 /* Initialization. */
9146 TAILQ_INIT(&pipeline->struct_types);
9147 TAILQ_INIT(&pipeline->port_in_types);
9148 TAILQ_INIT(&pipeline->ports_in);
9149 TAILQ_INIT(&pipeline->port_out_types);
9150 TAILQ_INIT(&pipeline->ports_out);
9151 TAILQ_INIT(&pipeline->extern_types);
9152 TAILQ_INIT(&pipeline->extern_objs);
9153 TAILQ_INIT(&pipeline->extern_funcs);
9154 TAILQ_INIT(&pipeline->headers);
9155 TAILQ_INIT(&pipeline->actions);
9156 TAILQ_INIT(&pipeline->table_types);
9157 TAILQ_INIT(&pipeline->tables);
9158 TAILQ_INIT(&pipeline->selectors);
9159 TAILQ_INIT(&pipeline->learners);
9160 TAILQ_INIT(&pipeline->regarrays);
9161 TAILQ_INIT(&pipeline->meter_profiles);
9162 TAILQ_INIT(&pipeline->metarrays);
9164 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9165 pipeline->numa_node = numa_node;
9167 status = port_in_types_register(pipeline);
9171 status = port_out_types_register(pipeline);
9175 status = table_types_register(pipeline);
9183 rte_swx_pipeline_free(pipeline);
9188 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9189 const char **instructions,
9190 uint32_t n_instructions)
9195 err = instruction_config(p, NULL, instructions, n_instructions);
9199 /* Thread instruction pointer reset. */
9200 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9201 struct thread *t = &p->threads[i];
9203 thread_ip_reset(p, t);
9210 pipeline_compile(struct rte_swx_pipeline *p);
9213 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9215 struct rte_swx_port_sink_params drop_port_params = {
9221 CHECK(p->build_done == 0, EEXIST);
9223 status = port_in_build(p);
9228 status = rte_swx_pipeline_port_out_config(p,
9235 status = port_out_build(p);
9239 status = struct_build(p);
9243 status = extern_obj_build(p);
9247 status = extern_func_build(p);
9251 status = header_build(p);
9255 status = metadata_build(p);
9259 status = instruction_table_build(p);
9263 status = action_build(p);
9267 status = table_build(p);
9271 status = selector_build(p);
9275 status = learner_build(p);
9279 status = table_state_build(p);
9283 status = regarray_build(p);
9287 status = metarray_build(p);
9293 pipeline_compile(p);
9298 metarray_build_free(p);
9299 regarray_build_free(p);
9300 table_state_build_free(p);
9301 learner_build_free(p);
9302 selector_build_free(p);
9303 table_build_free(p);
9304 action_build_free(p);
9305 instruction_table_build_free(p);
9306 metadata_build_free(p);
9307 header_build_free(p);
9308 extern_func_build_free(p);
9309 extern_obj_build_free(p);
9310 port_out_build_free(p);
9311 port_in_build_free(p);
9312 struct_build_free(p);
9318 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9322 for (i = 0; i < n_instructions; i++)
9327 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9331 for (i = 0; i < p->n_ports_out; i++) {
9332 struct port_out_runtime *port = &p->out[i];
9335 port->flush(port->obj);
9343 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9344 struct rte_swx_ctl_pipeline_info *pipeline)
9346 struct action *action;
9347 struct table *table;
9348 uint32_t n_actions = 0, n_tables = 0;
9350 if (!p || !pipeline)
9353 TAILQ_FOREACH(action, &p->actions, node)
9356 TAILQ_FOREACH(table, &p->tables, node)
9359 pipeline->n_ports_in = p->n_ports_in;
9360 pipeline->n_ports_out = p->n_ports_out;
9361 pipeline->n_actions = n_actions;
9362 pipeline->n_tables = n_tables;
9363 pipeline->n_selectors = p->n_selectors;
9364 pipeline->n_learners = p->n_learners;
9365 pipeline->n_regarrays = p->n_regarrays;
9366 pipeline->n_metarrays = p->n_metarrays;
9372 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9374 if (!p || !numa_node)
9377 *numa_node = p->numa_node;
9382 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9384 struct rte_swx_ctl_action_info *action)
9386 struct action *a = NULL;
9388 if (!p || (action_id >= p->n_actions) || !action)
9391 a = action_find_by_id(p, action_id);
9395 strcpy(action->name, a->name);
9396 action->n_args = a->st ? a->st->n_fields : 0;
9401 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9403 uint32_t action_arg_id,
9404 struct rte_swx_ctl_action_arg_info *action_arg)
9406 struct action *a = NULL;
9407 struct field *arg = NULL;
9409 if (!p || (action_id >= p->n_actions) || !action_arg)
9412 a = action_find_by_id(p, action_id);
9413 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9416 arg = &a->st->fields[action_arg_id];
9417 strcpy(action_arg->name, arg->name);
9418 action_arg->n_bits = arg->n_bits;
9419 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9425 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9427 struct rte_swx_ctl_table_info *table)
9429 struct table *t = NULL;
9434 t = table_find_by_id(p, table_id);
9438 strcpy(table->name, t->name);
9439 strcpy(table->args, t->args);
9440 table->n_match_fields = t->n_fields;
9441 table->n_actions = t->n_actions;
9442 table->default_action_is_const = t->default_action_is_const;
9443 table->size = t->size;
9448 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9450 uint32_t match_field_id,
9451 struct rte_swx_ctl_table_match_field_info *match_field)
9454 struct match_field *f;
9456 if (!p || (table_id >= p->n_tables) || !match_field)
9459 t = table_find_by_id(p, table_id);
9460 if (!t || (match_field_id >= t->n_fields))
9463 f = &t->fields[match_field_id];
9464 match_field->match_type = f->match_type;
9465 match_field->is_header = t->header ? 1 : 0;
9466 match_field->n_bits = f->field->n_bits;
9467 match_field->offset = f->field->offset;
9473 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9475 uint32_t table_action_id,
9476 struct rte_swx_ctl_table_action_info *table_action)
9480 if (!p || (table_id >= p->n_tables) || !table_action)
9483 t = table_find_by_id(p, table_id);
9484 if (!t || (table_action_id >= t->n_actions))
9487 table_action->action_id = t->actions[table_action_id]->id;
9489 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9490 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9496 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9498 struct rte_swx_table_ops *table_ops,
9503 if (!p || (table_id >= p->n_tables))
9506 t = table_find_by_id(p, table_id);
9512 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9522 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9523 uint32_t selector_id,
9524 struct rte_swx_ctl_selector_info *selector)
9526 struct selector *s = NULL;
9528 if (!p || !selector)
9531 s = selector_find_by_id(p, selector_id);
9535 strcpy(selector->name, s->name);
9537 selector->n_selector_fields = s->n_selector_fields;
9538 selector->n_groups_max = s->n_groups_max;
9539 selector->n_members_per_group_max = s->n_members_per_group_max;
9545 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9546 uint32_t selector_id,
9547 struct rte_swx_ctl_table_match_field_info *field)
9551 if (!p || (selector_id >= p->n_selectors) || !field)
9554 s = selector_find_by_id(p, selector_id);
9558 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9559 field->is_header = 0;
9560 field->n_bits = s->group_id_field->n_bits;
9561 field->offset = s->group_id_field->offset;
9567 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9568 uint32_t selector_id,
9569 uint32_t selector_field_id,
9570 struct rte_swx_ctl_table_match_field_info *field)
9575 if (!p || (selector_id >= p->n_selectors) || !field)
9578 s = selector_find_by_id(p, selector_id);
9579 if (!s || (selector_field_id >= s->n_selector_fields))
9582 f = s->selector_fields[selector_field_id];
9583 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9584 field->is_header = s->selector_header ? 1 : 0;
9585 field->n_bits = f->n_bits;
9586 field->offset = f->offset;
9592 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9593 uint32_t selector_id,
9594 struct rte_swx_ctl_table_match_field_info *field)
9598 if (!p || (selector_id >= p->n_selectors) || !field)
9601 s = selector_find_by_id(p, selector_id);
9605 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9606 field->is_header = 0;
9607 field->n_bits = s->member_id_field->n_bits;
9608 field->offset = s->member_id_field->offset;
9614 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9615 uint32_t learner_id,
9616 struct rte_swx_ctl_learner_info *learner)
9618 struct learner *l = NULL;
9623 l = learner_find_by_id(p, learner_id);
9627 strcpy(learner->name, l->name);
9629 learner->n_match_fields = l->n_fields;
9630 learner->n_actions = l->n_actions;
9631 learner->default_action_is_const = l->default_action_is_const;
9632 learner->size = l->size;
9638 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9639 uint32_t learner_id,
9640 uint32_t match_field_id,
9641 struct rte_swx_ctl_table_match_field_info *match_field)
9646 if (!p || (learner_id >= p->n_learners) || !match_field)
9649 l = learner_find_by_id(p, learner_id);
9650 if (!l || (match_field_id >= l->n_fields))
9653 f = l->fields[match_field_id];
9654 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9655 match_field->is_header = l->header ? 1 : 0;
9656 match_field->n_bits = f->n_bits;
9657 match_field->offset = f->offset;
9663 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9664 uint32_t learner_id,
9665 uint32_t learner_action_id,
9666 struct rte_swx_ctl_table_action_info *learner_action)
9670 if (!p || (learner_id >= p->n_learners) || !learner_action)
9673 l = learner_find_by_id(p, learner_id);
9674 if (!l || (learner_action_id >= l->n_actions))
9677 learner_action->action_id = l->actions[learner_action_id]->id;
9679 learner_action->action_is_for_table_entries =
9680 l->action_is_for_table_entries[learner_action_id];
9682 learner_action->action_is_for_default_entry =
9683 l->action_is_for_default_entry[learner_action_id];
9689 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9690 struct rte_swx_table_state **table_state)
9692 if (!p || !table_state || !p->build_done)
9695 *table_state = p->table_state;
9700 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9701 struct rte_swx_table_state *table_state)
9703 if (!p || !table_state || !p->build_done)
9706 p->table_state = table_state;
9711 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9713 struct rte_swx_port_in_stats *stats)
9715 struct port_in *port;
9720 port = port_in_find(p, port_id);
9724 port->type->ops.stats_read(port->obj, stats);
9729 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9731 struct rte_swx_port_out_stats *stats)
9733 struct port_out *port;
9738 port = port_out_find(p, port_id);
9742 port->type->ops.stats_read(port->obj, stats);
9747 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9748 const char *table_name,
9749 struct rte_swx_table_stats *stats)
9751 struct table *table;
9752 struct table_statistics *table_stats;
9754 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9757 table = table_find(p, table_name);
9761 table_stats = &p->table_stats[table->id];
9763 memcpy(stats->n_pkts_action,
9764 table_stats->n_pkts_action,
9765 p->n_actions * sizeof(uint64_t));
9767 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9768 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9774 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9775 const char *selector_name,
9776 struct rte_swx_pipeline_selector_stats *stats)
9780 if (!p || !selector_name || !selector_name[0] || !stats)
9783 s = selector_find(p, selector_name);
9787 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9793 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9794 const char *learner_name,
9795 struct rte_swx_learner_stats *stats)
9798 struct learner_statistics *learner_stats;
9800 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9803 l = learner_find(p, learner_name);
9807 learner_stats = &p->learner_stats[l->id];
9809 memcpy(stats->n_pkts_action,
9810 learner_stats->n_pkts_action,
9811 p->n_actions * sizeof(uint64_t));
9813 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9814 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9816 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9817 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9819 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9825 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9826 uint32_t regarray_id,
9827 struct rte_swx_ctl_regarray_info *regarray)
9831 if (!p || !regarray)
9834 r = regarray_find_by_id(p, regarray_id);
9838 strcpy(regarray->name, r->name);
9839 regarray->size = r->size;
9844 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9845 const char *regarray_name,
9846 uint32_t regarray_index,
9849 struct regarray *regarray;
9850 struct regarray_runtime *r;
9852 if (!p || !regarray_name || !value)
9855 regarray = regarray_find(p, regarray_name);
9856 if (!regarray || (regarray_index >= regarray->size))
9859 r = &p->regarray_runtime[regarray->id];
9860 *value = r->regarray[regarray_index];
9865 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9866 const char *regarray_name,
9867 uint32_t regarray_index,
9870 struct regarray *regarray;
9871 struct regarray_runtime *r;
9873 if (!p || !regarray_name)
9876 regarray = regarray_find(p, regarray_name);
9877 if (!regarray || (regarray_index >= regarray->size))
9880 r = &p->regarray_runtime[regarray->id];
9881 r->regarray[regarray_index] = value;
9886 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9887 uint32_t metarray_id,
9888 struct rte_swx_ctl_metarray_info *metarray)
9892 if (!p || !metarray)
9895 m = metarray_find_by_id(p, metarray_id);
9899 strcpy(metarray->name, m->name);
9900 metarray->size = m->size;
9905 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9907 struct rte_meter_trtcm_params *params)
9909 struct meter_profile *mp;
9913 CHECK_NAME(name, EINVAL);
9914 CHECK(params, EINVAL);
9915 CHECK(!meter_profile_find(p, name), EEXIST);
9917 /* Node allocation. */
9918 mp = calloc(1, sizeof(struct meter_profile));
9921 /* Node initialization. */
9922 strcpy(mp->name, name);
9923 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9924 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9930 /* Node add to tailq. */
9931 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9937 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9940 struct meter_profile *mp;
9943 CHECK_NAME(name, EINVAL);
9945 mp = meter_profile_find(p, name);
9947 CHECK(!mp->n_users, EBUSY);
9949 /* Remove node from tailq. */
9950 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9957 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9958 const char *metarray_name,
9959 uint32_t metarray_index)
9961 struct meter_profile *mp_old;
9962 struct metarray *metarray;
9963 struct metarray_runtime *metarray_runtime;
9967 CHECK_NAME(metarray_name, EINVAL);
9969 metarray = metarray_find(p, metarray_name);
9970 CHECK(metarray, EINVAL);
9971 CHECK(metarray_index < metarray->size, EINVAL);
9973 metarray_runtime = &p->metarray_runtime[metarray->id];
9974 m = &metarray_runtime->metarray[metarray_index];
9975 mp_old = m->profile;
9985 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9986 const char *metarray_name,
9987 uint32_t metarray_index,
9988 const char *profile_name)
9990 struct meter_profile *mp, *mp_old;
9991 struct metarray *metarray;
9992 struct metarray_runtime *metarray_runtime;
9996 CHECK_NAME(metarray_name, EINVAL);
9998 metarray = metarray_find(p, metarray_name);
9999 CHECK(metarray, EINVAL);
10000 CHECK(metarray_index < metarray->size, EINVAL);
10002 mp = meter_profile_find(p, profile_name);
10005 metarray_runtime = &p->metarray_runtime[metarray->id];
10006 m = &metarray_runtime->metarray[metarray_index];
10007 mp_old = m->profile;
10009 memset(m, 0, sizeof(struct meter));
10010 rte_meter_trtcm_config(&m->m, &mp->profile);
10012 m->color_mask = RTE_COLORS;
10021 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10022 const char *metarray_name,
10023 uint32_t metarray_index,
10024 struct rte_swx_ctl_meter_stats *stats)
10026 struct metarray *metarray;
10027 struct metarray_runtime *metarray_runtime;
10031 CHECK_NAME(metarray_name, EINVAL);
10033 metarray = metarray_find(p, metarray_name);
10034 CHECK(metarray, EINVAL);
10035 CHECK(metarray_index < metarray->size, EINVAL);
10037 CHECK(stats, EINVAL);
10039 metarray_runtime = &p->metarray_runtime[metarray->id];
10040 m = &metarray_runtime->metarray[metarray_index];
10042 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10043 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10049 * Pipeline compilation.
10051 static const char *
10052 instr_type_to_name(struct instruction *instr)
10054 switch (instr->type) {
10055 case INSTR_RX: return "INSTR_RX";
10057 case INSTR_TX: return "INSTR_TX";
10058 case INSTR_TX_I: return "INSTR_TX_I";
10059 case INSTR_DROP: return "INSTR_DROP";
10061 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10062 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10063 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10064 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10065 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10066 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10067 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10068 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10070 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10072 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10074 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10075 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10076 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10077 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10078 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10079 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10080 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10081 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10082 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10084 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10085 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10087 case INSTR_MOV: return "INSTR_MOV";
10088 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10089 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10090 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10091 case INSTR_MOV_I: return "INSTR_MOV_I";
10093 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10094 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10095 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10096 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10097 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10098 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10099 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10100 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10102 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10103 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10104 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10105 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10106 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10107 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10109 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10110 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10111 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10112 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10113 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10114 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10116 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10117 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10118 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10119 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10121 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10122 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10123 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10124 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10125 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10127 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10128 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10129 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10130 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10131 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10133 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10134 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10135 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10136 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10137 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10139 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10140 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10141 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10142 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10143 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10144 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10146 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10147 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10148 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10149 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10150 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10151 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10153 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10154 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10155 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10157 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10158 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10159 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10160 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10161 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10162 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10164 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10165 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10166 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10167 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10168 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10169 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10170 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10171 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10172 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10174 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10175 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10176 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10177 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10178 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10179 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10180 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10181 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10182 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10184 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10185 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10186 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10188 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10189 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10190 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10191 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10192 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10193 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10194 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10195 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10196 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10197 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10198 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10199 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10201 case INSTR_TABLE: return "INSTR_TABLE";
10202 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10203 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10204 case INSTR_LEARNER: return "INSTR_LEARNER";
10205 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10207 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10208 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10210 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10211 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10213 case INSTR_JMP: return "INSTR_JMP";
10214 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10215 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10216 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10217 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10218 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10219 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10220 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10221 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10222 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10223 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10224 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10225 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10226 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10227 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10228 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10229 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10230 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10231 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10232 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10233 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10234 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10235 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10236 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10237 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10238 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10239 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10240 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10241 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10243 case INSTR_RETURN: return "INSTR_RETURN";
10245 default: return "INSTR_UNKNOWN";
10250 (*instruction_export_t)(struct instruction *, FILE *);
10253 instr_io_export(struct instruction *instr, FILE *f)
10255 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10257 /* n_io, n_io_imm, n_hdrs. */
10258 if (instr->type == INSTR_RX ||
10259 instr->type == INSTR_TX ||
10260 instr->type == INSTR_HDR_EXTRACT_M ||
10261 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10264 if (instr->type == INSTR_TX_I)
10267 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10268 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10270 if (instr->type == INSTR_HDR_EXTRACT_M ||
10271 instr->type == INSTR_HDR_LOOKAHEAD ||
10272 instr->type == INSTR_HDR_EMIT)
10275 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10276 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10281 "\t\t.type = %s,\n",
10282 instr_type_to_name(instr));
10285 if (n_io || n_io_imm || n_hdrs)
10293 "\t\t\t\t.offset = %u,\n"
10294 "\t\t\t\t.n_bits = %u,\n"
10296 instr->io.io.offset,
10297 instr->io.io.n_bits);
10302 "\t\t\t\t.val = %u,\n"
10306 /* instr.io.hdr. */
10311 /* instr.io.hdr.header_id. */
10313 "\t\t\t.header_id = {");
10315 for (i = 0; i < n_hdrs; i++)
10318 instr->io.hdr.header_id[i]);
10323 /* instr.io.hdr.struct_id. */
10325 "\t\t\t.struct_id = {");
10327 for (i = 0; i < n_hdrs; i++)
10330 instr->io.hdr.struct_id[i]);
10335 /* instr.io.hdr.n_bytes. */
10337 "\t\t\t.n_bytes = {");
10339 for (i = 0; i < n_hdrs; i++)
10342 instr->io.hdr.n_bytes[i]);
10347 /* instr.io.hdr - closing curly brace. */
10352 /* instr.io - closing curly brace. */
10353 if (n_io || n_io_imm || n_hdrs)
10357 /* instr - closing curly brace. */
10363 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10367 "\t\t.type = %s,\n"
10369 "\t\t\t.header_id = %u,\n"
10372 instr_type_to_name(instr),
10373 instr->valid.header_id);
10377 instr_mov_export(struct instruction *instr, FILE *f)
10379 if (instr->type != INSTR_MOV_I)
10382 "\t\t.type = %s,\n"
10385 "\t\t\t\t.struct_id = %u,\n"
10386 "\t\t\t\t.n_bits = %u,\n"
10387 "\t\t\t\t.offset = %u,\n"
10390 "\t\t\t\t.struct_id = %u,\n"
10391 "\t\t\t\t.n_bits = %u,\n"
10392 "\t\t\t\t.offset = %u,\n"
10396 instr_type_to_name(instr),
10397 instr->mov.dst.struct_id,
10398 instr->mov.dst.n_bits,
10399 instr->mov.dst.offset,
10400 instr->mov.src.struct_id,
10401 instr->mov.src.n_bits,
10402 instr->mov.src.offset);
10406 "\t\t.type = %s,\n"
10409 "\t\t\t\t.struct_id = %u,\n"
10410 "\t\t\t\t.n_bits = %u,\n"
10411 "\t\t\t\t.offset = %u,\n"
10413 "\t\t\t.src_val = %" PRIu64 ",\n"
10416 instr_type_to_name(instr),
10417 instr->mov.dst.struct_id,
10418 instr->mov.dst.n_bits,
10419 instr->mov.dst.offset,
10420 instr->mov.src_val);
10424 instr_dma_ht_export(struct instruction *instr, FILE *f)
10426 uint32_t n_dma = 0, i;
10429 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10434 "\t\t.type = %s,\n",
10435 instr_type_to_name(instr));
10441 /* instr.dma.dst. */
10443 "\t\t\t.dst = {\n");
10445 /* instr.dma.dst.header_id. */
10447 "\t\t\t\t.header_id = {");
10449 for (i = 0; i < n_dma; i++)
10452 instr->dma.dst.header_id[i]);
10457 /* instr.dma.dst.struct_id. */
10459 "\t\t\t\t.struct_id = {");
10461 for (i = 0; i < n_dma; i++)
10464 instr->dma.dst.struct_id[i]);
10469 /* instr.dma.dst - closing curly brace. */
10473 /* instr.dma.src. */
10475 "\t\t\t.src = {\n");
10477 /* instr.dma.src.offset. */
10479 "\t\t\t\t.offset = {");
10481 for (i = 0; i < n_dma; i++)
10484 instr->dma.src.offset[i]);
10489 /* instr.dma.src - closing curly brace. */
10493 /* instr.dma.n_bytes. */
10495 "\t\t\t.n_bytes = {");
10497 for (i = 0; i < n_dma; i++)
10500 instr->dma.n_bytes[i]);
10505 /* instr.dma - closing curly brace. */
10509 /* instr - closing curly brace. */
10515 instr_alu_export(struct instruction *instr, FILE *f)
10519 if (instr->type == INSTR_ALU_ADD_MI ||
10520 instr->type == INSTR_ALU_ADD_HI ||
10521 instr->type == INSTR_ALU_SUB_MI ||
10522 instr->type == INSTR_ALU_SUB_HI ||
10523 instr->type == INSTR_ALU_SHL_MI ||
10524 instr->type == INSTR_ALU_SHL_HI ||
10525 instr->type == INSTR_ALU_SHR_MI ||
10526 instr->type == INSTR_ALU_SHR_HI ||
10527 instr->type == INSTR_ALU_AND_I ||
10528 instr->type == INSTR_ALU_OR_I ||
10529 instr->type == INSTR_ALU_XOR_I)
10535 "\t\t.type = %s,\n"
10538 "\t\t\t\t.struct_id = %u,\n"
10539 "\t\t\t\t.n_bits = %u,\n"
10540 "\t\t\t\t.offset = %u,\n"
10543 "\t\t\t\t.struct_id = %u,\n"
10544 "\t\t\t\t.n_bits = %u,\n"
10545 "\t\t\t\t.offset = %u,\n"
10549 instr_type_to_name(instr),
10550 instr->alu.dst.struct_id,
10551 instr->alu.dst.n_bits,
10552 instr->alu.dst.offset,
10553 instr->alu.src.struct_id,
10554 instr->alu.src.n_bits,
10555 instr->alu.src.offset);
10559 "\t\t.type = %s,\n"
10562 "\t\t\t\t.struct_id = %u,\n"
10563 "\t\t\t\t.n_bits = %u,\n"
10564 "\t\t\t\t.offset = %u,\n"
10566 "\t\t\t.src_val = %" PRIu64 ",\n"
10569 instr_type_to_name(instr),
10570 instr->alu.dst.struct_id,
10571 instr->alu.dst.n_bits,
10572 instr->alu.dst.offset,
10573 instr->alu.src_val);
10577 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10579 int prefetch = 0, idx_imm = 0, src_imm = 0;
10581 if (instr->type == INSTR_REGPREFETCH_RH ||
10582 instr->type == INSTR_REGPREFETCH_RM ||
10583 instr->type == INSTR_REGPREFETCH_RI)
10586 /* index is the 3rd operand for the regrd instruction and the 2nd
10587 * operand for the regwr and regadd instructions.
10589 if (instr->type == INSTR_REGPREFETCH_RI ||
10590 instr->type == INSTR_REGRD_HRI ||
10591 instr->type == INSTR_REGRD_MRI ||
10592 instr->type == INSTR_REGWR_RIH ||
10593 instr->type == INSTR_REGWR_RIM ||
10594 instr->type == INSTR_REGWR_RII ||
10595 instr->type == INSTR_REGADD_RIH ||
10596 instr->type == INSTR_REGADD_RIM ||
10597 instr->type == INSTR_REGADD_RII)
10600 /* src is the 3rd operand for the regwr and regadd instructions. */
10601 if (instr->type == INSTR_REGWR_RHI ||
10602 instr->type == INSTR_REGWR_RMI ||
10603 instr->type == INSTR_REGWR_RII ||
10604 instr->type == INSTR_REGADD_RHI ||
10605 instr->type == INSTR_REGADD_RMI ||
10606 instr->type == INSTR_REGADD_RII)
10609 /* instr.regarray.regarray_id. */
10612 "\t\t.type = %s,\n"
10613 "\t\t.regarray = {\n"
10614 "\t\t\t.regarray_id = %u,\n",
10615 instr_type_to_name(instr),
10616 instr->regarray.regarray_id);
10618 /* instr.regarray.idx / instr.regarray.idx_val. */
10621 "\t\t\t\t.idx = {\n"
10622 "\t\t\t\t\t.struct_id = %u,\n"
10623 "\t\t\t\t\t.n_bits = %u,\n"
10624 "\t\t\t\t\t.offset = %u,\n"
10626 instr->regarray.idx.struct_id,
10627 instr->regarray.idx.n_bits,
10628 instr->regarray.idx.offset);
10631 "\t\t\t\t.idx_val = %u,\n",
10632 instr->regarray.idx_val);
10634 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10638 "\t\t\t\t.dstsrc = {\n"
10639 "\t\t\t\t\t.struct_id = %u,\n"
10640 "\t\t\t\t\t.n_bits = %u,\n"
10641 "\t\t\t\t\t.offset = %u,\n"
10643 instr->regarray.dstsrc.struct_id,
10644 instr->regarray.dstsrc.n_bits,
10645 instr->regarray.dstsrc.offset);
10648 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10649 instr->regarray.dstsrc_val);
10652 /* instr.regarray and instr - closing curly braces. */
10659 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10661 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10663 if (instr->type == INSTR_METPREFETCH_H ||
10664 instr->type == INSTR_METPREFETCH_M ||
10665 instr->type == INSTR_METPREFETCH_I)
10669 if (instr->type == INSTR_METPREFETCH_I ||
10670 instr->type == INSTR_METER_IHM ||
10671 instr->type == INSTR_METER_IHI ||
10672 instr->type == INSTR_METER_IMM ||
10673 instr->type == INSTR_METER_IMI)
10676 /* color_in_imm. */
10677 if (instr->type == INSTR_METER_HHI ||
10678 instr->type == INSTR_METER_HMI ||
10679 instr->type == INSTR_METER_MHI ||
10680 instr->type == INSTR_METER_MMI ||
10681 instr->type == INSTR_METER_IHI ||
10682 instr->type == INSTR_METER_IMI)
10685 /* instr.meter.metarray_id. */
10688 "\t\t.type = %s,\n"
10690 "\t\t\t.metarray_id = %u,\n",
10691 instr_type_to_name(instr),
10692 instr->meter.metarray_id);
10694 /* instr.meter.idx / instr.meter.idx_val. */
10698 "\t\t\t\t.struct_id = %u,\n"
10699 "\t\t\t\t.n_bits = %u,\n"
10700 "\t\t\t\t.offset = %u,\n"
10702 instr->meter.idx.struct_id,
10703 instr->meter.idx.n_bits,
10704 instr->meter.idx.offset);
10707 "\t\t\t.idx_val = %u,\n",
10708 instr->meter.idx_val);
10711 /* instr.meter.length. */
10713 "\t\t\t.length = {\n"
10714 "\t\t\t\t.struct_id = %u,\n"
10715 "\t\t\t\t.n_bits = %u,\n"
10716 "\t\t\t\t.offset = %u,\n"
10718 instr->meter.length.struct_id,
10719 instr->meter.length.n_bits,
10720 instr->meter.length.offset);
10722 /* instr.meter.color_in / instr.meter.color_in_val. */
10725 "\t\t\t.color_in = {\n"
10726 "\t\t\t\t.struct_id = %u,\n"
10727 "\t\t\t\t.n_bits = %u,\n"
10728 "\t\t\t\t.offset = %u,\n"
10730 instr->meter.color_in.struct_id,
10731 instr->meter.color_in.n_bits,
10732 instr->meter.color_in.offset);
10735 "\t\t\t.color_in_val = %u,\n",
10736 (uint32_t)instr->meter.color_in_val);
10738 /* instr.meter.color_out. */
10740 "\t\t\t.color_out = {\n"
10741 "\t\t\t\t.struct_id = %u,\n"
10742 "\t\t\t\t.n_bits = %u,\n"
10743 "\t\t\t\t.offset = %u,\n"
10745 instr->meter.color_out.struct_id,
10746 instr->meter.color_out.n_bits,
10747 instr->meter.color_out.offset);
10750 /* instr.meter and instr - closing curly braces. */
10757 instr_table_export(struct instruction *instr,
10762 "\t\t.type = %s,\n"
10764 "\t\t\t.table_id = %u,\n"
10767 instr_type_to_name(instr),
10768 instr->table.table_id);
10772 instr_learn_export(struct instruction *instr, FILE *f)
10776 "\t\t.type = %s,\n"
10778 "\t\t\t\t.action_id = %u,\n"
10781 instr_type_to_name(instr),
10782 instr->learn.action_id);
10786 instr_forget_export(struct instruction *instr, FILE *f)
10790 "\t\t.type = %s,\n"
10792 instr_type_to_name(instr));
10796 instr_extern_export(struct instruction *instr, FILE *f)
10798 if (instr->type == INSTR_EXTERN_OBJ)
10801 "\t\t.type = %s,\n"
10802 "\t\t.ext_obj = {\n"
10803 "\t\t\t.ext_obj_id = %u,\n"
10804 "\t\t\t.func_id = %u,\n"
10807 instr_type_to_name(instr),
10808 instr->ext_obj.ext_obj_id,
10809 instr->ext_obj.func_id);
10813 "\t\t.type = %s,\n"
10814 "\t\t.ext_func = {\n"
10815 "\t\t\t.ext_func_id = %u,\n"
10818 instr_type_to_name(instr),
10819 instr->ext_func.ext_func_id);
10823 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10827 "\t\t.type = %s,\n"
10829 "\t\t\t.ip = NULL,\n",
10830 instr_type_to_name(instr));
10832 switch (instr->type) {
10833 case INSTR_JMP_VALID:
10834 case INSTR_JMP_INVALID:
10836 "\t\t\t.header_id = %u,\n",
10837 instr->jmp.header_id);
10840 case INSTR_JMP_ACTION_HIT:
10841 case INSTR_JMP_ACTION_MISS:
10843 "\t\t\t.action_id = %u,\n",
10844 instr->jmp.action_id);
10848 case INSTR_JMP_EQ_MH:
10849 case INSTR_JMP_EQ_HM:
10850 case INSTR_JMP_EQ_HH:
10851 case INSTR_JMP_NEQ:
10852 case INSTR_JMP_NEQ_MH:
10853 case INSTR_JMP_NEQ_HM:
10854 case INSTR_JMP_NEQ_HH:
10856 case INSTR_JMP_LT_MH:
10857 case INSTR_JMP_LT_HM:
10858 case INSTR_JMP_LT_HH:
10860 case INSTR_JMP_GT_MH:
10861 case INSTR_JMP_GT_HM:
10862 case INSTR_JMP_GT_HH:
10865 "\t\t\t\t.struct_id = %u,\n"
10866 "\t\t\t\t.n_bits = %u,\n"
10867 "\t\t\t\t.offset = %u,\n"
10870 "\t\t\t\t.struct_id = %u,\n"
10871 "\t\t\t\t.n_bits = %u,\n"
10872 "\t\t\t\t.offset = %u,\n"
10874 instr->jmp.a.struct_id,
10875 instr->jmp.a.n_bits,
10876 instr->jmp.a.offset,
10877 instr->jmp.b.struct_id,
10878 instr->jmp.b.n_bits,
10879 instr->jmp.b.offset);
10882 case INSTR_JMP_EQ_I:
10883 case INSTR_JMP_NEQ_I:
10884 case INSTR_JMP_LT_MI:
10885 case INSTR_JMP_LT_HI:
10886 case INSTR_JMP_GT_MI:
10887 case INSTR_JMP_GT_HI:
10890 "\t\t\t\t.struct_id = %u,\n"
10891 "\t\t\t\t.n_bits = %u,\n"
10892 "\t\t\t\t.offset = %u,\n"
10894 "\t\t\t.b_val = %" PRIu64 ",\n",
10895 instr->jmp.a.struct_id,
10896 instr->jmp.a.n_bits,
10897 instr->jmp.a.offset,
10911 instr_return_export(struct instruction *instr,
10916 "\t\t.type = %s,\n",
10917 instr_type_to_name(instr));
10923 static instruction_export_t export_table[] = {
10924 [INSTR_RX] = instr_io_export,
10926 [INSTR_TX] = instr_io_export,
10927 [INSTR_TX_I] = instr_io_export,
10928 [INSTR_DROP] = instr_io_export,
10930 [INSTR_HDR_EXTRACT] = instr_io_export,
10931 [INSTR_HDR_EXTRACT2] = instr_io_export,
10932 [INSTR_HDR_EXTRACT3] = instr_io_export,
10933 [INSTR_HDR_EXTRACT4] = instr_io_export,
10934 [INSTR_HDR_EXTRACT5] = instr_io_export,
10935 [INSTR_HDR_EXTRACT6] = instr_io_export,
10936 [INSTR_HDR_EXTRACT7] = instr_io_export,
10937 [INSTR_HDR_EXTRACT8] = instr_io_export,
10939 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10941 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10943 [INSTR_HDR_EMIT] = instr_io_export,
10944 [INSTR_HDR_EMIT_TX] = instr_io_export,
10945 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10946 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10947 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10948 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10949 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10950 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10951 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10953 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10954 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10956 [INSTR_MOV] = instr_mov_export,
10957 [INSTR_MOV_MH] = instr_mov_export,
10958 [INSTR_MOV_HM] = instr_mov_export,
10959 [INSTR_MOV_HH] = instr_mov_export,
10960 [INSTR_MOV_I] = instr_mov_export,
10962 [INSTR_DMA_HT] = instr_dma_ht_export,
10963 [INSTR_DMA_HT2] = instr_dma_ht_export,
10964 [INSTR_DMA_HT3] = instr_dma_ht_export,
10965 [INSTR_DMA_HT4] = instr_dma_ht_export,
10966 [INSTR_DMA_HT5] = instr_dma_ht_export,
10967 [INSTR_DMA_HT6] = instr_dma_ht_export,
10968 [INSTR_DMA_HT7] = instr_dma_ht_export,
10969 [INSTR_DMA_HT8] = instr_dma_ht_export,
10971 [INSTR_ALU_ADD] = instr_alu_export,
10972 [INSTR_ALU_ADD_MH] = instr_alu_export,
10973 [INSTR_ALU_ADD_HM] = instr_alu_export,
10974 [INSTR_ALU_ADD_HH] = instr_alu_export,
10975 [INSTR_ALU_ADD_MI] = instr_alu_export,
10976 [INSTR_ALU_ADD_HI] = instr_alu_export,
10978 [INSTR_ALU_SUB] = instr_alu_export,
10979 [INSTR_ALU_SUB_MH] = instr_alu_export,
10980 [INSTR_ALU_SUB_HM] = instr_alu_export,
10981 [INSTR_ALU_SUB_HH] = instr_alu_export,
10982 [INSTR_ALU_SUB_MI] = instr_alu_export,
10983 [INSTR_ALU_SUB_HI] = instr_alu_export,
10985 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10986 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10987 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10988 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10990 [INSTR_ALU_AND] = instr_alu_export,
10991 [INSTR_ALU_AND_MH] = instr_alu_export,
10992 [INSTR_ALU_AND_HM] = instr_alu_export,
10993 [INSTR_ALU_AND_HH] = instr_alu_export,
10994 [INSTR_ALU_AND_I] = instr_alu_export,
10996 [INSTR_ALU_OR] = instr_alu_export,
10997 [INSTR_ALU_OR_MH] = instr_alu_export,
10998 [INSTR_ALU_OR_HM] = instr_alu_export,
10999 [INSTR_ALU_OR_HH] = instr_alu_export,
11000 [INSTR_ALU_OR_I] = instr_alu_export,
11002 [INSTR_ALU_XOR] = instr_alu_export,
11003 [INSTR_ALU_XOR_MH] = instr_alu_export,
11004 [INSTR_ALU_XOR_HM] = instr_alu_export,
11005 [INSTR_ALU_XOR_HH] = instr_alu_export,
11006 [INSTR_ALU_XOR_I] = instr_alu_export,
11008 [INSTR_ALU_SHL] = instr_alu_export,
11009 [INSTR_ALU_SHL_MH] = instr_alu_export,
11010 [INSTR_ALU_SHL_HM] = instr_alu_export,
11011 [INSTR_ALU_SHL_HH] = instr_alu_export,
11012 [INSTR_ALU_SHL_MI] = instr_alu_export,
11013 [INSTR_ALU_SHL_HI] = instr_alu_export,
11015 [INSTR_ALU_SHR] = instr_alu_export,
11016 [INSTR_ALU_SHR_MH] = instr_alu_export,
11017 [INSTR_ALU_SHR_HM] = instr_alu_export,
11018 [INSTR_ALU_SHR_HH] = instr_alu_export,
11019 [INSTR_ALU_SHR_MI] = instr_alu_export,
11020 [INSTR_ALU_SHR_HI] = instr_alu_export,
11022 [INSTR_REGPREFETCH_RH] = instr_reg_export,
11023 [INSTR_REGPREFETCH_RM] = instr_reg_export,
11024 [INSTR_REGPREFETCH_RI] = instr_reg_export,
11026 [INSTR_REGRD_HRH] = instr_reg_export,
11027 [INSTR_REGRD_HRM] = instr_reg_export,
11028 [INSTR_REGRD_MRH] = instr_reg_export,
11029 [INSTR_REGRD_MRM] = instr_reg_export,
11030 [INSTR_REGRD_HRI] = instr_reg_export,
11031 [INSTR_REGRD_MRI] = instr_reg_export,
11033 [INSTR_REGWR_RHH] = instr_reg_export,
11034 [INSTR_REGWR_RHM] = instr_reg_export,
11035 [INSTR_REGWR_RMH] = instr_reg_export,
11036 [INSTR_REGWR_RMM] = instr_reg_export,
11037 [INSTR_REGWR_RHI] = instr_reg_export,
11038 [INSTR_REGWR_RMI] = instr_reg_export,
11039 [INSTR_REGWR_RIH] = instr_reg_export,
11040 [INSTR_REGWR_RIM] = instr_reg_export,
11041 [INSTR_REGWR_RII] = instr_reg_export,
11043 [INSTR_REGADD_RHH] = instr_reg_export,
11044 [INSTR_REGADD_RHM] = instr_reg_export,
11045 [INSTR_REGADD_RMH] = instr_reg_export,
11046 [INSTR_REGADD_RMM] = instr_reg_export,
11047 [INSTR_REGADD_RHI] = instr_reg_export,
11048 [INSTR_REGADD_RMI] = instr_reg_export,
11049 [INSTR_REGADD_RIH] = instr_reg_export,
11050 [INSTR_REGADD_RIM] = instr_reg_export,
11051 [INSTR_REGADD_RII] = instr_reg_export,
11053 [INSTR_METPREFETCH_H] = instr_meter_export,
11054 [INSTR_METPREFETCH_M] = instr_meter_export,
11055 [INSTR_METPREFETCH_I] = instr_meter_export,
11057 [INSTR_METER_HHM] = instr_meter_export,
11058 [INSTR_METER_HHI] = instr_meter_export,
11059 [INSTR_METER_HMM] = instr_meter_export,
11060 [INSTR_METER_HMI] = instr_meter_export,
11061 [INSTR_METER_MHM] = instr_meter_export,
11062 [INSTR_METER_MHI] = instr_meter_export,
11063 [INSTR_METER_MMM] = instr_meter_export,
11064 [INSTR_METER_MMI] = instr_meter_export,
11065 [INSTR_METER_IHM] = instr_meter_export,
11066 [INSTR_METER_IHI] = instr_meter_export,
11067 [INSTR_METER_IMM] = instr_meter_export,
11068 [INSTR_METER_IMI] = instr_meter_export,
11070 [INSTR_TABLE] = instr_table_export,
11071 [INSTR_TABLE_AF] = instr_table_export,
11072 [INSTR_SELECTOR] = instr_table_export,
11073 [INSTR_LEARNER] = instr_table_export,
11074 [INSTR_LEARNER_AF] = instr_table_export,
11076 [INSTR_LEARNER_LEARN] = instr_learn_export,
11077 [INSTR_LEARNER_FORGET] = instr_forget_export,
11079 [INSTR_EXTERN_OBJ] = instr_extern_export,
11080 [INSTR_EXTERN_FUNC] = instr_extern_export,
11082 [INSTR_JMP] = instr_jmp_export,
11083 [INSTR_JMP_VALID] = instr_jmp_export,
11084 [INSTR_JMP_INVALID] = instr_jmp_export,
11085 [INSTR_JMP_HIT] = instr_jmp_export,
11086 [INSTR_JMP_MISS] = instr_jmp_export,
11087 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11088 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11090 [INSTR_JMP_EQ] = instr_jmp_export,
11091 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11092 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11093 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11094 [INSTR_JMP_EQ_I] = instr_jmp_export,
11096 [INSTR_JMP_NEQ] = instr_jmp_export,
11097 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11098 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11099 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11100 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11102 [INSTR_JMP_LT] = instr_jmp_export,
11103 [INSTR_JMP_LT_MH] = instr_jmp_export,
11104 [INSTR_JMP_LT_HM] = instr_jmp_export,
11105 [INSTR_JMP_LT_HH] = instr_jmp_export,
11106 [INSTR_JMP_LT_MI] = instr_jmp_export,
11107 [INSTR_JMP_LT_HI] = instr_jmp_export,
11109 [INSTR_JMP_GT] = instr_jmp_export,
11110 [INSTR_JMP_GT_MH] = instr_jmp_export,
11111 [INSTR_JMP_GT_HM] = instr_jmp_export,
11112 [INSTR_JMP_GT_HH] = instr_jmp_export,
11113 [INSTR_JMP_GT_MI] = instr_jmp_export,
11114 [INSTR_JMP_GT_HI] = instr_jmp_export,
11116 [INSTR_RETURN] = instr_return_export,
11120 action_data_codegen(struct action *a, FILE *f)
11125 "static const struct instruction action_%s_instructions[] = {\n",
11128 for (i = 0; i < a->n_instructions; i++) {
11129 struct instruction *instr = &a->instructions[i];
11130 instruction_export_t func = export_table[instr->type];
11135 fprintf(f, "};\n");
11138 static const char *
11139 instr_type_to_func(struct instruction *instr)
11141 switch (instr->type) {
11142 case INSTR_RX: return NULL;
11144 case INSTR_TX: return "__instr_tx_exec";
11145 case INSTR_TX_I: return "__instr_tx_i_exec";
11146 case INSTR_DROP: return "__instr_drop_exec";
11148 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11149 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11150 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11151 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11152 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11153 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11154 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11155 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11157 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11159 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11161 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11162 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11163 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11164 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11165 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11166 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11167 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11168 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11169 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11171 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11172 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11174 case INSTR_MOV: return "__instr_mov_exec";
11175 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11176 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11177 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11178 case INSTR_MOV_I: return "__instr_mov_i_exec";
11180 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11181 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11182 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11183 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11184 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11185 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11186 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11187 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11189 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11190 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11191 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11192 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11193 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11194 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11196 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11197 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11198 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11199 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11200 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11201 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11203 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11204 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11205 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11206 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11208 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11209 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11210 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11211 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11212 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11214 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11215 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11216 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11217 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11218 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11220 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11221 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11222 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11223 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11224 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11226 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11227 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11228 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11229 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11230 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11231 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11233 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11234 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11235 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11236 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11237 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11238 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11240 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11241 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11242 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11244 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11245 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11246 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11247 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11248 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11249 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11251 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11252 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11253 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11254 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11255 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11256 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11257 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11258 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11259 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11261 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11262 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11263 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11264 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11265 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11266 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11267 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11268 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11269 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11271 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11272 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11273 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11275 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11276 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11277 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11278 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11279 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11280 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11281 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11282 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11283 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11284 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11285 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11286 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11288 case INSTR_TABLE: return NULL;
11289 case INSTR_TABLE_AF: return NULL;
11290 case INSTR_SELECTOR: return NULL;
11291 case INSTR_LEARNER: return NULL;
11292 case INSTR_LEARNER_AF: return NULL;
11294 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11295 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11297 case INSTR_EXTERN_OBJ: return NULL;
11298 case INSTR_EXTERN_FUNC: return NULL;
11300 case INSTR_JMP: return NULL;
11301 case INSTR_JMP_VALID: return NULL;
11302 case INSTR_JMP_INVALID: return NULL;
11303 case INSTR_JMP_HIT: return NULL;
11304 case INSTR_JMP_MISS: return NULL;
11305 case INSTR_JMP_ACTION_HIT: return NULL;
11306 case INSTR_JMP_ACTION_MISS: return NULL;
11307 case INSTR_JMP_EQ: return NULL;
11308 case INSTR_JMP_EQ_MH: return NULL;
11309 case INSTR_JMP_EQ_HM: return NULL;
11310 case INSTR_JMP_EQ_HH: return NULL;
11311 case INSTR_JMP_EQ_I: return NULL;
11312 case INSTR_JMP_NEQ: return NULL;
11313 case INSTR_JMP_NEQ_MH: return NULL;
11314 case INSTR_JMP_NEQ_HM: return NULL;
11315 case INSTR_JMP_NEQ_HH: return NULL;
11316 case INSTR_JMP_NEQ_I: return NULL;
11317 case INSTR_JMP_LT: return NULL;
11318 case INSTR_JMP_LT_MH: return NULL;
11319 case INSTR_JMP_LT_HM: return NULL;
11320 case INSTR_JMP_LT_HH: return NULL;
11321 case INSTR_JMP_LT_MI: return NULL;
11322 case INSTR_JMP_LT_HI: return NULL;
11323 case INSTR_JMP_GT: return NULL;
11324 case INSTR_JMP_GT_MH: return NULL;
11325 case INSTR_JMP_GT_HM: return NULL;
11326 case INSTR_JMP_GT_HH: return NULL;
11327 case INSTR_JMP_GT_MI: return NULL;
11328 case INSTR_JMP_GT_HI: return NULL;
11330 case INSTR_RETURN: return NULL;
11332 default: return NULL;
11337 action_instr_does_tx_codegen(struct action *a,
11338 uint32_t instr_pos,
11339 struct instruction *instr,
11343 "%s(p, t, &action_%s_instructions[%u]);\n"
11344 "\tthread_ip_reset(p, t);\n"
11345 "\tinstr_rx_exec(p);\n"
11347 instr_type_to_func(instr),
11353 action_instr_extern_obj_codegen(struct action *a,
11354 uint32_t instr_pos,
11358 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11364 action_instr_extern_func_codegen(struct action *a,
11365 uint32_t instr_pos,
11369 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11375 action_instr_jmp_codegen(struct action *a,
11376 uint32_t instr_pos,
11377 struct instruction *instr,
11378 struct instruction_data *data,
11381 switch (instr->type) {
11388 case INSTR_JMP_VALID:
11390 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11397 case INSTR_JMP_INVALID:
11399 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11406 case INSTR_JMP_HIT:
11413 case INSTR_JMP_MISS:
11420 case INSTR_JMP_ACTION_HIT:
11422 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11429 case INSTR_JMP_ACTION_MISS:
11431 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11440 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11441 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11450 case INSTR_JMP_EQ_MH:
11452 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11453 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11462 case INSTR_JMP_EQ_HM:
11464 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11465 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11474 case INSTR_JMP_EQ_HH:
11476 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11477 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11486 case INSTR_JMP_EQ_I:
11488 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11489 "action_%s_instructions[%u].jmp.b_val)\n"
11498 case INSTR_JMP_NEQ:
11500 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11501 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11510 case INSTR_JMP_NEQ_MH:
11512 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11513 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11522 case INSTR_JMP_NEQ_HM:
11524 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11525 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11534 case INSTR_JMP_NEQ_HH:
11536 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11537 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11546 case INSTR_JMP_NEQ_I:
11548 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11549 "action_%s_instructions[%u].jmp.b_val)\n"
11560 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11561 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11570 case INSTR_JMP_LT_MH:
11572 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11573 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11582 case INSTR_JMP_LT_HM:
11584 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11585 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11594 case INSTR_JMP_LT_HH:
11596 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11597 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11606 case INSTR_JMP_LT_MI:
11608 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11609 "action_%s_instructions[%u].jmp.b_val)\n"
11618 case INSTR_JMP_LT_HI:
11620 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11621 "action_%s_instructions[%u].jmp.b_val)\n"
11632 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11633 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11642 case INSTR_JMP_GT_MH:
11644 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11645 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11654 case INSTR_JMP_GT_HM:
11656 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11657 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11666 case INSTR_JMP_GT_HH:
11668 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11669 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11678 case INSTR_JMP_GT_MI:
11680 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11681 "action_%s_instructions[%u].jmp.b_val)\n"
11690 case INSTR_JMP_GT_HI:
11692 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11693 "action_%s_instructions[%u].jmp.b_val)\n"
11708 action_instr_return_codegen(FILE *f)
11715 action_instr_codegen(struct action *a, FILE *f)
11721 "action_%s_run(struct rte_swx_pipeline *p)\n"
11723 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11727 for (i = 0; i < a->n_instructions; i++) {
11728 struct instruction *instr = &a->instructions[i];
11729 struct instruction_data *data = &a->instruction_data[i];
11731 /* Label, if present. */
11732 if (data->label[0])
11733 fprintf(f, "\n%s : ", data->label);
11735 fprintf(f, "\n\t");
11737 /* TX instruction type. */
11738 if (instruction_does_tx(instr)) {
11739 action_instr_does_tx_codegen(a, i, instr, f);
11743 /* Extern object/function instruction type. */
11744 if (instr->type == INSTR_EXTERN_OBJ) {
11745 action_instr_extern_obj_codegen(a, i, f);
11749 if (instr->type == INSTR_EXTERN_FUNC) {
11750 action_instr_extern_func_codegen(a, i, f);
11754 /* Jump instruction type. */
11755 if (instruction_is_jmp(instr)) {
11756 action_instr_jmp_codegen(a, i, instr, data, f);
11760 /* Return instruction type. */
11761 if (instr->type == INSTR_RETURN) {
11762 action_instr_return_codegen(f);
11766 /* Any other instruction type. */
11768 "%s(p, t, &action_%s_instructions[%u]);\n",
11769 instr_type_to_func(instr),
11774 fprintf(f, "}\n\n");
11777 struct instruction_group {
11778 TAILQ_ENTRY(instruction_group) node;
11782 uint32_t first_instr_id;
11784 uint32_t last_instr_id;
11789 TAILQ_HEAD(instruction_group_list, instruction_group);
11791 static struct instruction_group *
11792 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11794 struct instruction_group *g;
11796 TAILQ_FOREACH(g, igl, node)
11797 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11804 instruction_group_list_free(struct instruction_group_list *igl)
11810 struct instruction_group *g;
11812 g = TAILQ_FIRST(igl);
11816 TAILQ_REMOVE(igl, g, node);
11823 static struct instruction_group_list *
11824 instruction_group_list_create(struct rte_swx_pipeline *p)
11826 struct instruction_group_list *igl = NULL;
11827 struct instruction_group *g = NULL;
11828 uint32_t n_groups = 0, i;
11830 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11834 igl = calloc(1, sizeof(struct instruction_group_list));
11840 /* Allocate the first group. */
11841 g = calloc(1, sizeof(struct instruction_group));
11845 /* Iteration 1: Separate the instructions into groups based on the thread yield
11846 * instructions. Do not worry about the jump instructions at this point.
11848 for (i = 0; i < p->n_instructions; i++) {
11849 struct instruction *instr = &p->instructions[i];
11851 /* Check for thread yield instructions. */
11852 if (!instruction_does_thread_yield(instr))
11855 /* If the current group contains at least one instruction, then finalize it (with
11856 * the previous instruction), add it to the list and allocate a new group (that
11857 * starts with the current instruction).
11859 if (i - g->first_instr_id) {
11860 /* Finalize the group. */
11861 g->last_instr_id = i - 1;
11863 /* Add the group to the list. Advance the number of groups. */
11864 TAILQ_INSERT_TAIL(igl, g, node);
11867 /* Allocate a new group. */
11868 g = calloc(1, sizeof(struct instruction_group));
11872 /* Initialize the new group. */
11873 g->group_id = n_groups;
11874 g->first_instr_id = i;
11877 /* Finalize the current group (with the current instruction, therefore this group
11878 * contains just the current thread yield instruction), add it to the list and
11879 * allocate a new group (that starts with the next instruction).
11882 /* Finalize the group. */
11883 g->last_instr_id = i;
11885 /* Add the group to the list. Advance the number of groups. */
11886 TAILQ_INSERT_TAIL(igl, g, node);
11889 /* Allocate a new group. */
11890 g = calloc(1, sizeof(struct instruction_group));
11894 /* Initialize the new group. */
11895 g->group_id = n_groups;
11896 g->first_instr_id = i + 1;
11899 /* Handle the last group. */
11900 if (i - g->first_instr_id) {
11901 /* Finalize the group. */
11902 g->last_instr_id = i - 1;
11904 /* Add the group to the list. Advance the number of groups. */
11905 TAILQ_INSERT_TAIL(igl, g, node);
11912 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11913 * the destination of a jump instruction located in a different group ("far jump"), then the
11914 * current group has to be split, so that the instruction representing the far jump
11915 * destination is at the start of its group.
11918 int is_modified = 0;
11920 for (i = 0; i < p->n_instructions; i++) {
11921 struct instruction_data *data = &p->instruction_data[i];
11922 struct instruction_group *g;
11925 /* Continue when the current instruction is not a jump destination. */
11926 if (!data->n_users)
11929 g = instruction_group_list_group_find(igl, i);
11933 /* Find out all the jump instructions with this destination. */
11934 for (j = 0; j < p->n_instructions; j++) {
11935 struct instruction *jmp_instr = &p->instructions[j];
11936 struct instruction_data *jmp_data = &p->instruction_data[j];
11937 struct instruction_group *jmp_g, *new_g;
11939 /* Continue when not a jump instruction. Even when jump instruction,
11940 * continue when the jump destination is not this instruction.
11942 if (!instruction_is_jmp(jmp_instr) ||
11943 strcmp(jmp_data->jmp_label, data->label))
11946 jmp_g = instruction_group_list_group_find(igl, j);
11950 /* Continue when both the jump instruction and the jump destination
11951 * instruction are in the same group. Even when in different groups,
11952 * still continue if the jump destination instruction is already the
11953 * first instruction of its group.
11955 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11958 /* Split the group of the current jump destination instruction to
11959 * make this instruction the first instruction of a new group.
11961 new_g = calloc(1, sizeof(struct instruction_group));
11965 new_g->group_id = n_groups;
11966 new_g->first_instr_id = i;
11967 new_g->last_instr_id = g->last_instr_id;
11969 g->last_instr_id = i - 1;
11971 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11975 /* The decision to split this group (to make the current instruction
11976 * the first instruction of a new group) is already taken and fully
11977 * implemented, so no need to search for more reasons to do it.
11983 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11984 * previously considered local (i.e. the jump destination is in the same group as
11985 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11986 * different group than the jump instruction). Wost case scenario: each instruction
11987 * that is a jump destination ends up as the first instruction of its group.
11993 /* Re-assign the group IDs to be in incremental order. */
11995 TAILQ_FOREACH(g, igl, node) {
12004 instruction_group_list_free(igl);
12012 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12013 uint32_t instr_pos,
12014 struct instruction *instr,
12018 "%s(p, t, &pipeline_instructions[%u]);\n"
12019 "\tthread_ip_reset(p, t);\n"
12020 "\tinstr_rx_exec(p);\n"
12022 instr_type_to_func(instr),
12027 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12028 struct instruction_group_list *igl,
12029 uint32_t jmp_instr_id,
12030 struct instruction *jmp_instr,
12031 struct instruction_data *jmp_data,
12034 struct instruction_group *jmp_g, *g;
12035 struct instruction_data *data;
12038 switch (jmp_instr->type) {
12042 case INSTR_JMP_VALID:
12044 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12048 case INSTR_JMP_INVALID:
12050 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12054 case INSTR_JMP_HIT:
12059 case INSTR_JMP_MISS:
12064 case INSTR_JMP_ACTION_HIT:
12066 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12070 case INSTR_JMP_ACTION_MISS:
12072 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12078 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12079 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12084 case INSTR_JMP_EQ_MH:
12086 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12087 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12092 case INSTR_JMP_EQ_HM:
12094 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12095 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12100 case INSTR_JMP_EQ_HH:
12102 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12103 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12108 case INSTR_JMP_EQ_I:
12110 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12111 "pipeline_instructions[%u].jmp.b_val)",
12116 case INSTR_JMP_NEQ:
12118 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12119 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12124 case INSTR_JMP_NEQ_MH:
12126 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12127 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12132 case INSTR_JMP_NEQ_HM:
12134 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12135 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12140 case INSTR_JMP_NEQ_HH:
12142 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12143 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12148 case INSTR_JMP_NEQ_I:
12150 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12151 "pipeline_instructions[%u].jmp.b_val)",
12158 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12159 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12164 case INSTR_JMP_LT_MH:
12166 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12167 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12172 case INSTR_JMP_LT_HM:
12174 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12175 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12180 case INSTR_JMP_LT_HH:
12182 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12183 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12188 case INSTR_JMP_LT_MI:
12190 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12191 "pipeline_instructions[%u].jmp.b_val)",
12196 case INSTR_JMP_LT_HI:
12198 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12199 "pipeline_instructions[%u].jmp.b_val)",
12206 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12207 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12212 case INSTR_JMP_GT_MH:
12214 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12215 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12220 case INSTR_JMP_GT_HM:
12222 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12223 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12228 case INSTR_JMP_GT_HH:
12230 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12231 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12236 case INSTR_JMP_GT_MI:
12238 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12239 "pipeline_instructions[%u].jmp.b_val)",
12244 case INSTR_JMP_GT_HI:
12246 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12247 "pipeline_instructions[%u].jmp.b_val)",
12256 /* Find the instruction group of the jump instruction. */
12257 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12261 /* Find the instruction group of the jump destination instruction. */
12262 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12266 instr_id = data - p->instruction_data;
12268 g = instruction_group_list_group_find(igl, instr_id);
12272 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12273 * instruction group).
12275 if (g->group_id == jmp_g->group_id)
12277 "\n\t\tgoto %s;\n",
12278 jmp_data->jmp_label);
12282 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12291 instruction_group_list_codegen(struct instruction_group_list *igl,
12292 struct rte_swx_pipeline *p,
12295 struct instruction_group *g;
12297 int is_required = 0;
12299 /* Check if code generation is required. */
12300 TAILQ_FOREACH(g, igl, node)
12301 if (g->first_instr_id < g->last_instr_id)
12307 /* Generate the code for the pipeline instruction array. */
12309 "static const struct instruction pipeline_instructions[] = {\n");
12311 for (i = 0; i < p->n_instructions; i++) {
12312 struct instruction *instr = &p->instructions[i];
12313 instruction_export_t func = export_table[instr->type];
12318 fprintf(f, "};\n\n");
12320 /* Generate the code for the pipeline functions: one function for each instruction group
12321 * that contains more than one instruction.
12323 TAILQ_FOREACH(g, igl, node) {
12324 struct instruction *last_instr;
12327 /* Skip if group contains a single instruction. */
12328 if (g->last_instr_id == g->first_instr_id)
12331 /* Generate new pipeline function. */
12334 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12336 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12340 /* Generate the code for each pipeline instruction. */
12341 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12342 struct instruction *instr = &p->instructions[j];
12343 struct instruction_data *data = &p->instruction_data[j];
12345 /* Label, if present. */
12346 if (data->label[0])
12347 fprintf(f, "\n%s : ", data->label);
12349 fprintf(f, "\n\t");
12351 /* TX instruction type. */
12352 if (instruction_does_tx(instr)) {
12353 pipeline_instr_does_tx_codegen(p, j, instr, f);
12357 /* Jump instruction type. */
12358 if (instruction_is_jmp(instr)) {
12359 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12363 /* Any other instruction type. */
12365 "%s(p, t, &pipeline_instructions[%u]);\n",
12366 instr_type_to_func(instr),
12370 /* Finalize the generated pipeline function. For some instructions such as TX,
12371 * emit-many-and-TX and unconditional jump, the next instruction has been already
12372 * decided unconditionally and the instruction pointer of the current thread set
12373 * accordingly; for all the other instructions, the instruction pointer must be
12376 last_instr = &p->instructions[g->last_instr_id];
12378 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12380 "thread_ip_inc(p);\n");
12389 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12391 struct instruction_group *g;
12392 uint32_t n_custom_instr = 0;
12394 /* Groups with a single instruction: no function is generated for this group, the group
12395 * keeps its current instruction. Groups with more than two instructions: one function and
12396 * the associated custom instruction get generated for each such group.
12398 TAILQ_FOREACH(g, igl, node) {
12399 if (g->first_instr_id == g->last_instr_id)
12405 return n_custom_instr;
12409 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12414 /* Create the .c file. */
12415 f = fopen("/tmp/pipeline.c", "w");
12419 /* Include the .h file. */
12420 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12422 /* Add the code for each action. */
12423 TAILQ_FOREACH(a, &p->actions, node) {
12424 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12426 action_data_codegen(a, f);
12430 action_instr_codegen(a, f);
12435 /* Add the pipeline code. */
12436 instruction_group_list_codegen(igl, p, f);
12438 /* Close the .c file. */
12444 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12445 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12449 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12452 struct instruction_group *g;
12453 char *dir_in, *buffer = NULL;
12454 const char *dir_out;
12457 /* Get the environment variables. */
12458 dir_in = getenv("RTE_INSTALL_DIR");
12466 /* Memory allocation for the command buffer. */
12467 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12474 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12475 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12476 "-I %s/lib/pipeline "
12477 "-I %s/lib/eal/include "
12478 "-I %s/lib/eal/x86/include "
12479 "-I %s/lib/eal/include/generic "
12483 "-I %s/lib/pipeline "
12486 "-I %s/lib/eal/linux/include "
12487 ">%s/pipeline.log 2>&1 "
12489 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12490 ">>%s/pipeline.log 2>&1",
12509 /* Build the shared object library. */
12510 status = system(buffer);
12514 /* Open library. */
12516 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12517 "%s/libpipeline.so",
12520 p->lib = dlopen(buffer, RTLD_LAZY);
12526 /* Get the action function symbols. */
12527 TAILQ_FOREACH(a, &p->actions, node) {
12528 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12530 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12531 if (!p->action_funcs[a->id]) {
12537 /* Get the pipeline function symbols. */
12538 TAILQ_FOREACH(g, igl, node) {
12539 if (g->first_instr_id == g->last_instr_id)
12542 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12544 g->func = dlsym(p->lib, buffer);
12552 if (status && p->lib) {
12563 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12564 struct instruction_group_list *igl)
12566 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12568 /* Check that enough space is available within the pipeline instruction table to store all
12569 * the custom instructions.
12571 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12578 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12580 struct instruction_group *g;
12583 /* Pipeline table instructions. */
12584 for (i = 0; i < p->n_instructions; i++) {
12585 struct instruction *instr = &p->instructions[i];
12587 if (instr->type == INSTR_TABLE)
12588 instr->type = INSTR_TABLE_AF;
12590 if (instr->type == INSTR_LEARNER)
12591 instr->type = INSTR_LEARNER_AF;
12594 /* Pipeline custom instructions. */
12596 TAILQ_FOREACH(g, igl, node) {
12597 struct instruction *instr = &p->instructions[g->first_instr_id];
12600 if (g->first_instr_id == g->last_instr_id)
12603 /* Install a new custom instruction. */
12604 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12606 /* First instruction of the group: change its type to the new custom instruction. */
12607 instr->type = INSTR_CUSTOM_0 + i;
12609 /* All the subsequent instructions of the group: invalidate. */
12610 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12611 struct instruction_data *data = &p->instruction_data[j];
12619 /* Remove the invalidated instructions. */
12620 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12622 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12623 * instructions that are the only instruction within their group, so they were left
12626 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12630 pipeline_compile(struct rte_swx_pipeline *p)
12632 struct instruction_group_list *igl = NULL;
12635 igl = instruction_group_list_create(p);
12641 /* Code generation. */
12642 status = pipeline_codegen(p, igl);
12646 /* Build and load the shared object library. */
12647 status = pipeline_libload(p, igl);
12651 /* Adjust instructions. */
12652 status = pipeline_adjust_check(p, igl);
12656 pipeline_adjust(p, igl);
12659 instruction_group_list_free(igl);