1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <rte_swx_port_ethdev.h>
11 #include <rte_swx_port_fd.h>
12 #include <rte_swx_port_ring.h>
13 #include "rte_swx_port_source_sink.h"
15 #include <rte_swx_table_em.h>
16 #include <rte_swx_table_wm.h>
18 #include "rte_swx_pipeline_internal.h"
20 #define CHECK(condition, err_code) \
26 #define CHECK_NAME(name, err_code) \
29 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
32 #define CHECK_INSTRUCTION(instr, err_code) \
35 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
36 RTE_SWX_INSTRUCTION_SIZE), \
42 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
44 #include <rte_malloc.h>
47 env_malloc(size_t size, size_t alignment, int numa_node)
49 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
53 env_free(void *start, size_t size __rte_unused)
63 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
67 if (numa_available() == -1)
70 start = numa_alloc_onnode(size, numa_node);
74 memset(start, 0, size);
79 env_free(void *start, size_t size)
81 if (numa_available() == -1)
84 numa_free(start, size);
92 static struct struct_type *
93 struct_type_find(struct rte_swx_pipeline *p, const char *name)
95 struct struct_type *elem;
97 TAILQ_FOREACH(elem, &p->struct_types, node)
98 if (strcmp(elem->name, name) == 0)
104 static struct field *
105 struct_type_field_find(struct struct_type *st, const char *name)
109 for (i = 0; i < st->n_fields; i++) {
110 struct field *f = &st->fields[i];
112 if (strcmp(f->name, name) == 0)
120 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
122 struct rte_swx_field_params *fields,
124 int last_field_has_variable_size)
126 struct struct_type *st;
130 CHECK_NAME(name, EINVAL);
131 CHECK(fields, EINVAL);
132 CHECK(n_fields, EINVAL);
134 for (i = 0; i < n_fields; i++) {
135 struct rte_swx_field_params *f = &fields[i];
136 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
139 CHECK_NAME(f->name, EINVAL);
140 CHECK(f->n_bits, EINVAL);
141 CHECK((f->n_bits <= 64) || var_size, EINVAL);
142 CHECK((f->n_bits & 7) == 0, EINVAL);
144 for (j = 0; j < i; j++) {
145 struct rte_swx_field_params *f_prev = &fields[j];
147 CHECK(strcmp(f->name, f_prev->name), EINVAL);
151 CHECK(!struct_type_find(p, name), EEXIST);
153 /* Node allocation. */
154 st = calloc(1, sizeof(struct struct_type));
157 st->fields = calloc(n_fields, sizeof(struct field));
163 /* Node initialization. */
164 strcpy(st->name, name);
165 for (i = 0; i < n_fields; i++) {
166 struct field *dst = &st->fields[i];
167 struct rte_swx_field_params *src = &fields[i];
168 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
170 strcpy(dst->name, src->name);
171 dst->n_bits = src->n_bits;
172 dst->offset = st->n_bits;
173 dst->var_size = var_size;
175 st->n_bits += src->n_bits;
176 st->n_bits_min += var_size ? 0 : src->n_bits;
178 st->n_fields = n_fields;
179 st->var_size = last_field_has_variable_size;
181 /* Node add to tailq. */
182 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
188 struct_build(struct rte_swx_pipeline *p)
192 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
193 struct thread *t = &p->threads[i];
195 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
196 CHECK(t->structs, ENOMEM);
203 struct_build_free(struct rte_swx_pipeline *p)
207 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
208 struct thread *t = &p->threads[i];
216 struct_free(struct rte_swx_pipeline *p)
218 struct_build_free(p);
222 struct struct_type *elem;
224 elem = TAILQ_FIRST(&p->struct_types);
228 TAILQ_REMOVE(&p->struct_types, elem, node);
237 static struct port_in_type *
238 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
240 struct port_in_type *elem;
245 TAILQ_FOREACH(elem, &p->port_in_types, node)
246 if (strcmp(elem->name, name) == 0)
253 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
255 struct rte_swx_port_in_ops *ops)
257 struct port_in_type *elem;
260 CHECK_NAME(name, EINVAL);
262 CHECK(ops->create, EINVAL);
263 CHECK(ops->free, EINVAL);
264 CHECK(ops->pkt_rx, EINVAL);
265 CHECK(ops->stats_read, EINVAL);
267 CHECK(!port_in_type_find(p, name), EEXIST);
269 /* Node allocation. */
270 elem = calloc(1, sizeof(struct port_in_type));
273 /* Node initialization. */
274 strcpy(elem->name, name);
275 memcpy(&elem->ops, ops, sizeof(*ops));
277 /* Node add to tailq. */
278 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
283 static struct port_in *
284 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
286 struct port_in *port;
288 TAILQ_FOREACH(port, &p->ports_in, node)
289 if (port->id == port_id)
296 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
298 const char *port_type_name,
301 struct port_in_type *type = NULL;
302 struct port_in *port = NULL;
307 CHECK(!port_in_find(p, port_id), EINVAL);
309 CHECK_NAME(port_type_name, EINVAL);
310 type = port_in_type_find(p, port_type_name);
313 obj = type->ops.create(args);
316 /* Node allocation. */
317 port = calloc(1, sizeof(struct port_in));
320 /* Node initialization. */
325 /* Node add to tailq. */
326 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
327 if (p->n_ports_in < port_id + 1)
328 p->n_ports_in = port_id + 1;
334 port_in_build(struct rte_swx_pipeline *p)
336 struct port_in *port;
339 CHECK(p->n_ports_in, EINVAL);
340 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
342 for (i = 0; i < p->n_ports_in; i++)
343 CHECK(port_in_find(p, i), EINVAL);
345 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
346 CHECK(p->in, ENOMEM);
348 TAILQ_FOREACH(port, &p->ports_in, node) {
349 struct port_in_runtime *in = &p->in[port->id];
351 in->pkt_rx = port->type->ops.pkt_rx;
359 port_in_build_free(struct rte_swx_pipeline *p)
366 port_in_free(struct rte_swx_pipeline *p)
368 port_in_build_free(p);
372 struct port_in *port;
374 port = TAILQ_FIRST(&p->ports_in);
378 TAILQ_REMOVE(&p->ports_in, port, node);
379 port->type->ops.free(port->obj);
383 /* Input port types. */
385 struct port_in_type *elem;
387 elem = TAILQ_FIRST(&p->port_in_types);
391 TAILQ_REMOVE(&p->port_in_types, elem, node);
399 static struct port_out_type *
400 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
402 struct port_out_type *elem;
407 TAILQ_FOREACH(elem, &p->port_out_types, node)
408 if (!strcmp(elem->name, name))
415 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
417 struct rte_swx_port_out_ops *ops)
419 struct port_out_type *elem;
422 CHECK_NAME(name, EINVAL);
424 CHECK(ops->create, EINVAL);
425 CHECK(ops->free, EINVAL);
426 CHECK(ops->pkt_tx, EINVAL);
427 CHECK(ops->stats_read, EINVAL);
429 CHECK(!port_out_type_find(p, name), EEXIST);
431 /* Node allocation. */
432 elem = calloc(1, sizeof(struct port_out_type));
435 /* Node initialization. */
436 strcpy(elem->name, name);
437 memcpy(&elem->ops, ops, sizeof(*ops));
439 /* Node add to tailq. */
440 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
445 static struct port_out *
446 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
448 struct port_out *port;
450 TAILQ_FOREACH(port, &p->ports_out, node)
451 if (port->id == port_id)
458 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
460 const char *port_type_name,
463 struct port_out_type *type = NULL;
464 struct port_out *port = NULL;
469 CHECK(!port_out_find(p, port_id), EINVAL);
471 CHECK_NAME(port_type_name, EINVAL);
472 type = port_out_type_find(p, port_type_name);
475 obj = type->ops.create(args);
478 /* Node allocation. */
479 port = calloc(1, sizeof(struct port_out));
482 /* Node initialization. */
487 /* Node add to tailq. */
488 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
489 if (p->n_ports_out < port_id + 1)
490 p->n_ports_out = port_id + 1;
496 port_out_build(struct rte_swx_pipeline *p)
498 struct port_out *port;
501 CHECK(p->n_ports_out, EINVAL);
503 for (i = 0; i < p->n_ports_out; i++)
504 CHECK(port_out_find(p, i), EINVAL);
506 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
507 CHECK(p->out, ENOMEM);
509 TAILQ_FOREACH(port, &p->ports_out, node) {
510 struct port_out_runtime *out = &p->out[port->id];
512 out->pkt_tx = port->type->ops.pkt_tx;
513 out->flush = port->type->ops.flush;
514 out->obj = port->obj;
521 port_out_build_free(struct rte_swx_pipeline *p)
528 port_out_free(struct rte_swx_pipeline *p)
530 port_out_build_free(p);
534 struct port_out *port;
536 port = TAILQ_FIRST(&p->ports_out);
540 TAILQ_REMOVE(&p->ports_out, port, node);
541 port->type->ops.free(port->obj);
545 /* Output port types. */
547 struct port_out_type *elem;
549 elem = TAILQ_FIRST(&p->port_out_types);
553 TAILQ_REMOVE(&p->port_out_types, elem, node);
561 static struct extern_type *
562 extern_type_find(struct rte_swx_pipeline *p, const char *name)
564 struct extern_type *elem;
566 TAILQ_FOREACH(elem, &p->extern_types, node)
567 if (strcmp(elem->name, name) == 0)
573 static struct extern_type_member_func *
574 extern_type_member_func_find(struct extern_type *type, const char *name)
576 struct extern_type_member_func *elem;
578 TAILQ_FOREACH(elem, &type->funcs, node)
579 if (strcmp(elem->name, name) == 0)
585 static struct extern_obj *
586 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
588 struct extern_obj *elem;
590 TAILQ_FOREACH(elem, &p->extern_objs, node)
591 if (strcmp(elem->name, name) == 0)
597 static struct extern_type_member_func *
598 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
600 struct extern_obj **obj)
602 struct extern_obj *object;
603 struct extern_type_member_func *func;
604 char *object_name, *func_name;
606 if (name[0] != 'e' || name[1] != '.')
609 object_name = strdup(&name[2]);
613 func_name = strchr(object_name, '.');
622 object = extern_obj_find(p, object_name);
628 func = extern_type_member_func_find(object->type, func_name);
641 static struct field *
642 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
644 struct extern_obj **object)
646 struct extern_obj *obj;
648 char *obj_name, *field_name;
650 if ((name[0] != 'e') || (name[1] != '.'))
653 obj_name = strdup(&name[2]);
657 field_name = strchr(obj_name, '.');
666 obj = extern_obj_find(p, obj_name);
672 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
686 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
688 const char *mailbox_struct_type_name,
689 rte_swx_extern_type_constructor_t constructor,
690 rte_swx_extern_type_destructor_t destructor)
692 struct extern_type *elem;
693 struct struct_type *mailbox_struct_type;
697 CHECK_NAME(name, EINVAL);
698 CHECK(!extern_type_find(p, name), EEXIST);
700 CHECK_NAME(mailbox_struct_type_name, EINVAL);
701 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
702 CHECK(mailbox_struct_type, EINVAL);
703 CHECK(!mailbox_struct_type->var_size, EINVAL);
705 CHECK(constructor, EINVAL);
706 CHECK(destructor, EINVAL);
708 /* Node allocation. */
709 elem = calloc(1, sizeof(struct extern_type));
712 /* Node initialization. */
713 strcpy(elem->name, name);
714 elem->mailbox_struct_type = mailbox_struct_type;
715 elem->constructor = constructor;
716 elem->destructor = destructor;
717 TAILQ_INIT(&elem->funcs);
719 /* Node add to tailq. */
720 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
726 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
727 const char *extern_type_name,
729 rte_swx_extern_type_member_func_t member_func)
731 struct extern_type *type;
732 struct extern_type_member_func *type_member;
736 CHECK_NAME(extern_type_name, EINVAL);
737 type = extern_type_find(p, extern_type_name);
739 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
741 CHECK_NAME(name, EINVAL);
742 CHECK(!extern_type_member_func_find(type, name), EEXIST);
744 CHECK(member_func, EINVAL);
746 /* Node allocation. */
747 type_member = calloc(1, sizeof(struct extern_type_member_func));
748 CHECK(type_member, ENOMEM);
750 /* Node initialization. */
751 strcpy(type_member->name, name);
752 type_member->func = member_func;
753 type_member->id = type->n_funcs;
755 /* Node add to tailq. */
756 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
763 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
764 const char *extern_type_name,
768 struct extern_type *type;
769 struct extern_obj *obj;
774 CHECK_NAME(extern_type_name, EINVAL);
775 type = extern_type_find(p, extern_type_name);
778 CHECK_NAME(name, EINVAL);
779 CHECK(!extern_obj_find(p, name), EEXIST);
781 /* Node allocation. */
782 obj = calloc(1, sizeof(struct extern_obj));
785 /* Object construction. */
786 obj_handle = type->constructor(args);
792 /* Node initialization. */
793 strcpy(obj->name, name);
795 obj->obj = obj_handle;
796 obj->struct_id = p->n_structs;
797 obj->id = p->n_extern_objs;
799 /* Node add to tailq. */
800 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
808 extern_obj_build(struct rte_swx_pipeline *p)
812 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
813 struct thread *t = &p->threads[i];
814 struct extern_obj *obj;
816 t->extern_objs = calloc(p->n_extern_objs,
817 sizeof(struct extern_obj_runtime));
818 CHECK(t->extern_objs, ENOMEM);
820 TAILQ_FOREACH(obj, &p->extern_objs, node) {
821 struct extern_obj_runtime *r =
822 &t->extern_objs[obj->id];
823 struct extern_type_member_func *func;
824 uint32_t mailbox_size =
825 obj->type->mailbox_struct_type->n_bits / 8;
829 r->mailbox = calloc(1, mailbox_size);
830 CHECK(r->mailbox, ENOMEM);
832 TAILQ_FOREACH(func, &obj->type->funcs, node)
833 r->funcs[func->id] = func->func;
835 t->structs[obj->struct_id] = r->mailbox;
843 extern_obj_build_free(struct rte_swx_pipeline *p)
847 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
848 struct thread *t = &p->threads[i];
854 for (j = 0; j < p->n_extern_objs; j++) {
855 struct extern_obj_runtime *r = &t->extern_objs[j];
860 free(t->extern_objs);
861 t->extern_objs = NULL;
866 extern_obj_free(struct rte_swx_pipeline *p)
868 extern_obj_build_free(p);
870 /* Extern objects. */
872 struct extern_obj *elem;
874 elem = TAILQ_FIRST(&p->extern_objs);
878 TAILQ_REMOVE(&p->extern_objs, elem, node);
880 elem->type->destructor(elem->obj);
886 struct extern_type *elem;
888 elem = TAILQ_FIRST(&p->extern_types);
892 TAILQ_REMOVE(&p->extern_types, elem, node);
895 struct extern_type_member_func *func;
897 func = TAILQ_FIRST(&elem->funcs);
901 TAILQ_REMOVE(&elem->funcs, func, node);
912 static struct extern_func *
913 extern_func_find(struct rte_swx_pipeline *p, const char *name)
915 struct extern_func *elem;
917 TAILQ_FOREACH(elem, &p->extern_funcs, node)
918 if (strcmp(elem->name, name) == 0)
924 static struct extern_func *
925 extern_func_parse(struct rte_swx_pipeline *p,
928 if (name[0] != 'f' || name[1] != '.')
931 return extern_func_find(p, &name[2]);
934 static struct field *
935 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
937 struct extern_func **function)
939 struct extern_func *func;
941 char *func_name, *field_name;
943 if ((name[0] != 'f') || (name[1] != '.'))
946 func_name = strdup(&name[2]);
950 field_name = strchr(func_name, '.');
959 func = extern_func_find(p, func_name);
965 f = struct_type_field_find(func->mailbox_struct_type, field_name);
979 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
981 const char *mailbox_struct_type_name,
982 rte_swx_extern_func_t func)
984 struct extern_func *f;
985 struct struct_type *mailbox_struct_type;
989 CHECK_NAME(name, EINVAL);
990 CHECK(!extern_func_find(p, name), EEXIST);
992 CHECK_NAME(mailbox_struct_type_name, EINVAL);
993 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
994 CHECK(mailbox_struct_type, EINVAL);
995 CHECK(!mailbox_struct_type->var_size, EINVAL);
999 /* Node allocation. */
1000 f = calloc(1, sizeof(struct extern_func));
1001 CHECK(func, ENOMEM);
1003 /* Node initialization. */
1004 strcpy(f->name, name);
1005 f->mailbox_struct_type = mailbox_struct_type;
1007 f->struct_id = p->n_structs;
1008 f->id = p->n_extern_funcs;
1010 /* Node add to tailq. */
1011 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1012 p->n_extern_funcs++;
1019 extern_func_build(struct rte_swx_pipeline *p)
1023 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1024 struct thread *t = &p->threads[i];
1025 struct extern_func *func;
1027 /* Memory allocation. */
1028 t->extern_funcs = calloc(p->n_extern_funcs,
1029 sizeof(struct extern_func_runtime));
1030 CHECK(t->extern_funcs, ENOMEM);
1032 /* Extern function. */
1033 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1034 struct extern_func_runtime *r =
1035 &t->extern_funcs[func->id];
1036 uint32_t mailbox_size =
1037 func->mailbox_struct_type->n_bits / 8;
1039 r->func = func->func;
1041 r->mailbox = calloc(1, mailbox_size);
1042 CHECK(r->mailbox, ENOMEM);
1044 t->structs[func->struct_id] = r->mailbox;
1052 extern_func_build_free(struct rte_swx_pipeline *p)
1056 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1057 struct thread *t = &p->threads[i];
1060 if (!t->extern_funcs)
1063 for (j = 0; j < p->n_extern_funcs; j++) {
1064 struct extern_func_runtime *r = &t->extern_funcs[j];
1069 free(t->extern_funcs);
1070 t->extern_funcs = NULL;
1075 extern_func_free(struct rte_swx_pipeline *p)
1077 extern_func_build_free(p);
1080 struct extern_func *elem;
1082 elem = TAILQ_FIRST(&p->extern_funcs);
1086 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1094 static struct header *
1095 header_find(struct rte_swx_pipeline *p, const char *name)
1097 struct header *elem;
1099 TAILQ_FOREACH(elem, &p->headers, node)
1100 if (strcmp(elem->name, name) == 0)
1106 static struct header *
1107 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1109 struct header *elem;
1111 TAILQ_FOREACH(elem, &p->headers, node)
1112 if (elem->struct_id == struct_id)
1118 static struct header *
1119 header_parse(struct rte_swx_pipeline *p,
1122 if (name[0] != 'h' || name[1] != '.')
1125 return header_find(p, &name[2]);
1128 static struct field *
1129 header_field_parse(struct rte_swx_pipeline *p,
1131 struct header **header)
1135 char *header_name, *field_name;
1137 if ((name[0] != 'h') || (name[1] != '.'))
1140 header_name = strdup(&name[2]);
1144 field_name = strchr(header_name, '.');
1153 h = header_find(p, header_name);
1159 f = struct_type_field_find(h->st, field_name);
1173 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1175 const char *struct_type_name)
1177 struct struct_type *st;
1179 size_t n_headers_max;
1182 CHECK_NAME(name, EINVAL);
1183 CHECK_NAME(struct_type_name, EINVAL);
1185 CHECK(!header_find(p, name), EEXIST);
1187 st = struct_type_find(p, struct_type_name);
1190 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1191 CHECK(p->n_headers < n_headers_max, ENOSPC);
1193 /* Node allocation. */
1194 h = calloc(1, sizeof(struct header));
1197 /* Node initialization. */
1198 strcpy(h->name, name);
1200 h->struct_id = p->n_structs;
1201 h->id = p->n_headers;
1203 /* Node add to tailq. */
1204 TAILQ_INSERT_TAIL(&p->headers, h, node);
1212 header_build(struct rte_swx_pipeline *p)
1215 uint32_t n_bytes = 0, i;
1217 TAILQ_FOREACH(h, &p->headers, node) {
1218 n_bytes += h->st->n_bits / 8;
1221 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1222 struct thread *t = &p->threads[i];
1223 uint32_t offset = 0;
1225 t->headers = calloc(p->n_headers,
1226 sizeof(struct header_runtime));
1227 CHECK(t->headers, ENOMEM);
1229 t->headers_out = calloc(p->n_headers,
1230 sizeof(struct header_out_runtime));
1231 CHECK(t->headers_out, ENOMEM);
1233 t->header_storage = calloc(1, n_bytes);
1234 CHECK(t->header_storage, ENOMEM);
1236 t->header_out_storage = calloc(1, n_bytes);
1237 CHECK(t->header_out_storage, ENOMEM);
1239 TAILQ_FOREACH(h, &p->headers, node) {
1240 uint8_t *header_storage;
1241 uint32_t n_bytes = h->st->n_bits / 8;
1243 header_storage = &t->header_storage[offset];
1246 t->headers[h->id].ptr0 = header_storage;
1247 t->headers[h->id].n_bytes = n_bytes;
1249 t->structs[h->struct_id] = header_storage;
1257 header_build_free(struct rte_swx_pipeline *p)
1261 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1262 struct thread *t = &p->threads[i];
1264 free(t->headers_out);
1265 t->headers_out = NULL;
1270 free(t->header_out_storage);
1271 t->header_out_storage = NULL;
1273 free(t->header_storage);
1274 t->header_storage = NULL;
1279 header_free(struct rte_swx_pipeline *p)
1281 header_build_free(p);
1284 struct header *elem;
1286 elem = TAILQ_FIRST(&p->headers);
1290 TAILQ_REMOVE(&p->headers, elem, node);
1298 static struct field *
1299 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1301 if (!p->metadata_st)
1304 if (name[0] != 'm' || name[1] != '.')
1307 return struct_type_field_find(p->metadata_st, &name[2]);
1311 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1312 const char *struct_type_name)
1314 struct struct_type *st = NULL;
1318 CHECK_NAME(struct_type_name, EINVAL);
1319 st = struct_type_find(p, struct_type_name);
1321 CHECK(!st->var_size, EINVAL);
1322 CHECK(!p->metadata_st, EINVAL);
1324 p->metadata_st = st;
1325 p->metadata_struct_id = p->n_structs;
1333 metadata_build(struct rte_swx_pipeline *p)
1335 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1338 /* Thread-level initialization. */
1339 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1340 struct thread *t = &p->threads[i];
1343 metadata = calloc(1, n_bytes);
1344 CHECK(metadata, ENOMEM);
1346 t->metadata = metadata;
1347 t->structs[p->metadata_struct_id] = metadata;
1354 metadata_build_free(struct rte_swx_pipeline *p)
1358 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1359 struct thread *t = &p->threads[i];
1367 metadata_free(struct rte_swx_pipeline *p)
1369 metadata_build_free(p);
1376 instruction_is_tx(enum instruction_type type)
1390 instruction_does_tx(struct instruction *instr)
1392 switch (instr->type) {
1396 case INSTR_HDR_EMIT_TX:
1397 case INSTR_HDR_EMIT2_TX:
1398 case INSTR_HDR_EMIT3_TX:
1399 case INSTR_HDR_EMIT4_TX:
1400 case INSTR_HDR_EMIT5_TX:
1401 case INSTR_HDR_EMIT6_TX:
1402 case INSTR_HDR_EMIT7_TX:
1403 case INSTR_HDR_EMIT8_TX:
1411 instruction_is_jmp(struct instruction *instr)
1413 switch (instr->type) {
1415 case INSTR_JMP_VALID:
1416 case INSTR_JMP_INVALID:
1418 case INSTR_JMP_MISS:
1419 case INSTR_JMP_ACTION_HIT:
1420 case INSTR_JMP_ACTION_MISS:
1422 case INSTR_JMP_EQ_MH:
1423 case INSTR_JMP_EQ_HM:
1424 case INSTR_JMP_EQ_HH:
1425 case INSTR_JMP_EQ_I:
1427 case INSTR_JMP_NEQ_MH:
1428 case INSTR_JMP_NEQ_HM:
1429 case INSTR_JMP_NEQ_HH:
1430 case INSTR_JMP_NEQ_I:
1432 case INSTR_JMP_LT_MH:
1433 case INSTR_JMP_LT_HM:
1434 case INSTR_JMP_LT_HH:
1435 case INSTR_JMP_LT_MI:
1436 case INSTR_JMP_LT_HI:
1438 case INSTR_JMP_GT_MH:
1439 case INSTR_JMP_GT_HM:
1440 case INSTR_JMP_GT_HH:
1441 case INSTR_JMP_GT_MI:
1442 case INSTR_JMP_GT_HI:
1451 instruction_does_thread_yield(struct instruction *instr)
1453 switch (instr->type) {
1456 case INSTR_TABLE_AF:
1457 case INSTR_SELECTOR:
1459 case INSTR_LEARNER_AF:
1460 case INSTR_EXTERN_OBJ:
1461 case INSTR_EXTERN_FUNC:
1468 static struct field *
1469 action_field_parse(struct action *action, const char *name);
1471 static struct field *
1472 struct_field_parse(struct rte_swx_pipeline *p,
1473 struct action *action,
1475 uint32_t *struct_id)
1482 struct header *header;
1484 f = header_field_parse(p, name, &header);
1488 *struct_id = header->struct_id;
1494 f = metadata_field_parse(p, name);
1498 *struct_id = p->metadata_struct_id;
1507 f = action_field_parse(action, name);
1517 struct extern_obj *obj;
1519 f = extern_obj_mailbox_field_parse(p, name, &obj);
1523 *struct_id = obj->struct_id;
1529 struct extern_func *func;
1531 f = extern_func_mailbox_field_parse(p, name, &func);
1535 *struct_id = func->struct_id;
1548 instr_rx_translate(struct rte_swx_pipeline *p,
1549 struct action *action,
1552 struct instruction *instr,
1553 struct instruction_data *data __rte_unused)
1557 CHECK(!action, EINVAL);
1558 CHECK(n_tokens == 2, EINVAL);
1560 f = metadata_field_parse(p, tokens[1]);
1563 instr->type = INSTR_RX;
1564 instr->io.io.offset = f->offset / 8;
1565 instr->io.io.n_bits = f->n_bits;
1573 instr_tx_translate(struct rte_swx_pipeline *p,
1574 struct action *action __rte_unused,
1577 struct instruction *instr,
1578 struct instruction_data *data __rte_unused)
1580 char *port = tokens[1];
1584 CHECK(n_tokens == 2, EINVAL);
1586 f = metadata_field_parse(p, port);
1588 instr->type = INSTR_TX;
1589 instr->io.io.offset = f->offset / 8;
1590 instr->io.io.n_bits = f->n_bits;
1595 port_val = strtoul(port, &port, 0);
1596 CHECK(!port[0], EINVAL);
1598 instr->type = INSTR_TX_I;
1599 instr->io.io.val = port_val;
1604 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1605 struct action *action __rte_unused,
1606 char **tokens __rte_unused,
1608 struct instruction *instr,
1609 struct instruction_data *data __rte_unused)
1611 CHECK(n_tokens == 1, EINVAL);
1614 instr->type = INSTR_DROP;
1619 instr_tx_exec(struct rte_swx_pipeline *p)
1621 struct thread *t = &p->threads[p->thread_id];
1622 struct instruction *ip = t->ip;
1624 __instr_tx_exec(p, t, ip);
1627 thread_ip_reset(p, t);
1632 instr_tx_i_exec(struct rte_swx_pipeline *p)
1634 struct thread *t = &p->threads[p->thread_id];
1635 struct instruction *ip = t->ip;
1637 __instr_tx_i_exec(p, t, ip);
1640 thread_ip_reset(p, t);
1645 instr_drop_exec(struct rte_swx_pipeline *p)
1647 struct thread *t = &p->threads[p->thread_id];
1648 struct instruction *ip = t->ip;
1650 __instr_drop_exec(p, t, ip);
1653 thread_ip_reset(p, t);
1661 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1662 struct action *action,
1665 struct instruction *instr,
1666 struct instruction_data *data __rte_unused)
1670 CHECK(!action, EINVAL);
1671 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1673 h = header_parse(p, tokens[1]);
1676 if (n_tokens == 2) {
1677 CHECK(!h->st->var_size, EINVAL);
1679 instr->type = INSTR_HDR_EXTRACT;
1680 instr->io.hdr.header_id[0] = h->id;
1681 instr->io.hdr.struct_id[0] = h->struct_id;
1682 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1686 CHECK(h->st->var_size, EINVAL);
1688 mf = metadata_field_parse(p, tokens[2]);
1690 CHECK(!mf->var_size, EINVAL);
1692 instr->type = INSTR_HDR_EXTRACT_M;
1693 instr->io.io.offset = mf->offset / 8;
1694 instr->io.io.n_bits = mf->n_bits;
1695 instr->io.hdr.header_id[0] = h->id;
1696 instr->io.hdr.struct_id[0] = h->struct_id;
1697 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1704 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1705 struct action *action,
1708 struct instruction *instr,
1709 struct instruction_data *data __rte_unused)
1713 CHECK(!action, EINVAL);
1714 CHECK(n_tokens == 2, EINVAL);
1716 h = header_parse(p, tokens[1]);
1718 CHECK(!h->st->var_size, EINVAL);
1720 instr->type = INSTR_HDR_LOOKAHEAD;
1721 instr->io.hdr.header_id[0] = h->id;
1722 instr->io.hdr.struct_id[0] = h->struct_id;
1723 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1729 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1731 struct thread *t = &p->threads[p->thread_id];
1732 struct instruction *ip = t->ip;
1734 __instr_hdr_extract_exec(p, t, ip);
1741 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1743 struct thread *t = &p->threads[p->thread_id];
1744 struct instruction *ip = t->ip;
1746 __instr_hdr_extract2_exec(p, t, ip);
1753 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1755 struct thread *t = &p->threads[p->thread_id];
1756 struct instruction *ip = t->ip;
1758 __instr_hdr_extract3_exec(p, t, ip);
1765 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1767 struct thread *t = &p->threads[p->thread_id];
1768 struct instruction *ip = t->ip;
1770 __instr_hdr_extract4_exec(p, t, ip);
1777 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1779 struct thread *t = &p->threads[p->thread_id];
1780 struct instruction *ip = t->ip;
1782 __instr_hdr_extract5_exec(p, t, ip);
1789 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1791 struct thread *t = &p->threads[p->thread_id];
1792 struct instruction *ip = t->ip;
1794 __instr_hdr_extract6_exec(p, t, ip);
1801 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1803 struct thread *t = &p->threads[p->thread_id];
1804 struct instruction *ip = t->ip;
1806 __instr_hdr_extract7_exec(p, t, ip);
1813 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1815 struct thread *t = &p->threads[p->thread_id];
1816 struct instruction *ip = t->ip;
1818 __instr_hdr_extract8_exec(p, t, ip);
1825 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1827 struct thread *t = &p->threads[p->thread_id];
1828 struct instruction *ip = t->ip;
1830 __instr_hdr_extract_m_exec(p, t, ip);
1837 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1839 struct thread *t = &p->threads[p->thread_id];
1840 struct instruction *ip = t->ip;
1842 __instr_hdr_lookahead_exec(p, t, ip);
1852 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1853 struct action *action __rte_unused,
1856 struct instruction *instr,
1857 struct instruction_data *data __rte_unused)
1861 CHECK(n_tokens == 2, EINVAL);
1863 h = header_parse(p, tokens[1]);
1866 instr->type = INSTR_HDR_EMIT;
1867 instr->io.hdr.header_id[0] = h->id;
1868 instr->io.hdr.struct_id[0] = h->struct_id;
1869 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1874 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1876 struct thread *t = &p->threads[p->thread_id];
1877 struct instruction *ip = t->ip;
1879 __instr_hdr_emit_exec(p, t, ip);
1886 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1888 struct thread *t = &p->threads[p->thread_id];
1889 struct instruction *ip = t->ip;
1891 __instr_hdr_emit_tx_exec(p, t, ip);
1894 thread_ip_reset(p, t);
1899 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1901 struct thread *t = &p->threads[p->thread_id];
1902 struct instruction *ip = t->ip;
1904 __instr_hdr_emit2_tx_exec(p, t, ip);
1907 thread_ip_reset(p, t);
1912 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1914 struct thread *t = &p->threads[p->thread_id];
1915 struct instruction *ip = t->ip;
1917 __instr_hdr_emit3_tx_exec(p, t, ip);
1920 thread_ip_reset(p, t);
1925 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1927 struct thread *t = &p->threads[p->thread_id];
1928 struct instruction *ip = t->ip;
1930 __instr_hdr_emit4_tx_exec(p, t, ip);
1933 thread_ip_reset(p, t);
1938 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1940 struct thread *t = &p->threads[p->thread_id];
1941 struct instruction *ip = t->ip;
1943 __instr_hdr_emit5_tx_exec(p, t, ip);
1946 thread_ip_reset(p, t);
1951 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1953 struct thread *t = &p->threads[p->thread_id];
1954 struct instruction *ip = t->ip;
1956 __instr_hdr_emit6_tx_exec(p, t, ip);
1959 thread_ip_reset(p, t);
1964 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1966 struct thread *t = &p->threads[p->thread_id];
1967 struct instruction *ip = t->ip;
1969 __instr_hdr_emit7_tx_exec(p, t, ip);
1972 thread_ip_reset(p, t);
1977 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1979 struct thread *t = &p->threads[p->thread_id];
1980 struct instruction *ip = t->ip;
1982 __instr_hdr_emit8_tx_exec(p, t, ip);
1985 thread_ip_reset(p, t);
1993 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1994 struct action *action __rte_unused,
1997 struct instruction *instr,
1998 struct instruction_data *data __rte_unused)
2002 CHECK(n_tokens == 2, EINVAL);
2004 h = header_parse(p, tokens[1]);
2007 instr->type = INSTR_HDR_VALIDATE;
2008 instr->valid.header_id = h->id;
2013 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2015 struct thread *t = &p->threads[p->thread_id];
2016 struct instruction *ip = t->ip;
2018 __instr_hdr_validate_exec(p, t, ip);
2028 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2029 struct action *action __rte_unused,
2032 struct instruction *instr,
2033 struct instruction_data *data __rte_unused)
2037 CHECK(n_tokens == 2, EINVAL);
2039 h = header_parse(p, tokens[1]);
2042 instr->type = INSTR_HDR_INVALIDATE;
2043 instr->valid.header_id = h->id;
2048 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2050 struct thread *t = &p->threads[p->thread_id];
2051 struct instruction *ip = t->ip;
2053 __instr_hdr_invalidate_exec(p, t, ip);
2062 static struct table *
2063 table_find(struct rte_swx_pipeline *p, const char *name);
2065 static struct selector *
2066 selector_find(struct rte_swx_pipeline *p, const char *name);
2068 static struct learner *
2069 learner_find(struct rte_swx_pipeline *p, const char *name);
2072 instr_table_translate(struct rte_swx_pipeline *p,
2073 struct action *action,
2076 struct instruction *instr,
2077 struct instruction_data *data __rte_unused)
2083 CHECK(!action, EINVAL);
2084 CHECK(n_tokens == 2, EINVAL);
2086 t = table_find(p, tokens[1]);
2088 instr->type = INSTR_TABLE;
2089 instr->table.table_id = t->id;
2093 s = selector_find(p, tokens[1]);
2095 instr->type = INSTR_SELECTOR;
2096 instr->table.table_id = s->id;
2100 l = learner_find(p, tokens[1]);
2102 instr->type = INSTR_LEARNER;
2103 instr->table.table_id = l->id;
2111 instr_table_exec(struct rte_swx_pipeline *p)
2113 struct thread *t = &p->threads[p->thread_id];
2114 struct instruction *ip = t->ip;
2115 uint32_t table_id = ip->table.table_id;
2116 struct rte_swx_table_state *ts = &t->table_state[table_id];
2117 struct table_runtime *table = &t->tables[table_id];
2118 struct table_statistics *stats = &p->table_stats[table_id];
2119 uint64_t action_id, n_pkts_hit, n_pkts_action;
2120 uint8_t *action_data;
2124 done = table->func(ts->obj,
2132 TRACE("[Thread %2u] table %u (not finalized)\n",
2140 action_id = hit ? action_id : ts->default_action_id;
2141 action_data = hit ? action_data : ts->default_action_data;
2142 n_pkts_hit = stats->n_pkts_hit[hit];
2143 n_pkts_action = stats->n_pkts_action[action_id];
2145 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2148 hit ? "hit" : "miss",
2149 (uint32_t)action_id);
2151 t->action_id = action_id;
2152 t->structs[0] = action_data;
2154 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2155 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2158 thread_ip_action_call(p, t, action_id);
2162 instr_table_af_exec(struct rte_swx_pipeline *p)
2164 struct thread *t = &p->threads[p->thread_id];
2165 struct instruction *ip = t->ip;
2166 uint32_t table_id = ip->table.table_id;
2167 struct rte_swx_table_state *ts = &t->table_state[table_id];
2168 struct table_runtime *table = &t->tables[table_id];
2169 struct table_statistics *stats = &p->table_stats[table_id];
2170 uint64_t action_id, n_pkts_hit, n_pkts_action;
2171 uint8_t *action_data;
2172 action_func_t action_func;
2176 done = table->func(ts->obj,
2184 TRACE("[Thread %2u] table %u (not finalized)\n",
2192 action_id = hit ? action_id : ts->default_action_id;
2193 action_data = hit ? action_data : ts->default_action_data;
2194 action_func = p->action_funcs[action_id];
2195 n_pkts_hit = stats->n_pkts_hit[hit];
2196 n_pkts_action = stats->n_pkts_action[action_id];
2198 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2201 hit ? "hit" : "miss",
2202 (uint32_t)action_id);
2204 t->action_id = action_id;
2205 t->structs[0] = action_data;
2207 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2208 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2218 instr_selector_exec(struct rte_swx_pipeline *p)
2220 struct thread *t = &p->threads[p->thread_id];
2221 struct instruction *ip = t->ip;
2222 uint32_t selector_id = ip->table.table_id;
2223 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2224 struct selector_runtime *selector = &t->selectors[selector_id];
2225 struct selector_statistics *stats = &p->selector_stats[selector_id];
2226 uint64_t n_pkts = stats->n_pkts;
2230 done = rte_swx_table_selector_select(ts->obj,
2232 selector->group_id_buffer,
2233 selector->selector_buffer,
2234 selector->member_id_buffer);
2237 TRACE("[Thread %2u] selector %u (not finalized)\n",
2246 TRACE("[Thread %2u] selector %u\n",
2250 stats->n_pkts = n_pkts + 1;
2257 instr_learner_exec(struct rte_swx_pipeline *p)
2259 struct thread *t = &p->threads[p->thread_id];
2260 struct instruction *ip = t->ip;
2261 uint32_t learner_id = ip->table.table_id;
2262 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2263 p->n_selectors + learner_id];
2264 struct learner_runtime *l = &t->learners[learner_id];
2265 struct learner_statistics *stats = &p->learner_stats[learner_id];
2266 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2267 uint8_t *action_data;
2271 time = rte_get_tsc_cycles();
2273 done = rte_swx_table_learner_lookup(ts->obj,
2282 TRACE("[Thread %2u] learner %u (not finalized)\n",
2290 action_id = hit ? action_id : ts->default_action_id;
2291 action_data = hit ? action_data : ts->default_action_data;
2292 n_pkts_hit = stats->n_pkts_hit[hit];
2293 n_pkts_action = stats->n_pkts_action[action_id];
2295 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2298 hit ? "hit" : "miss",
2299 (uint32_t)action_id);
2301 t->action_id = action_id;
2302 t->structs[0] = action_data;
2304 t->learner_id = learner_id;
2306 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2307 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2310 thread_ip_action_call(p, t, action_id);
2314 instr_learner_af_exec(struct rte_swx_pipeline *p)
2316 struct thread *t = &p->threads[p->thread_id];
2317 struct instruction *ip = t->ip;
2318 uint32_t learner_id = ip->table.table_id;
2319 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2320 p->n_selectors + learner_id];
2321 struct learner_runtime *l = &t->learners[learner_id];
2322 struct learner_statistics *stats = &p->learner_stats[learner_id];
2323 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2324 uint8_t *action_data;
2325 action_func_t action_func;
2329 time = rte_get_tsc_cycles();
2331 done = rte_swx_table_learner_lookup(ts->obj,
2340 TRACE("[Thread %2u] learner %u (not finalized)\n",
2348 action_id = hit ? action_id : ts->default_action_id;
2349 action_data = hit ? action_data : ts->default_action_data;
2350 action_func = p->action_funcs[action_id];
2351 n_pkts_hit = stats->n_pkts_hit[hit];
2352 n_pkts_action = stats->n_pkts_action[action_id];
2354 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2357 hit ? "hit" : "miss",
2358 (uint32_t)action_id);
2360 t->action_id = action_id;
2361 t->structs[0] = action_data;
2363 t->learner_id = learner_id;
2365 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2366 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2369 thread_ip_action_call(p, t, action_id);
2378 static struct action *
2379 action_find(struct rte_swx_pipeline *p, const char *name);
2382 action_has_nbo_args(struct action *a);
2385 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2388 instr_learn_translate(struct rte_swx_pipeline *p,
2389 struct action *action,
2392 struct instruction *instr,
2393 struct instruction_data *data __rte_unused)
2396 const char *mf_name;
2397 uint32_t mf_offset = 0;
2399 CHECK(action, EINVAL);
2400 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2402 a = action_find(p, tokens[1]);
2404 CHECK(!action_has_nbo_args(a), EINVAL);
2406 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2407 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2412 mf = metadata_field_parse(p, mf_name);
2415 mf_offset = mf->offset / 8;
2418 instr->type = INSTR_LEARNER_LEARN;
2419 instr->learn.action_id = a->id;
2420 instr->learn.mf_offset = mf_offset;
2426 instr_learn_exec(struct rte_swx_pipeline *p)
2428 struct thread *t = &p->threads[p->thread_id];
2429 struct instruction *ip = t->ip;
2431 __instr_learn_exec(p, t, ip);
2441 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2442 struct action *action,
2443 char **tokens __rte_unused,
2445 struct instruction *instr,
2446 struct instruction_data *data __rte_unused)
2448 CHECK(action, EINVAL);
2449 CHECK(n_tokens == 1, EINVAL);
2451 instr->type = INSTR_LEARNER_FORGET;
2457 instr_forget_exec(struct rte_swx_pipeline *p)
2459 struct thread *t = &p->threads[p->thread_id];
2460 struct instruction *ip = t->ip;
2462 __instr_forget_exec(p, t, ip);
2472 instr_extern_translate(struct rte_swx_pipeline *p,
2473 struct action *action __rte_unused,
2476 struct instruction *instr,
2477 struct instruction_data *data __rte_unused)
2479 char *token = tokens[1];
2481 CHECK(n_tokens == 2, EINVAL);
2483 if (token[0] == 'e') {
2484 struct extern_obj *obj;
2485 struct extern_type_member_func *func;
2487 func = extern_obj_member_func_parse(p, token, &obj);
2488 CHECK(func, EINVAL);
2490 instr->type = INSTR_EXTERN_OBJ;
2491 instr->ext_obj.ext_obj_id = obj->id;
2492 instr->ext_obj.func_id = func->id;
2497 if (token[0] == 'f') {
2498 struct extern_func *func;
2500 func = extern_func_parse(p, token);
2501 CHECK(func, EINVAL);
2503 instr->type = INSTR_EXTERN_FUNC;
2504 instr->ext_func.ext_func_id = func->id;
2513 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2515 struct thread *t = &p->threads[p->thread_id];
2516 struct instruction *ip = t->ip;
2519 /* Extern object member function execute. */
2520 done = __instr_extern_obj_exec(p, t, ip);
2523 thread_ip_inc_cond(t, done);
2524 thread_yield_cond(p, done ^ 1);
2528 instr_extern_func_exec(struct rte_swx_pipeline *p)
2530 struct thread *t = &p->threads[p->thread_id];
2531 struct instruction *ip = t->ip;
2534 /* Extern function execute. */
2535 done = __instr_extern_func_exec(p, t, ip);
2538 thread_ip_inc_cond(t, done);
2539 thread_yield_cond(p, done ^ 1);
2546 instr_mov_translate(struct rte_swx_pipeline *p,
2547 struct action *action,
2550 struct instruction *instr,
2551 struct instruction_data *data __rte_unused)
2553 char *dst = tokens[1], *src = tokens[2];
2554 struct field *fdst, *fsrc;
2556 uint32_t dst_struct_id = 0, src_struct_id = 0;
2558 CHECK(n_tokens == 3, EINVAL);
2560 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2561 CHECK(fdst, EINVAL);
2562 CHECK(!fdst->var_size, EINVAL);
2564 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2565 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2567 CHECK(!fsrc->var_size, EINVAL);
2569 instr->type = INSTR_MOV;
2570 if (dst[0] != 'h' && src[0] == 'h')
2571 instr->type = INSTR_MOV_MH;
2572 if (dst[0] == 'h' && src[0] != 'h')
2573 instr->type = INSTR_MOV_HM;
2574 if (dst[0] == 'h' && src[0] == 'h')
2575 instr->type = INSTR_MOV_HH;
2577 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2578 instr->mov.dst.n_bits = fdst->n_bits;
2579 instr->mov.dst.offset = fdst->offset / 8;
2580 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2581 instr->mov.src.n_bits = fsrc->n_bits;
2582 instr->mov.src.offset = fsrc->offset / 8;
2587 src_val = strtoull(src, &src, 0);
2588 CHECK(!src[0], EINVAL);
2591 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2593 instr->type = INSTR_MOV_I;
2594 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2595 instr->mov.dst.n_bits = fdst->n_bits;
2596 instr->mov.dst.offset = fdst->offset / 8;
2597 instr->mov.src_val = src_val;
2602 instr_mov_exec(struct rte_swx_pipeline *p)
2604 struct thread *t = &p->threads[p->thread_id];
2605 struct instruction *ip = t->ip;
2607 __instr_mov_exec(p, t, ip);
2614 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2616 struct thread *t = &p->threads[p->thread_id];
2617 struct instruction *ip = t->ip;
2619 __instr_mov_mh_exec(p, t, ip);
2626 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2628 struct thread *t = &p->threads[p->thread_id];
2629 struct instruction *ip = t->ip;
2631 __instr_mov_hm_exec(p, t, ip);
2638 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2640 struct thread *t = &p->threads[p->thread_id];
2641 struct instruction *ip = t->ip;
2643 __instr_mov_hh_exec(p, t, ip);
2650 instr_mov_i_exec(struct rte_swx_pipeline *p)
2652 struct thread *t = &p->threads[p->thread_id];
2653 struct instruction *ip = t->ip;
2655 __instr_mov_i_exec(p, t, ip);
2665 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2667 struct thread *t = &p->threads[p->thread_id];
2668 struct instruction *ip = t->ip;
2670 __instr_dma_ht_exec(p, t, ip);
2677 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2679 struct thread *t = &p->threads[p->thread_id];
2680 struct instruction *ip = t->ip;
2682 __instr_dma_ht2_exec(p, t, ip);
2689 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2691 struct thread *t = &p->threads[p->thread_id];
2692 struct instruction *ip = t->ip;
2694 __instr_dma_ht3_exec(p, t, ip);
2701 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2703 struct thread *t = &p->threads[p->thread_id];
2704 struct instruction *ip = t->ip;
2706 __instr_dma_ht4_exec(p, t, ip);
2713 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2715 struct thread *t = &p->threads[p->thread_id];
2716 struct instruction *ip = t->ip;
2718 __instr_dma_ht5_exec(p, t, ip);
2725 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2727 struct thread *t = &p->threads[p->thread_id];
2728 struct instruction *ip = t->ip;
2730 __instr_dma_ht6_exec(p, t, ip);
2737 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2739 struct thread *t = &p->threads[p->thread_id];
2740 struct instruction *ip = t->ip;
2742 __instr_dma_ht7_exec(p, t, ip);
2749 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2751 struct thread *t = &p->threads[p->thread_id];
2752 struct instruction *ip = t->ip;
2754 __instr_dma_ht8_exec(p, t, ip);
2764 instr_alu_add_translate(struct rte_swx_pipeline *p,
2765 struct action *action,
2768 struct instruction *instr,
2769 struct instruction_data *data __rte_unused)
2771 char *dst = tokens[1], *src = tokens[2];
2772 struct field *fdst, *fsrc;
2774 uint32_t dst_struct_id = 0, src_struct_id = 0;
2776 CHECK(n_tokens == 3, EINVAL);
2778 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2779 CHECK(fdst, EINVAL);
2780 CHECK(!fdst->var_size, EINVAL);
2782 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2783 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2785 CHECK(!fsrc->var_size, EINVAL);
2787 instr->type = INSTR_ALU_ADD;
2788 if (dst[0] == 'h' && src[0] != 'h')
2789 instr->type = INSTR_ALU_ADD_HM;
2790 if (dst[0] != 'h' && src[0] == 'h')
2791 instr->type = INSTR_ALU_ADD_MH;
2792 if (dst[0] == 'h' && src[0] == 'h')
2793 instr->type = INSTR_ALU_ADD_HH;
2795 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2796 instr->alu.dst.n_bits = fdst->n_bits;
2797 instr->alu.dst.offset = fdst->offset / 8;
2798 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2799 instr->alu.src.n_bits = fsrc->n_bits;
2800 instr->alu.src.offset = fsrc->offset / 8;
2804 /* ADD_MI, ADD_HI. */
2805 src_val = strtoull(src, &src, 0);
2806 CHECK(!src[0], EINVAL);
2808 instr->type = INSTR_ALU_ADD_MI;
2810 instr->type = INSTR_ALU_ADD_HI;
2812 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2813 instr->alu.dst.n_bits = fdst->n_bits;
2814 instr->alu.dst.offset = fdst->offset / 8;
2815 instr->alu.src_val = src_val;
2820 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2821 struct action *action,
2824 struct instruction *instr,
2825 struct instruction_data *data __rte_unused)
2827 char *dst = tokens[1], *src = tokens[2];
2828 struct field *fdst, *fsrc;
2830 uint32_t dst_struct_id = 0, src_struct_id = 0;
2832 CHECK(n_tokens == 3, EINVAL);
2834 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2835 CHECK(fdst, EINVAL);
2836 CHECK(!fdst->var_size, EINVAL);
2838 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2839 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2841 CHECK(!fsrc->var_size, EINVAL);
2843 instr->type = INSTR_ALU_SUB;
2844 if (dst[0] == 'h' && src[0] != 'h')
2845 instr->type = INSTR_ALU_SUB_HM;
2846 if (dst[0] != 'h' && src[0] == 'h')
2847 instr->type = INSTR_ALU_SUB_MH;
2848 if (dst[0] == 'h' && src[0] == 'h')
2849 instr->type = INSTR_ALU_SUB_HH;
2851 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2852 instr->alu.dst.n_bits = fdst->n_bits;
2853 instr->alu.dst.offset = fdst->offset / 8;
2854 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2855 instr->alu.src.n_bits = fsrc->n_bits;
2856 instr->alu.src.offset = fsrc->offset / 8;
2860 /* SUB_MI, SUB_HI. */
2861 src_val = strtoull(src, &src, 0);
2862 CHECK(!src[0], EINVAL);
2864 instr->type = INSTR_ALU_SUB_MI;
2866 instr->type = INSTR_ALU_SUB_HI;
2868 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2869 instr->alu.dst.n_bits = fdst->n_bits;
2870 instr->alu.dst.offset = fdst->offset / 8;
2871 instr->alu.src_val = src_val;
2876 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2877 struct action *action __rte_unused,
2880 struct instruction *instr,
2881 struct instruction_data *data __rte_unused)
2883 char *dst = tokens[1], *src = tokens[2];
2884 struct header *hdst, *hsrc;
2885 struct field *fdst, *fsrc;
2887 CHECK(n_tokens == 3, EINVAL);
2889 fdst = header_field_parse(p, dst, &hdst);
2890 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2891 CHECK(!fdst->var_size, EINVAL);
2894 fsrc = header_field_parse(p, src, &hsrc);
2896 CHECK(!fsrc->var_size, EINVAL);
2898 instr->type = INSTR_ALU_CKADD_FIELD;
2899 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2900 instr->alu.dst.n_bits = fdst->n_bits;
2901 instr->alu.dst.offset = fdst->offset / 8;
2902 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2903 instr->alu.src.n_bits = fsrc->n_bits;
2904 instr->alu.src.offset = fsrc->offset / 8;
2908 /* CKADD_STRUCT, CKADD_STRUCT20. */
2909 hsrc = header_parse(p, src);
2910 CHECK(hsrc, EINVAL);
2911 CHECK(!hsrc->st->var_size, EINVAL);
2913 instr->type = INSTR_ALU_CKADD_STRUCT;
2914 if ((hsrc->st->n_bits / 8) == 20)
2915 instr->type = INSTR_ALU_CKADD_STRUCT20;
2917 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2918 instr->alu.dst.n_bits = fdst->n_bits;
2919 instr->alu.dst.offset = fdst->offset / 8;
2920 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2921 instr->alu.src.n_bits = hsrc->st->n_bits;
2922 instr->alu.src.offset = 0; /* Unused. */
2927 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2928 struct action *action __rte_unused,
2931 struct instruction *instr,
2932 struct instruction_data *data __rte_unused)
2934 char *dst = tokens[1], *src = tokens[2];
2935 struct header *hdst, *hsrc;
2936 struct field *fdst, *fsrc;
2938 CHECK(n_tokens == 3, EINVAL);
2940 fdst = header_field_parse(p, dst, &hdst);
2941 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2942 CHECK(!fdst->var_size, EINVAL);
2944 fsrc = header_field_parse(p, src, &hsrc);
2945 CHECK(fsrc, EINVAL);
2946 CHECK(!fsrc->var_size, EINVAL);
2948 instr->type = INSTR_ALU_CKSUB_FIELD;
2949 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2950 instr->alu.dst.n_bits = fdst->n_bits;
2951 instr->alu.dst.offset = fdst->offset / 8;
2952 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2953 instr->alu.src.n_bits = fsrc->n_bits;
2954 instr->alu.src.offset = fsrc->offset / 8;
2959 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2960 struct action *action,
2963 struct instruction *instr,
2964 struct instruction_data *data __rte_unused)
2966 char *dst = tokens[1], *src = tokens[2];
2967 struct field *fdst, *fsrc;
2969 uint32_t dst_struct_id = 0, src_struct_id = 0;
2971 CHECK(n_tokens == 3, EINVAL);
2973 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2974 CHECK(fdst, EINVAL);
2975 CHECK(!fdst->var_size, EINVAL);
2977 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2978 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2980 CHECK(!fsrc->var_size, EINVAL);
2982 instr->type = INSTR_ALU_SHL;
2983 if (dst[0] == 'h' && src[0] != 'h')
2984 instr->type = INSTR_ALU_SHL_HM;
2985 if (dst[0] != 'h' && src[0] == 'h')
2986 instr->type = INSTR_ALU_SHL_MH;
2987 if (dst[0] == 'h' && src[0] == 'h')
2988 instr->type = INSTR_ALU_SHL_HH;
2990 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2991 instr->alu.dst.n_bits = fdst->n_bits;
2992 instr->alu.dst.offset = fdst->offset / 8;
2993 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2994 instr->alu.src.n_bits = fsrc->n_bits;
2995 instr->alu.src.offset = fsrc->offset / 8;
2999 /* SHL_MI, SHL_HI. */
3000 src_val = strtoull(src, &src, 0);
3001 CHECK(!src[0], EINVAL);
3003 instr->type = INSTR_ALU_SHL_MI;
3005 instr->type = INSTR_ALU_SHL_HI;
3007 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3008 instr->alu.dst.n_bits = fdst->n_bits;
3009 instr->alu.dst.offset = fdst->offset / 8;
3010 instr->alu.src_val = src_val;
3015 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3016 struct action *action,
3019 struct instruction *instr,
3020 struct instruction_data *data __rte_unused)
3022 char *dst = tokens[1], *src = tokens[2];
3023 struct field *fdst, *fsrc;
3025 uint32_t dst_struct_id = 0, src_struct_id = 0;
3027 CHECK(n_tokens == 3, EINVAL);
3029 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3030 CHECK(fdst, EINVAL);
3031 CHECK(!fdst->var_size, EINVAL);
3033 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3034 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3036 CHECK(!fsrc->var_size, EINVAL);
3038 instr->type = INSTR_ALU_SHR;
3039 if (dst[0] == 'h' && src[0] != 'h')
3040 instr->type = INSTR_ALU_SHR_HM;
3041 if (dst[0] != 'h' && src[0] == 'h')
3042 instr->type = INSTR_ALU_SHR_MH;
3043 if (dst[0] == 'h' && src[0] == 'h')
3044 instr->type = INSTR_ALU_SHR_HH;
3046 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3047 instr->alu.dst.n_bits = fdst->n_bits;
3048 instr->alu.dst.offset = fdst->offset / 8;
3049 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3050 instr->alu.src.n_bits = fsrc->n_bits;
3051 instr->alu.src.offset = fsrc->offset / 8;
3055 /* SHR_MI, SHR_HI. */
3056 src_val = strtoull(src, &src, 0);
3057 CHECK(!src[0], EINVAL);
3059 instr->type = INSTR_ALU_SHR_MI;
3061 instr->type = INSTR_ALU_SHR_HI;
3063 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3064 instr->alu.dst.n_bits = fdst->n_bits;
3065 instr->alu.dst.offset = fdst->offset / 8;
3066 instr->alu.src_val = src_val;
3071 instr_alu_and_translate(struct rte_swx_pipeline *p,
3072 struct action *action,
3075 struct instruction *instr,
3076 struct instruction_data *data __rte_unused)
3078 char *dst = tokens[1], *src = tokens[2];
3079 struct field *fdst, *fsrc;
3081 uint32_t dst_struct_id = 0, src_struct_id = 0;
3083 CHECK(n_tokens == 3, EINVAL);
3085 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3086 CHECK(fdst, EINVAL);
3087 CHECK(!fdst->var_size, EINVAL);
3089 /* AND, AND_MH, AND_HM, AND_HH. */
3090 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3092 CHECK(!fsrc->var_size, EINVAL);
3094 instr->type = INSTR_ALU_AND;
3095 if (dst[0] != 'h' && src[0] == 'h')
3096 instr->type = INSTR_ALU_AND_MH;
3097 if (dst[0] == 'h' && src[0] != 'h')
3098 instr->type = INSTR_ALU_AND_HM;
3099 if (dst[0] == 'h' && src[0] == 'h')
3100 instr->type = INSTR_ALU_AND_HH;
3102 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3103 instr->alu.dst.n_bits = fdst->n_bits;
3104 instr->alu.dst.offset = fdst->offset / 8;
3105 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3106 instr->alu.src.n_bits = fsrc->n_bits;
3107 instr->alu.src.offset = fsrc->offset / 8;
3112 src_val = strtoull(src, &src, 0);
3113 CHECK(!src[0], EINVAL);
3116 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3118 instr->type = INSTR_ALU_AND_I;
3119 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3120 instr->alu.dst.n_bits = fdst->n_bits;
3121 instr->alu.dst.offset = fdst->offset / 8;
3122 instr->alu.src_val = src_val;
3127 instr_alu_or_translate(struct rte_swx_pipeline *p,
3128 struct action *action,
3131 struct instruction *instr,
3132 struct instruction_data *data __rte_unused)
3134 char *dst = tokens[1], *src = tokens[2];
3135 struct field *fdst, *fsrc;
3137 uint32_t dst_struct_id = 0, src_struct_id = 0;
3139 CHECK(n_tokens == 3, EINVAL);
3141 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3142 CHECK(fdst, EINVAL);
3143 CHECK(!fdst->var_size, EINVAL);
3145 /* OR, OR_MH, OR_HM, OR_HH. */
3146 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3148 CHECK(!fsrc->var_size, EINVAL);
3150 instr->type = INSTR_ALU_OR;
3151 if (dst[0] != 'h' && src[0] == 'h')
3152 instr->type = INSTR_ALU_OR_MH;
3153 if (dst[0] == 'h' && src[0] != 'h')
3154 instr->type = INSTR_ALU_OR_HM;
3155 if (dst[0] == 'h' && src[0] == 'h')
3156 instr->type = INSTR_ALU_OR_HH;
3158 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3159 instr->alu.dst.n_bits = fdst->n_bits;
3160 instr->alu.dst.offset = fdst->offset / 8;
3161 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3162 instr->alu.src.n_bits = fsrc->n_bits;
3163 instr->alu.src.offset = fsrc->offset / 8;
3168 src_val = strtoull(src, &src, 0);
3169 CHECK(!src[0], EINVAL);
3172 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3174 instr->type = INSTR_ALU_OR_I;
3175 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3176 instr->alu.dst.n_bits = fdst->n_bits;
3177 instr->alu.dst.offset = fdst->offset / 8;
3178 instr->alu.src_val = src_val;
3183 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3184 struct action *action,
3187 struct instruction *instr,
3188 struct instruction_data *data __rte_unused)
3190 char *dst = tokens[1], *src = tokens[2];
3191 struct field *fdst, *fsrc;
3193 uint32_t dst_struct_id = 0, src_struct_id = 0;
3195 CHECK(n_tokens == 3, EINVAL);
3197 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3198 CHECK(fdst, EINVAL);
3199 CHECK(!fdst->var_size, EINVAL);
3201 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3202 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3204 CHECK(!fsrc->var_size, EINVAL);
3206 instr->type = INSTR_ALU_XOR;
3207 if (dst[0] != 'h' && src[0] == 'h')
3208 instr->type = INSTR_ALU_XOR_MH;
3209 if (dst[0] == 'h' && src[0] != 'h')
3210 instr->type = INSTR_ALU_XOR_HM;
3211 if (dst[0] == 'h' && src[0] == 'h')
3212 instr->type = INSTR_ALU_XOR_HH;
3214 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3215 instr->alu.dst.n_bits = fdst->n_bits;
3216 instr->alu.dst.offset = fdst->offset / 8;
3217 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3218 instr->alu.src.n_bits = fsrc->n_bits;
3219 instr->alu.src.offset = fsrc->offset / 8;
3224 src_val = strtoull(src, &src, 0);
3225 CHECK(!src[0], EINVAL);
3228 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3230 instr->type = INSTR_ALU_XOR_I;
3231 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3232 instr->alu.dst.n_bits = fdst->n_bits;
3233 instr->alu.dst.offset = fdst->offset / 8;
3234 instr->alu.src_val = src_val;
3239 instr_alu_add_exec(struct rte_swx_pipeline *p)
3241 struct thread *t = &p->threads[p->thread_id];
3242 struct instruction *ip = t->ip;
3245 __instr_alu_add_exec(p, t, ip);
3252 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3254 struct thread *t = &p->threads[p->thread_id];
3255 struct instruction *ip = t->ip;
3258 __instr_alu_add_mh_exec(p, t, ip);
3265 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3267 struct thread *t = &p->threads[p->thread_id];
3268 struct instruction *ip = t->ip;
3271 __instr_alu_add_hm_exec(p, t, ip);
3278 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3280 struct thread *t = &p->threads[p->thread_id];
3281 struct instruction *ip = t->ip;
3284 __instr_alu_add_hh_exec(p, t, ip);
3291 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3293 struct thread *t = &p->threads[p->thread_id];
3294 struct instruction *ip = t->ip;
3297 __instr_alu_add_mi_exec(p, t, ip);
3304 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3306 struct thread *t = &p->threads[p->thread_id];
3307 struct instruction *ip = t->ip;
3310 __instr_alu_add_hi_exec(p, t, ip);
3317 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3319 struct thread *t = &p->threads[p->thread_id];
3320 struct instruction *ip = t->ip;
3323 __instr_alu_sub_exec(p, t, ip);
3330 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3332 struct thread *t = &p->threads[p->thread_id];
3333 struct instruction *ip = t->ip;
3336 __instr_alu_sub_mh_exec(p, t, ip);
3343 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3345 struct thread *t = &p->threads[p->thread_id];
3346 struct instruction *ip = t->ip;
3349 __instr_alu_sub_hm_exec(p, t, ip);
3356 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3358 struct thread *t = &p->threads[p->thread_id];
3359 struct instruction *ip = t->ip;
3362 __instr_alu_sub_hh_exec(p, t, ip);
3369 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3371 struct thread *t = &p->threads[p->thread_id];
3372 struct instruction *ip = t->ip;
3375 __instr_alu_sub_mi_exec(p, t, ip);
3382 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3384 struct thread *t = &p->threads[p->thread_id];
3385 struct instruction *ip = t->ip;
3388 __instr_alu_sub_hi_exec(p, t, ip);
3395 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3397 struct thread *t = &p->threads[p->thread_id];
3398 struct instruction *ip = t->ip;
3401 __instr_alu_shl_exec(p, t, ip);
3408 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3410 struct thread *t = &p->threads[p->thread_id];
3411 struct instruction *ip = t->ip;
3414 __instr_alu_shl_mh_exec(p, t, ip);
3421 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3423 struct thread *t = &p->threads[p->thread_id];
3424 struct instruction *ip = t->ip;
3427 __instr_alu_shl_hm_exec(p, t, ip);
3434 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3436 struct thread *t = &p->threads[p->thread_id];
3437 struct instruction *ip = t->ip;
3440 __instr_alu_shl_hh_exec(p, t, ip);
3447 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3449 struct thread *t = &p->threads[p->thread_id];
3450 struct instruction *ip = t->ip;
3453 __instr_alu_shl_mi_exec(p, t, ip);
3460 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3462 struct thread *t = &p->threads[p->thread_id];
3463 struct instruction *ip = t->ip;
3466 __instr_alu_shl_hi_exec(p, t, ip);
3473 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3475 struct thread *t = &p->threads[p->thread_id];
3476 struct instruction *ip = t->ip;
3479 __instr_alu_shr_exec(p, t, ip);
3486 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3488 struct thread *t = &p->threads[p->thread_id];
3489 struct instruction *ip = t->ip;
3492 __instr_alu_shr_mh_exec(p, t, ip);
3499 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3501 struct thread *t = &p->threads[p->thread_id];
3502 struct instruction *ip = t->ip;
3505 __instr_alu_shr_hm_exec(p, t, ip);
3512 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3514 struct thread *t = &p->threads[p->thread_id];
3515 struct instruction *ip = t->ip;
3518 __instr_alu_shr_hh_exec(p, t, ip);
3525 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3527 struct thread *t = &p->threads[p->thread_id];
3528 struct instruction *ip = t->ip;
3531 __instr_alu_shr_mi_exec(p, t, ip);
3538 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3540 struct thread *t = &p->threads[p->thread_id];
3541 struct instruction *ip = t->ip;
3544 __instr_alu_shr_hi_exec(p, t, ip);
3551 instr_alu_and_exec(struct rte_swx_pipeline *p)
3553 struct thread *t = &p->threads[p->thread_id];
3554 struct instruction *ip = t->ip;
3557 __instr_alu_and_exec(p, t, ip);
3564 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3566 struct thread *t = &p->threads[p->thread_id];
3567 struct instruction *ip = t->ip;
3570 __instr_alu_and_mh_exec(p, t, ip);
3577 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3579 struct thread *t = &p->threads[p->thread_id];
3580 struct instruction *ip = t->ip;
3583 __instr_alu_and_hm_exec(p, t, ip);
3590 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3592 struct thread *t = &p->threads[p->thread_id];
3593 struct instruction *ip = t->ip;
3596 __instr_alu_and_hh_exec(p, t, ip);
3603 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3605 struct thread *t = &p->threads[p->thread_id];
3606 struct instruction *ip = t->ip;
3609 __instr_alu_and_i_exec(p, t, ip);
3616 instr_alu_or_exec(struct rte_swx_pipeline *p)
3618 struct thread *t = &p->threads[p->thread_id];
3619 struct instruction *ip = t->ip;
3622 __instr_alu_or_exec(p, t, ip);
3629 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3631 struct thread *t = &p->threads[p->thread_id];
3632 struct instruction *ip = t->ip;
3635 __instr_alu_or_mh_exec(p, t, ip);
3642 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3644 struct thread *t = &p->threads[p->thread_id];
3645 struct instruction *ip = t->ip;
3648 __instr_alu_or_hm_exec(p, t, ip);
3655 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3657 struct thread *t = &p->threads[p->thread_id];
3658 struct instruction *ip = t->ip;
3661 __instr_alu_or_hh_exec(p, t, ip);
3668 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3670 struct thread *t = &p->threads[p->thread_id];
3671 struct instruction *ip = t->ip;
3674 __instr_alu_or_i_exec(p, t, ip);
3681 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3683 struct thread *t = &p->threads[p->thread_id];
3684 struct instruction *ip = t->ip;
3687 __instr_alu_xor_exec(p, t, ip);
3694 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3696 struct thread *t = &p->threads[p->thread_id];
3697 struct instruction *ip = t->ip;
3700 __instr_alu_xor_mh_exec(p, t, ip);
3707 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3709 struct thread *t = &p->threads[p->thread_id];
3710 struct instruction *ip = t->ip;
3713 __instr_alu_xor_hm_exec(p, t, ip);
3720 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3722 struct thread *t = &p->threads[p->thread_id];
3723 struct instruction *ip = t->ip;
3726 __instr_alu_xor_hh_exec(p, t, ip);
3733 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3735 struct thread *t = &p->threads[p->thread_id];
3736 struct instruction *ip = t->ip;
3739 __instr_alu_xor_i_exec(p, t, ip);
3746 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3748 struct thread *t = &p->threads[p->thread_id];
3749 struct instruction *ip = t->ip;
3752 __instr_alu_ckadd_field_exec(p, t, ip);
3759 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3761 struct thread *t = &p->threads[p->thread_id];
3762 struct instruction *ip = t->ip;
3765 __instr_alu_cksub_field_exec(p, t, ip);
3772 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3774 struct thread *t = &p->threads[p->thread_id];
3775 struct instruction *ip = t->ip;
3778 __instr_alu_ckadd_struct20_exec(p, t, ip);
3785 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3787 struct thread *t = &p->threads[p->thread_id];
3788 struct instruction *ip = t->ip;
3791 __instr_alu_ckadd_struct_exec(p, t, ip);
3800 static struct regarray *
3801 regarray_find(struct rte_swx_pipeline *p, const char *name);
3804 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3805 struct action *action,
3808 struct instruction *instr,
3809 struct instruction_data *data __rte_unused)
3811 char *regarray = tokens[1], *idx = tokens[2];
3814 uint32_t idx_struct_id, idx_val;
3816 CHECK(n_tokens == 3, EINVAL);
3818 r = regarray_find(p, regarray);
3821 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3822 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3824 CHECK(!fidx->var_size, EINVAL);
3826 instr->type = INSTR_REGPREFETCH_RM;
3828 instr->type = INSTR_REGPREFETCH_RH;
3830 instr->regarray.regarray_id = r->id;
3831 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3832 instr->regarray.idx.n_bits = fidx->n_bits;
3833 instr->regarray.idx.offset = fidx->offset / 8;
3834 instr->regarray.dstsrc_val = 0; /* Unused. */
3838 /* REGPREFETCH_RI. */
3839 idx_val = strtoul(idx, &idx, 0);
3840 CHECK(!idx[0], EINVAL);
3842 instr->type = INSTR_REGPREFETCH_RI;
3843 instr->regarray.regarray_id = r->id;
3844 instr->regarray.idx_val = idx_val;
3845 instr->regarray.dstsrc_val = 0; /* Unused. */
3850 instr_regrd_translate(struct rte_swx_pipeline *p,
3851 struct action *action,
3854 struct instruction *instr,
3855 struct instruction_data *data __rte_unused)
3857 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3859 struct field *fdst, *fidx;
3860 uint32_t dst_struct_id, idx_struct_id, idx_val;
3862 CHECK(n_tokens == 4, EINVAL);
3864 r = regarray_find(p, regarray);
3867 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3868 CHECK(fdst, EINVAL);
3869 CHECK(!fdst->var_size, EINVAL);
3871 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3872 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3874 CHECK(!fidx->var_size, EINVAL);
3876 instr->type = INSTR_REGRD_MRM;
3877 if (dst[0] == 'h' && idx[0] != 'h')
3878 instr->type = INSTR_REGRD_HRM;
3879 if (dst[0] != 'h' && idx[0] == 'h')
3880 instr->type = INSTR_REGRD_MRH;
3881 if (dst[0] == 'h' && idx[0] == 'h')
3882 instr->type = INSTR_REGRD_HRH;
3884 instr->regarray.regarray_id = r->id;
3885 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3886 instr->regarray.idx.n_bits = fidx->n_bits;
3887 instr->regarray.idx.offset = fidx->offset / 8;
3888 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3889 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3890 instr->regarray.dstsrc.offset = fdst->offset / 8;
3894 /* REGRD_MRI, REGRD_HRI. */
3895 idx_val = strtoul(idx, &idx, 0);
3896 CHECK(!idx[0], EINVAL);
3898 instr->type = INSTR_REGRD_MRI;
3900 instr->type = INSTR_REGRD_HRI;
3902 instr->regarray.regarray_id = r->id;
3903 instr->regarray.idx_val = idx_val;
3904 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3905 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3906 instr->regarray.dstsrc.offset = fdst->offset / 8;
3911 instr_regwr_translate(struct rte_swx_pipeline *p,
3912 struct action *action,
3915 struct instruction *instr,
3916 struct instruction_data *data __rte_unused)
3918 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3920 struct field *fidx, *fsrc;
3922 uint32_t idx_struct_id, idx_val, src_struct_id;
3924 CHECK(n_tokens == 4, EINVAL);
3926 r = regarray_find(p, regarray);
3929 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3930 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3931 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3933 CHECK(!fidx->var_size, EINVAL);
3934 CHECK(!fsrc->var_size, EINVAL);
3936 instr->type = INSTR_REGWR_RMM;
3937 if (idx[0] == 'h' && src[0] != 'h')
3938 instr->type = INSTR_REGWR_RHM;
3939 if (idx[0] != 'h' && src[0] == 'h')
3940 instr->type = INSTR_REGWR_RMH;
3941 if (idx[0] == 'h' && src[0] == 'h')
3942 instr->type = INSTR_REGWR_RHH;
3944 instr->regarray.regarray_id = r->id;
3945 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3946 instr->regarray.idx.n_bits = fidx->n_bits;
3947 instr->regarray.idx.offset = fidx->offset / 8;
3948 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3949 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3950 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3954 /* REGWR_RHI, REGWR_RMI. */
3955 if (fidx && !fsrc) {
3956 CHECK(!fidx->var_size, EINVAL);
3958 src_val = strtoull(src, &src, 0);
3959 CHECK(!src[0], EINVAL);
3961 instr->type = INSTR_REGWR_RMI;
3963 instr->type = INSTR_REGWR_RHI;
3965 instr->regarray.regarray_id = r->id;
3966 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3967 instr->regarray.idx.n_bits = fidx->n_bits;
3968 instr->regarray.idx.offset = fidx->offset / 8;
3969 instr->regarray.dstsrc_val = src_val;
3973 /* REGWR_RIH, REGWR_RIM. */
3974 if (!fidx && fsrc) {
3975 idx_val = strtoul(idx, &idx, 0);
3976 CHECK(!idx[0], EINVAL);
3978 CHECK(!fsrc->var_size, EINVAL);
3980 instr->type = INSTR_REGWR_RIM;
3982 instr->type = INSTR_REGWR_RIH;
3984 instr->regarray.regarray_id = r->id;
3985 instr->regarray.idx_val = idx_val;
3986 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3987 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3988 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3993 src_val = strtoull(src, &src, 0);
3994 CHECK(!src[0], EINVAL);
3996 idx_val = strtoul(idx, &idx, 0);
3997 CHECK(!idx[0], EINVAL);
3999 instr->type = INSTR_REGWR_RII;
4000 instr->regarray.idx_val = idx_val;
4001 instr->regarray.dstsrc_val = src_val;
4007 instr_regadd_translate(struct rte_swx_pipeline *p,
4008 struct action *action,
4011 struct instruction *instr,
4012 struct instruction_data *data __rte_unused)
4014 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4016 struct field *fidx, *fsrc;
4018 uint32_t idx_struct_id, idx_val, src_struct_id;
4020 CHECK(n_tokens == 4, EINVAL);
4022 r = regarray_find(p, regarray);
4025 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4026 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4027 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4029 CHECK(!fidx->var_size, EINVAL);
4030 CHECK(!fsrc->var_size, EINVAL);
4032 instr->type = INSTR_REGADD_RMM;
4033 if (idx[0] == 'h' && src[0] != 'h')
4034 instr->type = INSTR_REGADD_RHM;
4035 if (idx[0] != 'h' && src[0] == 'h')
4036 instr->type = INSTR_REGADD_RMH;
4037 if (idx[0] == 'h' && src[0] == 'h')
4038 instr->type = INSTR_REGADD_RHH;
4040 instr->regarray.regarray_id = r->id;
4041 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4042 instr->regarray.idx.n_bits = fidx->n_bits;
4043 instr->regarray.idx.offset = fidx->offset / 8;
4044 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4045 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4046 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4050 /* REGADD_RHI, REGADD_RMI. */
4051 if (fidx && !fsrc) {
4052 CHECK(!fidx->var_size, EINVAL);
4054 src_val = strtoull(src, &src, 0);
4055 CHECK(!src[0], EINVAL);
4057 instr->type = INSTR_REGADD_RMI;
4059 instr->type = INSTR_REGADD_RHI;
4061 instr->regarray.regarray_id = r->id;
4062 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4063 instr->regarray.idx.n_bits = fidx->n_bits;
4064 instr->regarray.idx.offset = fidx->offset / 8;
4065 instr->regarray.dstsrc_val = src_val;
4069 /* REGADD_RIH, REGADD_RIM. */
4070 if (!fidx && fsrc) {
4071 idx_val = strtoul(idx, &idx, 0);
4072 CHECK(!idx[0], EINVAL);
4074 CHECK(!fsrc->var_size, EINVAL);
4076 instr->type = INSTR_REGADD_RIM;
4078 instr->type = INSTR_REGADD_RIH;
4080 instr->regarray.regarray_id = r->id;
4081 instr->regarray.idx_val = idx_val;
4082 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4083 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4084 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4089 src_val = strtoull(src, &src, 0);
4090 CHECK(!src[0], EINVAL);
4092 idx_val = strtoul(idx, &idx, 0);
4093 CHECK(!idx[0], EINVAL);
4095 instr->type = INSTR_REGADD_RII;
4096 instr->regarray.idx_val = idx_val;
4097 instr->regarray.dstsrc_val = src_val;
4102 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4104 struct thread *t = &p->threads[p->thread_id];
4105 struct instruction *ip = t->ip;
4108 __instr_regprefetch_rh_exec(p, t, ip);
4115 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4117 struct thread *t = &p->threads[p->thread_id];
4118 struct instruction *ip = t->ip;
4121 __instr_regprefetch_rm_exec(p, t, ip);
4128 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4130 struct thread *t = &p->threads[p->thread_id];
4131 struct instruction *ip = t->ip;
4134 __instr_regprefetch_ri_exec(p, t, ip);
4141 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4143 struct thread *t = &p->threads[p->thread_id];
4144 struct instruction *ip = t->ip;
4147 __instr_regrd_hrh_exec(p, t, ip);
4154 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4156 struct thread *t = &p->threads[p->thread_id];
4157 struct instruction *ip = t->ip;
4160 __instr_regrd_hrm_exec(p, t, ip);
4167 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4169 struct thread *t = &p->threads[p->thread_id];
4170 struct instruction *ip = t->ip;
4173 __instr_regrd_mrh_exec(p, t, ip);
4180 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4182 struct thread *t = &p->threads[p->thread_id];
4183 struct instruction *ip = t->ip;
4186 __instr_regrd_mrm_exec(p, t, ip);
4193 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4195 struct thread *t = &p->threads[p->thread_id];
4196 struct instruction *ip = t->ip;
4199 __instr_regrd_hri_exec(p, t, ip);
4206 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4208 struct thread *t = &p->threads[p->thread_id];
4209 struct instruction *ip = t->ip;
4212 __instr_regrd_mri_exec(p, t, ip);
4219 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4221 struct thread *t = &p->threads[p->thread_id];
4222 struct instruction *ip = t->ip;
4225 __instr_regwr_rhh_exec(p, t, ip);
4232 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4234 struct thread *t = &p->threads[p->thread_id];
4235 struct instruction *ip = t->ip;
4238 __instr_regwr_rhm_exec(p, t, ip);
4245 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4247 struct thread *t = &p->threads[p->thread_id];
4248 struct instruction *ip = t->ip;
4251 __instr_regwr_rmh_exec(p, t, ip);
4258 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4260 struct thread *t = &p->threads[p->thread_id];
4261 struct instruction *ip = t->ip;
4264 __instr_regwr_rmm_exec(p, t, ip);
4271 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4273 struct thread *t = &p->threads[p->thread_id];
4274 struct instruction *ip = t->ip;
4277 __instr_regwr_rhi_exec(p, t, ip);
4284 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4286 struct thread *t = &p->threads[p->thread_id];
4287 struct instruction *ip = t->ip;
4290 __instr_regwr_rmi_exec(p, t, ip);
4297 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4299 struct thread *t = &p->threads[p->thread_id];
4300 struct instruction *ip = t->ip;
4303 __instr_regwr_rih_exec(p, t, ip);
4310 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4312 struct thread *t = &p->threads[p->thread_id];
4313 struct instruction *ip = t->ip;
4316 __instr_regwr_rim_exec(p, t, ip);
4323 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4325 struct thread *t = &p->threads[p->thread_id];
4326 struct instruction *ip = t->ip;
4329 __instr_regwr_rii_exec(p, t, ip);
4336 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4338 struct thread *t = &p->threads[p->thread_id];
4339 struct instruction *ip = t->ip;
4342 __instr_regadd_rhh_exec(p, t, ip);
4349 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4351 struct thread *t = &p->threads[p->thread_id];
4352 struct instruction *ip = t->ip;
4355 __instr_regadd_rhm_exec(p, t, ip);
4362 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4364 struct thread *t = &p->threads[p->thread_id];
4365 struct instruction *ip = t->ip;
4368 __instr_regadd_rmh_exec(p, t, ip);
4375 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4377 struct thread *t = &p->threads[p->thread_id];
4378 struct instruction *ip = t->ip;
4381 __instr_regadd_rmm_exec(p, t, ip);
4388 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4390 struct thread *t = &p->threads[p->thread_id];
4391 struct instruction *ip = t->ip;
4394 __instr_regadd_rhi_exec(p, t, ip);
4401 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4403 struct thread *t = &p->threads[p->thread_id];
4404 struct instruction *ip = t->ip;
4407 __instr_regadd_rmi_exec(p, t, ip);
4414 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4416 struct thread *t = &p->threads[p->thread_id];
4417 struct instruction *ip = t->ip;
4420 __instr_regadd_rih_exec(p, t, ip);
4427 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4429 struct thread *t = &p->threads[p->thread_id];
4430 struct instruction *ip = t->ip;
4433 __instr_regadd_rim_exec(p, t, ip);
4440 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4442 struct thread *t = &p->threads[p->thread_id];
4443 struct instruction *ip = t->ip;
4446 __instr_regadd_rii_exec(p, t, ip);
4455 static struct metarray *
4456 metarray_find(struct rte_swx_pipeline *p, const char *name);
4459 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4460 struct action *action,
4463 struct instruction *instr,
4464 struct instruction_data *data __rte_unused)
4466 char *metarray = tokens[1], *idx = tokens[2];
4469 uint32_t idx_struct_id, idx_val;
4471 CHECK(n_tokens == 3, EINVAL);
4473 m = metarray_find(p, metarray);
4476 /* METPREFETCH_H, METPREFETCH_M. */
4477 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4479 CHECK(!fidx->var_size, EINVAL);
4481 instr->type = INSTR_METPREFETCH_M;
4483 instr->type = INSTR_METPREFETCH_H;
4485 instr->meter.metarray_id = m->id;
4486 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4487 instr->meter.idx.n_bits = fidx->n_bits;
4488 instr->meter.idx.offset = fidx->offset / 8;
4492 /* METPREFETCH_I. */
4493 idx_val = strtoul(idx, &idx, 0);
4494 CHECK(!idx[0], EINVAL);
4496 instr->type = INSTR_METPREFETCH_I;
4497 instr->meter.metarray_id = m->id;
4498 instr->meter.idx_val = idx_val;
4503 instr_meter_translate(struct rte_swx_pipeline *p,
4504 struct action *action,
4507 struct instruction *instr,
4508 struct instruction_data *data __rte_unused)
4510 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4511 char *color_in = tokens[4], *color_out = tokens[5];
4513 struct field *fidx, *flength, *fcin, *fcout;
4514 uint32_t idx_struct_id, length_struct_id;
4515 uint32_t color_in_struct_id, color_out_struct_id;
4517 CHECK(n_tokens == 6, EINVAL);
4519 m = metarray_find(p, metarray);
4522 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4524 flength = struct_field_parse(p, action, length, &length_struct_id);
4525 CHECK(flength, EINVAL);
4526 CHECK(!flength->var_size, EINVAL);
4528 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4530 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4531 CHECK(fcout, EINVAL);
4532 CHECK(!fcout->var_size, EINVAL);
4534 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4536 CHECK(!fidx->var_size, EINVAL);
4537 CHECK(!fcin->var_size, EINVAL);
4539 instr->type = INSTR_METER_MMM;
4540 if (idx[0] == 'h' && length[0] == 'h')
4541 instr->type = INSTR_METER_HHM;
4542 if (idx[0] == 'h' && length[0] != 'h')
4543 instr->type = INSTR_METER_HMM;
4544 if (idx[0] != 'h' && length[0] == 'h')
4545 instr->type = INSTR_METER_MHM;
4547 instr->meter.metarray_id = m->id;
4549 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4550 instr->meter.idx.n_bits = fidx->n_bits;
4551 instr->meter.idx.offset = fidx->offset / 8;
4553 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4554 instr->meter.length.n_bits = flength->n_bits;
4555 instr->meter.length.offset = flength->offset / 8;
4557 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4558 instr->meter.color_in.n_bits = fcin->n_bits;
4559 instr->meter.color_in.offset = fcin->offset / 8;
4561 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4562 instr->meter.color_out.n_bits = fcout->n_bits;
4563 instr->meter.color_out.offset = fcout->offset / 8;
4566 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4567 if (fidx && !fcin) {
4568 uint32_t color_in_val;
4570 CHECK(!fidx->var_size, EINVAL);
4572 color_in_val = strtoul(color_in, &color_in, 0);
4573 CHECK(!color_in[0], EINVAL);
4575 instr->type = INSTR_METER_MMI;
4576 if (idx[0] == 'h' && length[0] == 'h')
4577 instr->type = INSTR_METER_HHI;
4578 if (idx[0] == 'h' && length[0] != 'h')
4579 instr->type = INSTR_METER_HMI;
4580 if (idx[0] != 'h' && length[0] == 'h')
4581 instr->type = INSTR_METER_MHI;
4583 instr->meter.metarray_id = m->id;
4585 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4586 instr->meter.idx.n_bits = fidx->n_bits;
4587 instr->meter.idx.offset = fidx->offset / 8;
4589 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4590 instr->meter.length.n_bits = flength->n_bits;
4591 instr->meter.length.offset = flength->offset / 8;
4593 instr->meter.color_in_val = color_in_val;
4595 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4596 instr->meter.color_out.n_bits = fcout->n_bits;
4597 instr->meter.color_out.offset = fcout->offset / 8;
4600 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4601 if (!fidx && fcin) {
4604 idx_val = strtoul(idx, &idx, 0);
4605 CHECK(!idx[0], EINVAL);
4607 CHECK(!fcin->var_size, EINVAL);
4609 instr->type = INSTR_METER_IMM;
4610 if (length[0] == 'h')
4611 instr->type = INSTR_METER_IHM;
4613 instr->meter.metarray_id = m->id;
4615 instr->meter.idx_val = idx_val;
4617 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4618 instr->meter.length.n_bits = flength->n_bits;
4619 instr->meter.length.offset = flength->offset / 8;
4621 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4622 instr->meter.color_in.n_bits = fcin->n_bits;
4623 instr->meter.color_in.offset = fcin->offset / 8;
4625 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4626 instr->meter.color_out.n_bits = fcout->n_bits;
4627 instr->meter.color_out.offset = fcout->offset / 8;
4630 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4631 if (!fidx && !fcin) {
4632 uint32_t idx_val, color_in_val;
4634 idx_val = strtoul(idx, &idx, 0);
4635 CHECK(!idx[0], EINVAL);
4637 color_in_val = strtoul(color_in, &color_in, 0);
4638 CHECK(!color_in[0], EINVAL);
4640 instr->type = INSTR_METER_IMI;
4641 if (length[0] == 'h')
4642 instr->type = INSTR_METER_IHI;
4644 instr->meter.metarray_id = m->id;
4646 instr->meter.idx_val = idx_val;
4648 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4649 instr->meter.length.n_bits = flength->n_bits;
4650 instr->meter.length.offset = flength->offset / 8;
4652 instr->meter.color_in_val = color_in_val;
4654 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4655 instr->meter.color_out.n_bits = fcout->n_bits;
4656 instr->meter.color_out.offset = fcout->offset / 8;
4663 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4665 struct thread *t = &p->threads[p->thread_id];
4666 struct instruction *ip = t->ip;
4669 __instr_metprefetch_h_exec(p, t, ip);
4676 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4678 struct thread *t = &p->threads[p->thread_id];
4679 struct instruction *ip = t->ip;
4682 __instr_metprefetch_m_exec(p, t, ip);
4689 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4691 struct thread *t = &p->threads[p->thread_id];
4692 struct instruction *ip = t->ip;
4695 __instr_metprefetch_i_exec(p, t, ip);
4702 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4704 struct thread *t = &p->threads[p->thread_id];
4705 struct instruction *ip = t->ip;
4708 __instr_meter_hhm_exec(p, t, ip);
4715 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4717 struct thread *t = &p->threads[p->thread_id];
4718 struct instruction *ip = t->ip;
4721 __instr_meter_hhi_exec(p, t, ip);
4728 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4730 struct thread *t = &p->threads[p->thread_id];
4731 struct instruction *ip = t->ip;
4734 __instr_meter_hmm_exec(p, t, ip);
4741 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4743 struct thread *t = &p->threads[p->thread_id];
4744 struct instruction *ip = t->ip;
4747 __instr_meter_hmi_exec(p, t, ip);
4754 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4756 struct thread *t = &p->threads[p->thread_id];
4757 struct instruction *ip = t->ip;
4760 __instr_meter_mhm_exec(p, t, ip);
4767 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4769 struct thread *t = &p->threads[p->thread_id];
4770 struct instruction *ip = t->ip;
4773 __instr_meter_mhi_exec(p, t, ip);
4780 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4782 struct thread *t = &p->threads[p->thread_id];
4783 struct instruction *ip = t->ip;
4786 __instr_meter_mmm_exec(p, t, ip);
4793 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4795 struct thread *t = &p->threads[p->thread_id];
4796 struct instruction *ip = t->ip;
4799 __instr_meter_mmi_exec(p, t, ip);
4806 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4808 struct thread *t = &p->threads[p->thread_id];
4809 struct instruction *ip = t->ip;
4812 __instr_meter_ihm_exec(p, t, ip);
4819 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4821 struct thread *t = &p->threads[p->thread_id];
4822 struct instruction *ip = t->ip;
4825 __instr_meter_ihi_exec(p, t, ip);
4832 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4834 struct thread *t = &p->threads[p->thread_id];
4835 struct instruction *ip = t->ip;
4838 __instr_meter_imm_exec(p, t, ip);
4845 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4847 struct thread *t = &p->threads[p->thread_id];
4848 struct instruction *ip = t->ip;
4851 __instr_meter_imi_exec(p, t, ip);
4861 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4862 struct action *action __rte_unused,
4865 struct instruction *instr,
4866 struct instruction_data *data)
4868 CHECK(n_tokens == 2, EINVAL);
4870 strcpy(data->jmp_label, tokens[1]);
4872 instr->type = INSTR_JMP;
4873 instr->jmp.ip = NULL; /* Resolved later. */
4878 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4879 struct action *action __rte_unused,
4882 struct instruction *instr,
4883 struct instruction_data *data)
4887 CHECK(n_tokens == 3, EINVAL);
4889 strcpy(data->jmp_label, tokens[1]);
4891 h = header_parse(p, tokens[2]);
4894 instr->type = INSTR_JMP_VALID;
4895 instr->jmp.ip = NULL; /* Resolved later. */
4896 instr->jmp.header_id = h->id;
4901 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4902 struct action *action __rte_unused,
4905 struct instruction *instr,
4906 struct instruction_data *data)
4910 CHECK(n_tokens == 3, EINVAL);
4912 strcpy(data->jmp_label, tokens[1]);
4914 h = header_parse(p, tokens[2]);
4917 instr->type = INSTR_JMP_INVALID;
4918 instr->jmp.ip = NULL; /* Resolved later. */
4919 instr->jmp.header_id = h->id;
4924 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4925 struct action *action,
4928 struct instruction *instr,
4929 struct instruction_data *data)
4931 CHECK(!action, EINVAL);
4932 CHECK(n_tokens == 2, EINVAL);
4934 strcpy(data->jmp_label, tokens[1]);
4936 instr->type = INSTR_JMP_HIT;
4937 instr->jmp.ip = NULL; /* Resolved later. */
4942 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4943 struct action *action,
4946 struct instruction *instr,
4947 struct instruction_data *data)
4949 CHECK(!action, EINVAL);
4950 CHECK(n_tokens == 2, EINVAL);
4952 strcpy(data->jmp_label, tokens[1]);
4954 instr->type = INSTR_JMP_MISS;
4955 instr->jmp.ip = NULL; /* Resolved later. */
4960 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4961 struct action *action,
4964 struct instruction *instr,
4965 struct instruction_data *data)
4969 CHECK(!action, EINVAL);
4970 CHECK(n_tokens == 3, EINVAL);
4972 strcpy(data->jmp_label, tokens[1]);
4974 a = action_find(p, tokens[2]);
4977 instr->type = INSTR_JMP_ACTION_HIT;
4978 instr->jmp.ip = NULL; /* Resolved later. */
4979 instr->jmp.action_id = a->id;
4984 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4985 struct action *action,
4988 struct instruction *instr,
4989 struct instruction_data *data)
4993 CHECK(!action, EINVAL);
4994 CHECK(n_tokens == 3, EINVAL);
4996 strcpy(data->jmp_label, tokens[1]);
4998 a = action_find(p, tokens[2]);
5001 instr->type = INSTR_JMP_ACTION_MISS;
5002 instr->jmp.ip = NULL; /* Resolved later. */
5003 instr->jmp.action_id = a->id;
5008 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5009 struct action *action,
5012 struct instruction *instr,
5013 struct instruction_data *data)
5015 char *a = tokens[2], *b = tokens[3];
5016 struct field *fa, *fb;
5018 uint32_t a_struct_id, b_struct_id;
5020 CHECK(n_tokens == 4, EINVAL);
5022 strcpy(data->jmp_label, tokens[1]);
5024 fa = struct_field_parse(p, action, a, &a_struct_id);
5026 CHECK(!fa->var_size, EINVAL);
5028 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5029 fb = struct_field_parse(p, action, b, &b_struct_id);
5031 CHECK(!fb->var_size, EINVAL);
5033 instr->type = INSTR_JMP_EQ;
5034 if (a[0] != 'h' && b[0] == 'h')
5035 instr->type = INSTR_JMP_EQ_MH;
5036 if (a[0] == 'h' && b[0] != 'h')
5037 instr->type = INSTR_JMP_EQ_HM;
5038 if (a[0] == 'h' && b[0] == 'h')
5039 instr->type = INSTR_JMP_EQ_HH;
5040 instr->jmp.ip = NULL; /* Resolved later. */
5042 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5043 instr->jmp.a.n_bits = fa->n_bits;
5044 instr->jmp.a.offset = fa->offset / 8;
5045 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5046 instr->jmp.b.n_bits = fb->n_bits;
5047 instr->jmp.b.offset = fb->offset / 8;
5052 b_val = strtoull(b, &b, 0);
5053 CHECK(!b[0], EINVAL);
5056 b_val = hton64(b_val) >> (64 - fa->n_bits);
5058 instr->type = INSTR_JMP_EQ_I;
5059 instr->jmp.ip = NULL; /* Resolved later. */
5060 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5061 instr->jmp.a.n_bits = fa->n_bits;
5062 instr->jmp.a.offset = fa->offset / 8;
5063 instr->jmp.b_val = b_val;
5068 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5069 struct action *action,
5072 struct instruction *instr,
5073 struct instruction_data *data)
5075 char *a = tokens[2], *b = tokens[3];
5076 struct field *fa, *fb;
5078 uint32_t a_struct_id, b_struct_id;
5080 CHECK(n_tokens == 4, EINVAL);
5082 strcpy(data->jmp_label, tokens[1]);
5084 fa = struct_field_parse(p, action, a, &a_struct_id);
5086 CHECK(!fa->var_size, EINVAL);
5088 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5089 fb = struct_field_parse(p, action, b, &b_struct_id);
5091 CHECK(!fb->var_size, EINVAL);
5093 instr->type = INSTR_JMP_NEQ;
5094 if (a[0] != 'h' && b[0] == 'h')
5095 instr->type = INSTR_JMP_NEQ_MH;
5096 if (a[0] == 'h' && b[0] != 'h')
5097 instr->type = INSTR_JMP_NEQ_HM;
5098 if (a[0] == 'h' && b[0] == 'h')
5099 instr->type = INSTR_JMP_NEQ_HH;
5100 instr->jmp.ip = NULL; /* Resolved later. */
5102 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5103 instr->jmp.a.n_bits = fa->n_bits;
5104 instr->jmp.a.offset = fa->offset / 8;
5105 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5106 instr->jmp.b.n_bits = fb->n_bits;
5107 instr->jmp.b.offset = fb->offset / 8;
5112 b_val = strtoull(b, &b, 0);
5113 CHECK(!b[0], EINVAL);
5116 b_val = hton64(b_val) >> (64 - fa->n_bits);
5118 instr->type = INSTR_JMP_NEQ_I;
5119 instr->jmp.ip = NULL; /* Resolved later. */
5120 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5121 instr->jmp.a.n_bits = fa->n_bits;
5122 instr->jmp.a.offset = fa->offset / 8;
5123 instr->jmp.b_val = b_val;
5128 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5129 struct action *action,
5132 struct instruction *instr,
5133 struct instruction_data *data)
5135 char *a = tokens[2], *b = tokens[3];
5136 struct field *fa, *fb;
5138 uint32_t a_struct_id, b_struct_id;
5140 CHECK(n_tokens == 4, EINVAL);
5142 strcpy(data->jmp_label, tokens[1]);
5144 fa = struct_field_parse(p, action, a, &a_struct_id);
5146 CHECK(!fa->var_size, EINVAL);
5148 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5149 fb = struct_field_parse(p, action, b, &b_struct_id);
5151 CHECK(!fb->var_size, EINVAL);
5153 instr->type = INSTR_JMP_LT;
5154 if (a[0] == 'h' && b[0] != 'h')
5155 instr->type = INSTR_JMP_LT_HM;
5156 if (a[0] != 'h' && b[0] == 'h')
5157 instr->type = INSTR_JMP_LT_MH;
5158 if (a[0] == 'h' && b[0] == 'h')
5159 instr->type = INSTR_JMP_LT_HH;
5160 instr->jmp.ip = NULL; /* Resolved later. */
5162 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5163 instr->jmp.a.n_bits = fa->n_bits;
5164 instr->jmp.a.offset = fa->offset / 8;
5165 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5166 instr->jmp.b.n_bits = fb->n_bits;
5167 instr->jmp.b.offset = fb->offset / 8;
5171 /* JMP_LT_MI, JMP_LT_HI. */
5172 b_val = strtoull(b, &b, 0);
5173 CHECK(!b[0], EINVAL);
5175 instr->type = INSTR_JMP_LT_MI;
5177 instr->type = INSTR_JMP_LT_HI;
5178 instr->jmp.ip = NULL; /* Resolved later. */
5180 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5181 instr->jmp.a.n_bits = fa->n_bits;
5182 instr->jmp.a.offset = fa->offset / 8;
5183 instr->jmp.b_val = b_val;
5188 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5189 struct action *action,
5192 struct instruction *instr,
5193 struct instruction_data *data)
5195 char *a = tokens[2], *b = tokens[3];
5196 struct field *fa, *fb;
5198 uint32_t a_struct_id, b_struct_id;
5200 CHECK(n_tokens == 4, EINVAL);
5202 strcpy(data->jmp_label, tokens[1]);
5204 fa = struct_field_parse(p, action, a, &a_struct_id);
5206 CHECK(!fa->var_size, EINVAL);
5208 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5209 fb = struct_field_parse(p, action, b, &b_struct_id);
5211 CHECK(!fb->var_size, EINVAL);
5213 instr->type = INSTR_JMP_GT;
5214 if (a[0] == 'h' && b[0] != 'h')
5215 instr->type = INSTR_JMP_GT_HM;
5216 if (a[0] != 'h' && b[0] == 'h')
5217 instr->type = INSTR_JMP_GT_MH;
5218 if (a[0] == 'h' && b[0] == 'h')
5219 instr->type = INSTR_JMP_GT_HH;
5220 instr->jmp.ip = NULL; /* Resolved later. */
5222 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5223 instr->jmp.a.n_bits = fa->n_bits;
5224 instr->jmp.a.offset = fa->offset / 8;
5225 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5226 instr->jmp.b.n_bits = fb->n_bits;
5227 instr->jmp.b.offset = fb->offset / 8;
5231 /* JMP_GT_MI, JMP_GT_HI. */
5232 b_val = strtoull(b, &b, 0);
5233 CHECK(!b[0], EINVAL);
5235 instr->type = INSTR_JMP_GT_MI;
5237 instr->type = INSTR_JMP_GT_HI;
5238 instr->jmp.ip = NULL; /* Resolved later. */
5240 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5241 instr->jmp.a.n_bits = fa->n_bits;
5242 instr->jmp.a.offset = fa->offset / 8;
5243 instr->jmp.b_val = b_val;
5248 instr_jmp_exec(struct rte_swx_pipeline *p)
5250 struct thread *t = &p->threads[p->thread_id];
5251 struct instruction *ip = t->ip;
5253 TRACE("[Thread %2u] jmp\n", p->thread_id);
5255 thread_ip_set(t, ip->jmp.ip);
5259 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5261 struct thread *t = &p->threads[p->thread_id];
5262 struct instruction *ip = t->ip;
5263 uint32_t header_id = ip->jmp.header_id;
5265 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5267 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5271 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5273 struct thread *t = &p->threads[p->thread_id];
5274 struct instruction *ip = t->ip;
5275 uint32_t header_id = ip->jmp.header_id;
5277 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5279 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5283 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5285 struct thread *t = &p->threads[p->thread_id];
5286 struct instruction *ip = t->ip;
5287 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5289 TRACE("[Thread %2u] jmph\n", p->thread_id);
5291 t->ip = ip_next[t->hit];
5295 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5297 struct thread *t = &p->threads[p->thread_id];
5298 struct instruction *ip = t->ip;
5299 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5301 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5303 t->ip = ip_next[t->hit];
5307 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5309 struct thread *t = &p->threads[p->thread_id];
5310 struct instruction *ip = t->ip;
5312 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5314 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5318 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5320 struct thread *t = &p->threads[p->thread_id];
5321 struct instruction *ip = t->ip;
5323 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5325 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5329 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5331 struct thread *t = &p->threads[p->thread_id];
5332 struct instruction *ip = t->ip;
5334 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5340 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5342 struct thread *t = &p->threads[p->thread_id];
5343 struct instruction *ip = t->ip;
5345 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5347 JMP_CMP_MH(t, ip, ==);
5351 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5353 struct thread *t = &p->threads[p->thread_id];
5354 struct instruction *ip = t->ip;
5356 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5358 JMP_CMP_HM(t, ip, ==);
5362 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5364 struct thread *t = &p->threads[p->thread_id];
5365 struct instruction *ip = t->ip;
5367 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5369 JMP_CMP_HH_FAST(t, ip, ==);
5373 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5375 struct thread *t = &p->threads[p->thread_id];
5376 struct instruction *ip = t->ip;
5378 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5380 JMP_CMP_I(t, ip, ==);
5384 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5386 struct thread *t = &p->threads[p->thread_id];
5387 struct instruction *ip = t->ip;
5389 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5395 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5397 struct thread *t = &p->threads[p->thread_id];
5398 struct instruction *ip = t->ip;
5400 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5402 JMP_CMP_MH(t, ip, !=);
5406 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5408 struct thread *t = &p->threads[p->thread_id];
5409 struct instruction *ip = t->ip;
5411 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5413 JMP_CMP_HM(t, ip, !=);
5417 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5419 struct thread *t = &p->threads[p->thread_id];
5420 struct instruction *ip = t->ip;
5422 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5424 JMP_CMP_HH_FAST(t, ip, !=);
5428 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5430 struct thread *t = &p->threads[p->thread_id];
5431 struct instruction *ip = t->ip;
5433 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5435 JMP_CMP_I(t, ip, !=);
5439 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5441 struct thread *t = &p->threads[p->thread_id];
5442 struct instruction *ip = t->ip;
5444 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5450 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5452 struct thread *t = &p->threads[p->thread_id];
5453 struct instruction *ip = t->ip;
5455 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5457 JMP_CMP_MH(t, ip, <);
5461 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5463 struct thread *t = &p->threads[p->thread_id];
5464 struct instruction *ip = t->ip;
5466 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5468 JMP_CMP_HM(t, ip, <);
5472 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5474 struct thread *t = &p->threads[p->thread_id];
5475 struct instruction *ip = t->ip;
5477 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5479 JMP_CMP_HH(t, ip, <);
5483 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5485 struct thread *t = &p->threads[p->thread_id];
5486 struct instruction *ip = t->ip;
5488 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5490 JMP_CMP_MI(t, ip, <);
5494 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5496 struct thread *t = &p->threads[p->thread_id];
5497 struct instruction *ip = t->ip;
5499 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5501 JMP_CMP_HI(t, ip, <);
5505 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5507 struct thread *t = &p->threads[p->thread_id];
5508 struct instruction *ip = t->ip;
5510 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5516 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5518 struct thread *t = &p->threads[p->thread_id];
5519 struct instruction *ip = t->ip;
5521 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5523 JMP_CMP_MH(t, ip, >);
5527 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5529 struct thread *t = &p->threads[p->thread_id];
5530 struct instruction *ip = t->ip;
5532 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5534 JMP_CMP_HM(t, ip, >);
5538 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5540 struct thread *t = &p->threads[p->thread_id];
5541 struct instruction *ip = t->ip;
5543 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5545 JMP_CMP_HH(t, ip, >);
5549 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5551 struct thread *t = &p->threads[p->thread_id];
5552 struct instruction *ip = t->ip;
5554 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5556 JMP_CMP_MI(t, ip, >);
5560 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5562 struct thread *t = &p->threads[p->thread_id];
5563 struct instruction *ip = t->ip;
5565 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5567 JMP_CMP_HI(t, ip, >);
5574 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5575 struct action *action,
5576 char **tokens __rte_unused,
5578 struct instruction *instr,
5579 struct instruction_data *data __rte_unused)
5581 CHECK(action, EINVAL);
5582 CHECK(n_tokens == 1, EINVAL);
5584 instr->type = INSTR_RETURN;
5589 instr_return_exec(struct rte_swx_pipeline *p)
5591 struct thread *t = &p->threads[p->thread_id];
5593 TRACE("[Thread %2u] return\n", p->thread_id);
5599 instr_translate(struct rte_swx_pipeline *p,
5600 struct action *action,
5602 struct instruction *instr,
5603 struct instruction_data *data)
5605 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5606 int n_tokens = 0, tpos = 0;
5608 /* Parse the instruction string into tokens. */
5612 token = strtok_r(string, " \t\v", &string);
5616 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5617 CHECK_NAME(token, EINVAL);
5619 tokens[n_tokens] = token;
5623 CHECK(n_tokens, EINVAL);
5625 /* Handle the optional instruction label. */
5626 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5627 strcpy(data->label, tokens[0]);
5630 CHECK(n_tokens - tpos, EINVAL);
5633 /* Identify the instruction type. */
5634 if (!strcmp(tokens[tpos], "rx"))
5635 return instr_rx_translate(p,
5642 if (!strcmp(tokens[tpos], "tx"))
5643 return instr_tx_translate(p,
5650 if (!strcmp(tokens[tpos], "drop"))
5651 return instr_drop_translate(p,
5658 if (!strcmp(tokens[tpos], "extract"))
5659 return instr_hdr_extract_translate(p,
5666 if (!strcmp(tokens[tpos], "lookahead"))
5667 return instr_hdr_lookahead_translate(p,
5674 if (!strcmp(tokens[tpos], "emit"))
5675 return instr_hdr_emit_translate(p,
5682 if (!strcmp(tokens[tpos], "validate"))
5683 return instr_hdr_validate_translate(p,
5690 if (!strcmp(tokens[tpos], "invalidate"))
5691 return instr_hdr_invalidate_translate(p,
5698 if (!strcmp(tokens[tpos], "mov"))
5699 return instr_mov_translate(p,
5706 if (!strcmp(tokens[tpos], "add"))
5707 return instr_alu_add_translate(p,
5714 if (!strcmp(tokens[tpos], "sub"))
5715 return instr_alu_sub_translate(p,
5722 if (!strcmp(tokens[tpos], "ckadd"))
5723 return instr_alu_ckadd_translate(p,
5730 if (!strcmp(tokens[tpos], "cksub"))
5731 return instr_alu_cksub_translate(p,
5738 if (!strcmp(tokens[tpos], "and"))
5739 return instr_alu_and_translate(p,
5746 if (!strcmp(tokens[tpos], "or"))
5747 return instr_alu_or_translate(p,
5754 if (!strcmp(tokens[tpos], "xor"))
5755 return instr_alu_xor_translate(p,
5762 if (!strcmp(tokens[tpos], "shl"))
5763 return instr_alu_shl_translate(p,
5770 if (!strcmp(tokens[tpos], "shr"))
5771 return instr_alu_shr_translate(p,
5778 if (!strcmp(tokens[tpos], "regprefetch"))
5779 return instr_regprefetch_translate(p,
5786 if (!strcmp(tokens[tpos], "regrd"))
5787 return instr_regrd_translate(p,
5794 if (!strcmp(tokens[tpos], "regwr"))
5795 return instr_regwr_translate(p,
5802 if (!strcmp(tokens[tpos], "regadd"))
5803 return instr_regadd_translate(p,
5810 if (!strcmp(tokens[tpos], "metprefetch"))
5811 return instr_metprefetch_translate(p,
5818 if (!strcmp(tokens[tpos], "meter"))
5819 return instr_meter_translate(p,
5826 if (!strcmp(tokens[tpos], "table"))
5827 return instr_table_translate(p,
5834 if (!strcmp(tokens[tpos], "learn"))
5835 return instr_learn_translate(p,
5842 if (!strcmp(tokens[tpos], "forget"))
5843 return instr_forget_translate(p,
5850 if (!strcmp(tokens[tpos], "extern"))
5851 return instr_extern_translate(p,
5858 if (!strcmp(tokens[tpos], "jmp"))
5859 return instr_jmp_translate(p,
5866 if (!strcmp(tokens[tpos], "jmpv"))
5867 return instr_jmp_valid_translate(p,
5874 if (!strcmp(tokens[tpos], "jmpnv"))
5875 return instr_jmp_invalid_translate(p,
5882 if (!strcmp(tokens[tpos], "jmph"))
5883 return instr_jmp_hit_translate(p,
5890 if (!strcmp(tokens[tpos], "jmpnh"))
5891 return instr_jmp_miss_translate(p,
5898 if (!strcmp(tokens[tpos], "jmpa"))
5899 return instr_jmp_action_hit_translate(p,
5906 if (!strcmp(tokens[tpos], "jmpna"))
5907 return instr_jmp_action_miss_translate(p,
5914 if (!strcmp(tokens[tpos], "jmpeq"))
5915 return instr_jmp_eq_translate(p,
5922 if (!strcmp(tokens[tpos], "jmpneq"))
5923 return instr_jmp_neq_translate(p,
5930 if (!strcmp(tokens[tpos], "jmplt"))
5931 return instr_jmp_lt_translate(p,
5938 if (!strcmp(tokens[tpos], "jmpgt"))
5939 return instr_jmp_gt_translate(p,
5946 if (!strcmp(tokens[tpos], "return"))
5947 return instr_return_translate(p,
5957 static struct instruction_data *
5958 label_find(struct instruction_data *data, uint32_t n, const char *label)
5962 for (i = 0; i < n; i++)
5963 if (!strcmp(label, data[i].label))
5970 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5972 uint32_t count = 0, i;
5977 for (i = 0; i < n; i++)
5978 if (!strcmp(label, data[i].jmp_label))
5985 instr_label_check(struct instruction_data *instruction_data,
5986 uint32_t n_instructions)
5990 /* Check that all instruction labels are unique. */
5991 for (i = 0; i < n_instructions; i++) {
5992 struct instruction_data *data = &instruction_data[i];
5993 char *label = data->label;
5999 for (j = i + 1; j < n_instructions; j++)
6000 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6003 /* Get users for each instruction label. */
6004 for (i = 0; i < n_instructions; i++) {
6005 struct instruction_data *data = &instruction_data[i];
6006 char *label = data->label;
6008 data->n_users = label_is_used(instruction_data,
6017 instr_jmp_resolve(struct instruction *instructions,
6018 struct instruction_data *instruction_data,
6019 uint32_t n_instructions)
6023 for (i = 0; i < n_instructions; i++) {
6024 struct instruction *instr = &instructions[i];
6025 struct instruction_data *data = &instruction_data[i];
6026 struct instruction_data *found;
6028 if (!instruction_is_jmp(instr))
6031 found = label_find(instruction_data,
6034 CHECK(found, EINVAL);
6036 instr->jmp.ip = &instructions[found - instruction_data];
6043 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6045 struct instruction *instr,
6046 struct instruction_data *data __rte_unused,
6047 uint32_t n_instructions)
6050 enum instruction_type type;
6053 /* Check that the first instruction is rx. */
6054 CHECK(instr[0].type == INSTR_RX, EINVAL);
6056 /* Check that there is at least one tx instruction. */
6057 for (i = 0; i < n_instructions; i++) {
6058 type = instr[i].type;
6060 if (instruction_is_tx(type))
6063 CHECK(i < n_instructions, EINVAL);
6065 /* Check that the last instruction is either tx or unconditional
6068 type = instr[n_instructions - 1].type;
6069 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6073 enum instruction_type type;
6076 /* Check that there is at least one return or tx instruction. */
6077 for (i = 0; i < n_instructions; i++) {
6078 type = instr[i].type;
6080 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6083 CHECK(i < n_instructions, EINVAL);
6090 instr_compact(struct instruction *instructions,
6091 struct instruction_data *instruction_data,
6092 uint32_t n_instructions)
6094 uint32_t i, pos = 0;
6096 /* Eliminate the invalid instructions that have been optimized out. */
6097 for (i = 0; i < n_instructions; i++) {
6098 struct instruction *instr = &instructions[i];
6099 struct instruction_data *data = &instruction_data[i];
6105 memcpy(&instructions[pos], instr, sizeof(*instr));
6106 memcpy(&instruction_data[pos], data, sizeof(*data));
6116 instr_pattern_extract_many_search(struct instruction *instr,
6117 struct instruction_data *data,
6119 uint32_t *n_pattern_instr)
6123 for (i = 0; i < n_instr; i++) {
6124 if (data[i].invalid)
6127 if (instr[i].type != INSTR_HDR_EXTRACT)
6130 if (i == RTE_DIM(instr->io.hdr.header_id))
6133 if (i && data[i].n_users)
6140 *n_pattern_instr = i;
6145 instr_pattern_extract_many_replace(struct instruction *instr,
6146 struct instruction_data *data,
6151 for (i = 1; i < n_instr; i++) {
6153 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6154 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6155 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6157 data[i].invalid = 1;
6162 instr_pattern_extract_many_optimize(struct instruction *instructions,
6163 struct instruction_data *instruction_data,
6164 uint32_t n_instructions)
6168 for (i = 0; i < n_instructions; ) {
6169 struct instruction *instr = &instructions[i];
6170 struct instruction_data *data = &instruction_data[i];
6171 uint32_t n_instr = 0;
6175 detected = instr_pattern_extract_many_search(instr,
6180 instr_pattern_extract_many_replace(instr,
6187 /* No pattern starting at the current instruction. */
6191 /* Eliminate the invalid instructions that have been optimized out. */
6192 n_instructions = instr_compact(instructions,
6196 return n_instructions;
6200 instr_pattern_emit_many_tx_search(struct instruction *instr,
6201 struct instruction_data *data,
6203 uint32_t *n_pattern_instr)
6207 for (i = 0; i < n_instr; i++) {
6208 if (data[i].invalid)
6211 if (instr[i].type != INSTR_HDR_EMIT)
6214 if (i == RTE_DIM(instr->io.hdr.header_id))
6217 if (i && data[i].n_users)
6224 if (instr[i].type != INSTR_TX)
6227 if (data[i].n_users)
6232 *n_pattern_instr = i;
6237 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6238 struct instruction_data *data,
6243 /* Any emit instruction in addition to the first one. */
6244 for (i = 1; i < n_instr - 1; i++) {
6246 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6247 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6248 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6250 data[i].invalid = 1;
6253 /* The TX instruction is the last one in the pattern. */
6255 instr[0].io.io.offset = instr[i].io.io.offset;
6256 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6257 data[i].invalid = 1;
6261 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6262 struct instruction_data *instruction_data,
6263 uint32_t n_instructions)
6267 for (i = 0; i < n_instructions; ) {
6268 struct instruction *instr = &instructions[i];
6269 struct instruction_data *data = &instruction_data[i];
6270 uint32_t n_instr = 0;
6273 /* Emit many + TX. */
6274 detected = instr_pattern_emit_many_tx_search(instr,
6279 instr_pattern_emit_many_tx_replace(instr,
6286 /* No pattern starting at the current instruction. */
6290 /* Eliminate the invalid instructions that have been optimized out. */
6291 n_instructions = instr_compact(instructions,
6295 return n_instructions;
6299 action_arg_src_mov_count(struct action *a,
6301 struct instruction *instructions,
6302 struct instruction_data *instruction_data,
6303 uint32_t n_instructions);
6306 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6308 struct instruction *instr,
6309 struct instruction_data *data,
6311 struct instruction *instructions,
6312 struct instruction_data *instruction_data,
6313 uint32_t n_instructions,
6314 uint32_t *n_pattern_instr)
6317 uint32_t src_field_id, i, j;
6319 /* Prerequisites. */
6323 /* First instruction: MOV_HM. */
6324 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6327 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6328 if (!h || h->st->var_size)
6331 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6332 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6335 if (src_field_id == a->st->n_fields)
6338 if (instr[0].mov.dst.offset ||
6339 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6340 instr[0].mov.src.struct_id ||
6341 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6342 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6345 if ((n_instr < h->st->n_fields + 1) ||
6346 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6349 /* Subsequent instructions: MOV_HM. */
6350 for (i = 1; i < h->st->n_fields; i++)
6351 if (data[i].invalid ||
6353 (instr[i].type != INSTR_MOV_HM) ||
6354 (instr[i].mov.dst.struct_id != h->struct_id) ||
6355 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6356 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6357 instr[i].mov.src.struct_id ||
6358 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6359 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6360 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6363 /* Last instruction: HDR_VALIDATE. */
6364 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6365 (instr[i].valid.header_id != h->id))
6368 /* Check that none of the action args that are used as source for this
6369 * DMA transfer are not used as source in any other mov instruction.
6371 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6374 n_users = action_arg_src_mov_count(a,
6383 *n_pattern_instr = 1 + i;
6388 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6390 struct instruction *instr,
6391 struct instruction_data *data,
6395 uint32_t src_field_id, src_offset, i;
6397 /* Read from the instructions before they are modified. */
6398 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6402 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6403 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6406 if (src_field_id == a->st->n_fields)
6409 src_offset = instr[0].mov.src.offset;
6411 /* Modify the instructions. */
6412 instr[0].type = INSTR_DMA_HT;
6413 instr[0].dma.dst.header_id[0] = h->id;
6414 instr[0].dma.dst.struct_id[0] = h->struct_id;
6415 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6416 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6418 for (i = 1; i < n_instr; i++)
6419 data[i].invalid = 1;
6421 /* Update the endianness of the action arguments to header endianness. */
6422 for (i = 0; i < h->st->n_fields; i++)
6423 a->args_endianness[src_field_id + i] = 1;
6427 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6429 struct instruction *instructions,
6430 struct instruction_data *instruction_data,
6431 uint32_t n_instructions)
6436 return n_instructions;
6438 for (i = 0; i < n_instructions; ) {
6439 struct instruction *instr = &instructions[i];
6440 struct instruction_data *data = &instruction_data[i];
6441 uint32_t n_instr = 0;
6444 /* Mov all + validate. */
6445 detected = instr_pattern_mov_all_validate_search(p,
6455 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6460 /* No pattern starting at the current instruction. */
6464 /* Eliminate the invalid instructions that have been optimized out. */
6465 n_instructions = instr_compact(instructions,
6469 return n_instructions;
6473 instr_pattern_dma_many_search(struct instruction *instr,
6474 struct instruction_data *data,
6476 uint32_t *n_pattern_instr)
6480 for (i = 0; i < n_instr; i++) {
6481 if (data[i].invalid)
6484 if (instr[i].type != INSTR_DMA_HT)
6487 if (i == RTE_DIM(instr->dma.dst.header_id))
6490 if (i && data[i].n_users)
6497 *n_pattern_instr = i;
6502 instr_pattern_dma_many_replace(struct instruction *instr,
6503 struct instruction_data *data,
6508 for (i = 1; i < n_instr; i++) {
6510 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6511 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6512 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6513 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6515 data[i].invalid = 1;
6520 instr_pattern_dma_many_optimize(struct instruction *instructions,
6521 struct instruction_data *instruction_data,
6522 uint32_t n_instructions)
6526 for (i = 0; i < n_instructions; ) {
6527 struct instruction *instr = &instructions[i];
6528 struct instruction_data *data = &instruction_data[i];
6529 uint32_t n_instr = 0;
6533 detected = instr_pattern_dma_many_search(instr,
6538 instr_pattern_dma_many_replace(instr, data, n_instr);
6543 /* No pattern starting at the current instruction. */
6547 /* Eliminate the invalid instructions that have been optimized out. */
6548 n_instructions = instr_compact(instructions,
6552 return n_instructions;
6556 instr_optimize(struct rte_swx_pipeline *p,
6558 struct instruction *instructions,
6559 struct instruction_data *instruction_data,
6560 uint32_t n_instructions)
6563 n_instructions = instr_pattern_extract_many_optimize(instructions,
6567 /* Emit many + TX. */
6568 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6572 /* Mov all + validate. */
6573 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6580 n_instructions = instr_pattern_dma_many_optimize(instructions,
6584 return n_instructions;
6588 instruction_config(struct rte_swx_pipeline *p,
6590 const char **instructions,
6591 uint32_t n_instructions)
6593 struct instruction *instr = NULL;
6594 struct instruction_data *data = NULL;
6598 CHECK(n_instructions, EINVAL);
6599 CHECK(instructions, EINVAL);
6600 for (i = 0; i < n_instructions; i++)
6601 CHECK_INSTRUCTION(instructions[i], EINVAL);
6603 /* Memory allocation. */
6604 instr = calloc(n_instructions, sizeof(struct instruction));
6610 data = calloc(n_instructions, sizeof(struct instruction_data));
6616 for (i = 0; i < n_instructions; i++) {
6617 char *string = strdup(instructions[i]);
6623 err = instr_translate(p, a, string, &instr[i], &data[i]);
6632 err = instr_label_check(data, n_instructions);
6636 err = instr_verify(p, a, instr, data, n_instructions);
6640 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6642 err = instr_jmp_resolve(instr, data, n_instructions);
6647 a->instructions = instr;
6648 a->instruction_data = data;
6649 a->n_instructions = n_instructions;
6651 p->instructions = instr;
6652 p->instruction_data = data;
6653 p->n_instructions = n_instructions;
6664 static instr_exec_t instruction_table[] = {
6665 [INSTR_RX] = instr_rx_exec,
6666 [INSTR_TX] = instr_tx_exec,
6667 [INSTR_TX_I] = instr_tx_i_exec,
6668 [INSTR_DROP] = instr_drop_exec,
6670 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6671 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6672 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6673 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6674 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6675 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6676 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6677 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6678 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6679 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6681 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6682 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6683 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6684 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6685 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6686 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6687 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6688 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6689 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6691 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6692 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6694 [INSTR_MOV] = instr_mov_exec,
6695 [INSTR_MOV_MH] = instr_mov_mh_exec,
6696 [INSTR_MOV_HM] = instr_mov_hm_exec,
6697 [INSTR_MOV_HH] = instr_mov_hh_exec,
6698 [INSTR_MOV_I] = instr_mov_i_exec,
6700 [INSTR_DMA_HT] = instr_dma_ht_exec,
6701 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6702 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6703 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6704 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6705 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6706 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6707 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6709 [INSTR_ALU_ADD] = instr_alu_add_exec,
6710 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6711 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6712 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6713 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6714 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6716 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6717 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6718 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6719 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6720 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6721 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6723 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6724 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6725 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6726 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6728 [INSTR_ALU_AND] = instr_alu_and_exec,
6729 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6730 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6731 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6732 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6734 [INSTR_ALU_OR] = instr_alu_or_exec,
6735 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6736 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6737 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6738 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6740 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6741 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6742 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6743 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6744 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6746 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6747 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6748 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6749 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6750 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6751 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6753 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6754 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6755 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6756 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6757 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6758 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6760 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6761 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6762 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6764 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6765 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6766 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6767 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6768 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6769 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6771 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6772 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6773 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6774 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6775 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6776 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6777 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6778 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6779 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6781 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6782 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6783 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6784 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6785 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6786 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6787 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6788 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6789 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6791 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6792 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6793 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6795 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6796 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6797 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6798 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6799 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6800 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6801 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6802 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6803 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6804 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6805 [INSTR_METER_IMM] = instr_meter_imm_exec,
6806 [INSTR_METER_IMI] = instr_meter_imi_exec,
6808 [INSTR_TABLE] = instr_table_exec,
6809 [INSTR_TABLE_AF] = instr_table_af_exec,
6810 [INSTR_SELECTOR] = instr_selector_exec,
6811 [INSTR_LEARNER] = instr_learner_exec,
6812 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6813 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6814 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6815 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6816 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6818 [INSTR_JMP] = instr_jmp_exec,
6819 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6820 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6821 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6822 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6823 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6824 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6826 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6827 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6828 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6829 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6830 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6832 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6833 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6834 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6835 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6836 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6838 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6839 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6840 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6841 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6842 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6843 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6845 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6846 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6847 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6848 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6849 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6850 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6852 [INSTR_RETURN] = instr_return_exec,
6856 instruction_table_build(struct rte_swx_pipeline *p)
6858 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6859 sizeof(struct instr_exec_t *));
6860 if (!p->instruction_table)
6863 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6869 instruction_table_build_free(struct rte_swx_pipeline *p)
6871 if (!p->instruction_table)
6874 free(p->instruction_table);
6875 p->instruction_table = NULL;
6879 instruction_table_free(struct rte_swx_pipeline *p)
6881 instruction_table_build_free(p);
6885 instr_exec(struct rte_swx_pipeline *p)
6887 struct thread *t = &p->threads[p->thread_id];
6888 struct instruction *ip = t->ip;
6889 instr_exec_t instr = p->instruction_table[ip->type];
6897 static struct action *
6898 action_find(struct rte_swx_pipeline *p, const char *name)
6900 struct action *elem;
6905 TAILQ_FOREACH(elem, &p->actions, node)
6906 if (strcmp(elem->name, name) == 0)
6912 static struct action *
6913 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6915 struct action *action = NULL;
6917 TAILQ_FOREACH(action, &p->actions, node)
6918 if (action->id == id)
6924 static struct field *
6925 action_field_find(struct action *a, const char *name)
6927 return a->st ? struct_type_field_find(a->st, name) : NULL;
6930 static struct field *
6931 action_field_parse(struct action *action, const char *name)
6933 if (name[0] != 't' || name[1] != '.')
6936 return action_field_find(action, &name[2]);
6940 action_has_nbo_args(struct action *a)
6944 /* Return if the action does not have any args. */
6946 return 0; /* FALSE */
6948 for (i = 0; i < a->st->n_fields; i++)
6949 if (a->args_endianness[i])
6950 return 1; /* TRUE */
6952 return 0; /* FALSE */
6956 action_does_learning(struct action *a)
6960 for (i = 0; i < a->n_instructions; i++)
6961 switch (a->instructions[i].type) {
6962 case INSTR_LEARNER_LEARN:
6963 return 1; /* TRUE */
6965 case INSTR_LEARNER_FORGET:
6966 return 1; /* TRUE */
6972 return 0; /* FALSE */
6976 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6978 const char *args_struct_type_name,
6979 const char **instructions,
6980 uint32_t n_instructions)
6982 struct struct_type *args_struct_type = NULL;
6988 CHECK_NAME(name, EINVAL);
6989 CHECK(!action_find(p, name), EEXIST);
6991 if (args_struct_type_name) {
6992 CHECK_NAME(args_struct_type_name, EINVAL);
6993 args_struct_type = struct_type_find(p, args_struct_type_name);
6994 CHECK(args_struct_type, EINVAL);
6995 CHECK(!args_struct_type->var_size, EINVAL);
6998 /* Node allocation. */
6999 a = calloc(1, sizeof(struct action));
7001 if (args_struct_type) {
7002 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7003 if (!a->args_endianness) {
7009 /* Node initialization. */
7010 strcpy(a->name, name);
7011 a->st = args_struct_type;
7012 a->id = p->n_actions;
7014 /* Instruction translation. */
7015 err = instruction_config(p, a, instructions, n_instructions);
7017 free(a->args_endianness);
7022 /* Node add to tailq. */
7023 TAILQ_INSERT_TAIL(&p->actions, a, node);
7030 action_build(struct rte_swx_pipeline *p)
7032 struct action *action;
7034 /* p->action_instructions. */
7035 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7036 CHECK(p->action_instructions, ENOMEM);
7038 TAILQ_FOREACH(action, &p->actions, node)
7039 p->action_instructions[action->id] = action->instructions;
7041 /* p->action_funcs. */
7042 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7043 CHECK(p->action_funcs, ENOMEM);
7049 action_build_free(struct rte_swx_pipeline *p)
7051 free(p->action_funcs);
7052 p->action_funcs = NULL;
7054 free(p->action_instructions);
7055 p->action_instructions = NULL;
7059 action_free(struct rte_swx_pipeline *p)
7061 action_build_free(p);
7064 struct action *action;
7066 action = TAILQ_FIRST(&p->actions);
7070 TAILQ_REMOVE(&p->actions, action, node);
7071 free(action->instruction_data);
7072 free(action->instructions);
7078 action_arg_src_mov_count(struct action *a,
7080 struct instruction *instructions,
7081 struct instruction_data *instruction_data,
7082 uint32_t n_instructions)
7084 uint32_t offset, n_users = 0, i;
7087 (arg_id >= a->st->n_fields) ||
7089 !instruction_data ||
7093 offset = a->st->fields[arg_id].offset / 8;
7095 for (i = 0; i < n_instructions; i++) {
7096 struct instruction *instr = &instructions[i];
7097 struct instruction_data *data = &instruction_data[i];
7099 if (data->invalid ||
7100 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7101 instr->mov.src.struct_id ||
7102 (instr->mov.src.offset != offset))
7114 static struct table_type *
7115 table_type_find(struct rte_swx_pipeline *p, const char *name)
7117 struct table_type *elem;
7119 TAILQ_FOREACH(elem, &p->table_types, node)
7120 if (strcmp(elem->name, name) == 0)
7126 static struct table_type *
7127 table_type_resolve(struct rte_swx_pipeline *p,
7128 const char *recommended_type_name,
7129 enum rte_swx_table_match_type match_type)
7131 struct table_type *elem;
7133 /* Only consider the recommended type if the match type is correct. */
7134 if (recommended_type_name)
7135 TAILQ_FOREACH(elem, &p->table_types, node)
7136 if (!strcmp(elem->name, recommended_type_name) &&
7137 (elem->match_type == match_type))
7140 /* Ignore the recommended type and get the first element with this match
7143 TAILQ_FOREACH(elem, &p->table_types, node)
7144 if (elem->match_type == match_type)
7150 static struct table *
7151 table_find(struct rte_swx_pipeline *p, const char *name)
7155 TAILQ_FOREACH(elem, &p->tables, node)
7156 if (strcmp(elem->name, name) == 0)
7162 static struct table *
7163 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7165 struct table *table = NULL;
7167 TAILQ_FOREACH(table, &p->tables, node)
7168 if (table->id == id)
7175 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7177 enum rte_swx_table_match_type match_type,
7178 struct rte_swx_table_ops *ops)
7180 struct table_type *elem;
7184 CHECK_NAME(name, EINVAL);
7185 CHECK(!table_type_find(p, name), EEXIST);
7188 CHECK(ops->create, EINVAL);
7189 CHECK(ops->lkp, EINVAL);
7190 CHECK(ops->free, EINVAL);
7192 /* Node allocation. */
7193 elem = calloc(1, sizeof(struct table_type));
7194 CHECK(elem, ENOMEM);
7196 /* Node initialization. */
7197 strcpy(elem->name, name);
7198 elem->match_type = match_type;
7199 memcpy(&elem->ops, ops, sizeof(*ops));
7201 /* Node add to tailq. */
7202 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7208 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7210 enum rte_swx_table_match_type *match_type)
7212 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7214 for (i = 0; i < n_fields; i++) {
7215 struct rte_swx_match_field_params *f = &fields[i];
7217 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7220 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7224 if ((n_fields_lpm > 1) ||
7225 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7228 *match_type = (n_fields_em == n_fields) ?
7229 RTE_SWX_TABLE_MATCH_EXACT :
7230 RTE_SWX_TABLE_MATCH_WILDCARD;
7236 table_match_fields_check(struct rte_swx_pipeline *p,
7237 struct rte_swx_pipeline_table_params *params,
7238 struct header **header)
7240 struct header *h0 = NULL;
7241 struct field *hf, *mf;
7242 uint32_t *offset = NULL, i;
7245 /* Return if no match fields. */
7246 if (!params->n_fields) {
7247 if (params->fields) {
7258 /* Memory allocation. */
7259 offset = calloc(params->n_fields, sizeof(uint32_t));
7265 /* Check that all the match fields belong to either the same header or
7268 hf = header_field_parse(p, params->fields[0].name, &h0);
7269 mf = metadata_field_parse(p, params->fields[0].name);
7270 if ((!hf && !mf) || (hf && hf->var_size)) {
7275 offset[0] = h0 ? hf->offset : mf->offset;
7277 for (i = 1; i < params->n_fields; i++)
7281 hf = header_field_parse(p, params->fields[i].name, &h);
7282 if (!hf || (h->id != h0->id) || hf->var_size) {
7287 offset[i] = hf->offset;
7289 mf = metadata_field_parse(p, params->fields[i].name);
7295 offset[i] = mf->offset;
7298 /* Check that there are no duplicated match fields. */
7299 for (i = 0; i < params->n_fields; i++) {
7302 for (j = 0; j < i; j++)
7303 if (offset[j] == offset[i]) {
7319 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7321 struct rte_swx_pipeline_table_params *params,
7322 const char *recommended_table_type_name,
7326 struct table_type *type;
7327 struct table *t = NULL;
7328 struct action *default_action;
7329 struct header *header = NULL;
7330 uint32_t action_data_size_max = 0, i;
7335 CHECK_NAME(name, EINVAL);
7336 CHECK(!table_find(p, name), EEXIST);
7337 CHECK(!selector_find(p, name), EEXIST);
7338 CHECK(!learner_find(p, name), EEXIST);
7340 CHECK(params, EINVAL);
7343 status = table_match_fields_check(p, params, &header);
7347 /* Action checks. */
7348 CHECK(params->n_actions, EINVAL);
7349 CHECK(params->action_names, EINVAL);
7350 for (i = 0; i < params->n_actions; i++) {
7351 const char *action_name = params->action_names[i];
7353 uint32_t action_data_size;
7354 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7356 CHECK_NAME(action_name, EINVAL);
7358 a = action_find(p, action_name);
7360 CHECK(!action_does_learning(a), EINVAL);
7362 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7363 if (action_data_size > action_data_size_max)
7364 action_data_size_max = action_data_size;
7366 if (params->action_is_for_table_entries)
7367 action_is_for_table_entries = params->action_is_for_table_entries[i];
7368 if (params->action_is_for_default_entry)
7369 action_is_for_default_entry = params->action_is_for_default_entry[i];
7370 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7373 CHECK_NAME(params->default_action_name, EINVAL);
7374 for (i = 0; i < p->n_actions; i++)
7375 if (!strcmp(params->action_names[i],
7376 params->default_action_name))
7378 CHECK(i < params->n_actions, EINVAL);
7379 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7382 default_action = action_find(p, params->default_action_name);
7383 CHECK((default_action->st && params->default_action_data) ||
7384 !params->default_action_data, EINVAL);
7386 /* Table type checks. */
7387 if (recommended_table_type_name)
7388 CHECK_NAME(recommended_table_type_name, EINVAL);
7390 if (params->n_fields) {
7391 enum rte_swx_table_match_type match_type;
7393 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7397 type = table_type_resolve(p, recommended_table_type_name, match_type);
7398 CHECK(type, EINVAL);
7403 /* Memory allocation. */
7404 t = calloc(1, sizeof(struct table));
7408 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7412 t->actions = calloc(params->n_actions, sizeof(struct action *));
7416 if (action_data_size_max) {
7417 t->default_action_data = calloc(1, action_data_size_max);
7418 if (!t->default_action_data)
7422 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7423 if (!t->action_is_for_table_entries)
7426 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7427 if (!t->action_is_for_default_entry)
7430 /* Node initialization. */
7431 strcpy(t->name, name);
7432 if (args && args[0])
7433 strcpy(t->args, args);
7436 for (i = 0; i < params->n_fields; i++) {
7437 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7438 struct match_field *f = &t->fields[i];
7440 f->match_type = field->match_type;
7442 header_field_parse(p, field->name, NULL) :
7443 metadata_field_parse(p, field->name);
7445 t->n_fields = params->n_fields;
7448 for (i = 0; i < params->n_actions; i++) {
7449 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7451 if (params->action_is_for_table_entries)
7452 action_is_for_table_entries = params->action_is_for_table_entries[i];
7453 if (params->action_is_for_default_entry)
7454 action_is_for_default_entry = params->action_is_for_default_entry[i];
7456 t->actions[i] = action_find(p, params->action_names[i]);
7457 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7458 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7460 t->default_action = default_action;
7461 if (default_action->st)
7462 memcpy(t->default_action_data,
7463 params->default_action_data,
7464 default_action->st->n_bits / 8);
7465 t->n_actions = params->n_actions;
7466 t->default_action_is_const = params->default_action_is_const;
7467 t->action_data_size_max = action_data_size_max;
7470 t->id = p->n_tables;
7472 /* Node add to tailq. */
7473 TAILQ_INSERT_TAIL(&p->tables, t, node);
7482 free(t->action_is_for_default_entry);
7483 free(t->action_is_for_table_entries);
7484 free(t->default_action_data);
7492 static struct rte_swx_table_params *
7493 table_params_get(struct table *table)
7495 struct rte_swx_table_params *params;
7496 struct field *first, *last;
7498 uint32_t key_size, key_offset, action_data_size, i;
7500 /* Memory allocation. */
7501 params = calloc(1, sizeof(struct rte_swx_table_params));
7505 /* Find first (smallest offset) and last (biggest offset) match fields. */
7506 first = table->fields[0].field;
7507 last = table->fields[0].field;
7509 for (i = 0; i < table->n_fields; i++) {
7510 struct field *f = table->fields[i].field;
7512 if (f->offset < first->offset)
7515 if (f->offset > last->offset)
7519 /* Key offset and size. */
7520 key_offset = first->offset / 8;
7521 key_size = (last->offset + last->n_bits - first->offset) / 8;
7523 /* Memory allocation. */
7524 key_mask = calloc(1, key_size);
7531 for (i = 0; i < table->n_fields; i++) {
7532 struct field *f = table->fields[i].field;
7533 uint32_t start = (f->offset - first->offset) / 8;
7534 size_t size = f->n_bits / 8;
7536 memset(&key_mask[start], 0xFF, size);
7539 /* Action data size. */
7540 action_data_size = 0;
7541 for (i = 0; i < table->n_actions; i++) {
7542 struct action *action = table->actions[i];
7543 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7545 if (ads > action_data_size)
7546 action_data_size = ads;
7550 params->match_type = table->type->match_type;
7551 params->key_size = key_size;
7552 params->key_offset = key_offset;
7553 params->key_mask0 = key_mask;
7554 params->action_data_size = action_data_size;
7555 params->n_keys_max = table->size;
7561 table_params_free(struct rte_swx_table_params *params)
7566 free(params->key_mask0);
7571 table_stub_lkp(void *table __rte_unused,
7572 void *mailbox __rte_unused,
7573 uint8_t **key __rte_unused,
7574 uint64_t *action_id __rte_unused,
7575 uint8_t **action_data __rte_unused,
7579 return 1; /* DONE. */
7583 table_build(struct rte_swx_pipeline *p)
7587 /* Per pipeline: table statistics. */
7588 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7589 CHECK(p->table_stats, ENOMEM);
7591 for (i = 0; i < p->n_tables; i++) {
7592 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7593 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7596 /* Per thread: table runt-time. */
7597 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7598 struct thread *t = &p->threads[i];
7599 struct table *table;
7601 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7602 CHECK(t->tables, ENOMEM);
7604 TAILQ_FOREACH(table, &p->tables, node) {
7605 struct table_runtime *r = &t->tables[table->id];
7610 size = table->type->ops.mailbox_size_get();
7613 r->func = table->type->ops.lkp;
7617 r->mailbox = calloc(1, size);
7618 CHECK(r->mailbox, ENOMEM);
7622 r->key = table->header ?
7623 &t->structs[table->header->struct_id] :
7624 &t->structs[p->metadata_struct_id];
7626 r->func = table_stub_lkp;
7635 table_build_free(struct rte_swx_pipeline *p)
7639 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7640 struct thread *t = &p->threads[i];
7646 for (j = 0; j < p->n_tables; j++) {
7647 struct table_runtime *r = &t->tables[j];
7656 if (p->table_stats) {
7657 for (i = 0; i < p->n_tables; i++)
7658 free(p->table_stats[i].n_pkts_action);
7660 free(p->table_stats);
7665 table_free(struct rte_swx_pipeline *p)
7667 table_build_free(p);
7673 elem = TAILQ_FIRST(&p->tables);
7677 TAILQ_REMOVE(&p->tables, elem, node);
7679 free(elem->actions);
7680 free(elem->default_action_data);
7686 struct table_type *elem;
7688 elem = TAILQ_FIRST(&p->table_types);
7692 TAILQ_REMOVE(&p->table_types, elem, node);
7700 static struct selector *
7701 selector_find(struct rte_swx_pipeline *p, const char *name)
7705 TAILQ_FOREACH(s, &p->selectors, node)
7706 if (strcmp(s->name, name) == 0)
7712 static struct selector *
7713 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7715 struct selector *s = NULL;
7717 TAILQ_FOREACH(s, &p->selectors, node)
7725 selector_fields_check(struct rte_swx_pipeline *p,
7726 struct rte_swx_pipeline_selector_params *params,
7727 struct header **header)
7729 struct header *h0 = NULL;
7730 struct field *hf, *mf;
7733 /* Return if no selector fields. */
7734 if (!params->n_selector_fields || !params->selector_field_names)
7737 /* Check that all the selector fields either belong to the same header
7738 * or are all meta-data fields.
7740 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7741 mf = metadata_field_parse(p, params->selector_field_names[0]);
7745 for (i = 1; i < params->n_selector_fields; i++)
7749 hf = header_field_parse(p, params->selector_field_names[i], &h);
7750 if (!hf || (h->id != h0->id))
7753 mf = metadata_field_parse(p, params->selector_field_names[i]);
7758 /* Check that there are no duplicated match fields. */
7759 for (i = 0; i < params->n_selector_fields; i++) {
7760 const char *field_name = params->selector_field_names[i];
7763 for (j = i + 1; j < params->n_selector_fields; j++)
7764 if (!strcmp(params->selector_field_names[j], field_name))
7776 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7778 struct rte_swx_pipeline_selector_params *params)
7781 struct header *selector_header = NULL;
7782 struct field *group_id_field, *member_id_field;
7788 CHECK_NAME(name, EINVAL);
7789 CHECK(!table_find(p, name), EEXIST);
7790 CHECK(!selector_find(p, name), EEXIST);
7791 CHECK(!learner_find(p, name), EEXIST);
7793 CHECK(params, EINVAL);
7795 CHECK_NAME(params->group_id_field_name, EINVAL);
7796 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7797 CHECK(group_id_field, EINVAL);
7799 for (i = 0; i < params->n_selector_fields; i++) {
7800 const char *field_name = params->selector_field_names[i];
7802 CHECK_NAME(field_name, EINVAL);
7804 status = selector_fields_check(p, params, &selector_header);
7808 CHECK_NAME(params->member_id_field_name, EINVAL);
7809 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7810 CHECK(member_id_field, EINVAL);
7812 CHECK(params->n_groups_max, EINVAL);
7814 CHECK(params->n_members_per_group_max, EINVAL);
7816 /* Memory allocation. */
7817 s = calloc(1, sizeof(struct selector));
7823 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7824 if (!s->selector_fields) {
7829 /* Node initialization. */
7830 strcpy(s->name, name);
7832 s->group_id_field = group_id_field;
7834 for (i = 0; i < params->n_selector_fields; i++) {
7835 const char *field_name = params->selector_field_names[i];
7837 s->selector_fields[i] = selector_header ?
7838 header_field_parse(p, field_name, NULL) :
7839 metadata_field_parse(p, field_name);
7842 s->n_selector_fields = params->n_selector_fields;
7844 s->selector_header = selector_header;
7846 s->member_id_field = member_id_field;
7848 s->n_groups_max = params->n_groups_max;
7850 s->n_members_per_group_max = params->n_members_per_group_max;
7852 s->id = p->n_selectors;
7854 /* Node add to tailq. */
7855 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7864 free(s->selector_fields);
7872 selector_params_free(struct rte_swx_table_selector_params *params)
7877 free(params->selector_mask);
7882 static struct rte_swx_table_selector_params *
7883 selector_table_params_get(struct selector *s)
7885 struct rte_swx_table_selector_params *params = NULL;
7886 struct field *first, *last;
7889 /* Memory allocation. */
7890 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7895 params->group_id_offset = s->group_id_field->offset / 8;
7897 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7898 first = s->selector_fields[0];
7899 last = s->selector_fields[0];
7901 for (i = 0; i < s->n_selector_fields; i++) {
7902 struct field *f = s->selector_fields[i];
7904 if (f->offset < first->offset)
7907 if (f->offset > last->offset)
7911 /* Selector offset and size. */
7912 params->selector_offset = first->offset / 8;
7913 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7915 /* Memory allocation. */
7916 params->selector_mask = calloc(1, params->selector_size);
7917 if (!params->selector_mask)
7920 /* Selector mask. */
7921 for (i = 0; i < s->n_selector_fields; i++) {
7922 struct field *f = s->selector_fields[i];
7923 uint32_t start = (f->offset - first->offset) / 8;
7924 size_t size = f->n_bits / 8;
7926 memset(¶ms->selector_mask[start], 0xFF, size);
7930 params->member_id_offset = s->member_id_field->offset / 8;
7932 /* Maximum number of groups. */
7933 params->n_groups_max = s->n_groups_max;
7935 /* Maximum number of members per group. */
7936 params->n_members_per_group_max = s->n_members_per_group_max;
7941 selector_params_free(params);
7946 selector_build_free(struct rte_swx_pipeline *p)
7950 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7951 struct thread *t = &p->threads[i];
7957 for (j = 0; j < p->n_selectors; j++) {
7958 struct selector_runtime *r = &t->selectors[j];
7964 t->selectors = NULL;
7967 free(p->selector_stats);
7968 p->selector_stats = NULL;
7972 selector_build(struct rte_swx_pipeline *p)
7977 /* Per pipeline: selector statistics. */
7978 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7979 if (!p->selector_stats) {
7984 /* Per thread: selector run-time. */
7985 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7986 struct thread *t = &p->threads[i];
7989 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7990 if (!t->selectors) {
7995 TAILQ_FOREACH(s, &p->selectors, node) {
7996 struct selector_runtime *r = &t->selectors[s->id];
8000 size = rte_swx_table_selector_mailbox_size_get();
8002 r->mailbox = calloc(1, size);
8009 /* r->group_id_buffer. */
8010 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8012 /* r->selector_buffer. */
8013 r->selector_buffer = s->selector_header ?
8014 &t->structs[s->selector_header->struct_id] :
8015 &t->structs[p->metadata_struct_id];
8017 /* r->member_id_buffer. */
8018 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8025 selector_build_free(p);
8030 selector_free(struct rte_swx_pipeline *p)
8032 selector_build_free(p);
8034 /* Selector tables. */
8036 struct selector *elem;
8038 elem = TAILQ_FIRST(&p->selectors);
8042 TAILQ_REMOVE(&p->selectors, elem, node);
8043 free(elem->selector_fields);
8051 static struct learner *
8052 learner_find(struct rte_swx_pipeline *p, const char *name)
8056 TAILQ_FOREACH(l, &p->learners, node)
8057 if (!strcmp(l->name, name))
8063 static struct learner *
8064 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8066 struct learner *l = NULL;
8068 TAILQ_FOREACH(l, &p->learners, node)
8076 learner_match_fields_check(struct rte_swx_pipeline *p,
8077 struct rte_swx_pipeline_learner_params *params,
8078 struct header **header)
8080 struct header *h0 = NULL;
8081 struct field *hf, *mf;
8084 /* Return if no match fields. */
8085 if (!params->n_fields || !params->field_names)
8088 /* Check that all the match fields either belong to the same header
8089 * or are all meta-data fields.
8091 hf = header_field_parse(p, params->field_names[0], &h0);
8092 mf = metadata_field_parse(p, params->field_names[0]);
8096 for (i = 1; i < params->n_fields; i++)
8100 hf = header_field_parse(p, params->field_names[i], &h);
8101 if (!hf || (h->id != h0->id))
8104 mf = metadata_field_parse(p, params->field_names[i]);
8109 /* Check that there are no duplicated match fields. */
8110 for (i = 0; i < params->n_fields; i++) {
8111 const char *field_name = params->field_names[i];
8114 for (j = i + 1; j < params->n_fields; j++)
8115 if (!strcmp(params->field_names[j], field_name))
8127 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8129 struct struct_type *mst = p->metadata_st, *ast = a->st;
8130 struct field *mf, *af;
8140 /* Check that mf_name is the name of a valid meta-data field. */
8141 CHECK_NAME(mf_name, EINVAL);
8142 mf = metadata_field_parse(p, mf_name);
8145 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8146 * all the action arguments.
8148 mf_pos = mf - mst->fields;
8149 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8151 /* Check that the size of each of the identified meta-data fields matches exactly the size
8152 * of the corresponding action argument.
8154 for (i = 0; i < ast->n_fields; i++) {
8155 mf = &mst->fields[mf_pos + i];
8156 af = &ast->fields[i];
8158 CHECK(mf->n_bits == af->n_bits, EINVAL);
8165 learner_action_learning_check(struct rte_swx_pipeline *p,
8166 struct action *action,
8167 const char **action_names,
8172 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8173 * the action passed as argument to the "learn" instruction) is also enabled for the
8174 * current learner table.
8176 for (i = 0; i < action->n_instructions; i++) {
8177 struct instruction *instr = &action->instructions[i];
8178 uint32_t found = 0, j;
8180 if (instr->type != INSTR_LEARNER_LEARN)
8183 for (j = 0; j < n_actions; j++) {
8186 a = action_find(p, action_names[j]);
8190 if (a->id == instr->learn.action_id)
8202 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8204 struct rte_swx_pipeline_learner_params *params,
8208 struct learner *l = NULL;
8209 struct action *default_action;
8210 struct header *header = NULL;
8211 uint32_t action_data_size_max = 0, i;
8216 CHECK_NAME(name, EINVAL);
8217 CHECK(!table_find(p, name), EEXIST);
8218 CHECK(!selector_find(p, name), EEXIST);
8219 CHECK(!learner_find(p, name), EEXIST);
8221 CHECK(params, EINVAL);
8224 status = learner_match_fields_check(p, params, &header);
8228 /* Action checks. */
8229 CHECK(params->n_actions, EINVAL);
8230 CHECK(params->action_names, EINVAL);
8231 for (i = 0; i < params->n_actions; i++) {
8232 const char *action_name = params->action_names[i];
8234 uint32_t action_data_size;
8235 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8237 CHECK_NAME(action_name, EINVAL);
8239 a = action_find(p, action_name);
8242 status = learner_action_learning_check(p,
8244 params->action_names,
8249 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8250 if (action_data_size > action_data_size_max)
8251 action_data_size_max = action_data_size;
8253 if (params->action_is_for_table_entries)
8254 action_is_for_table_entries = params->action_is_for_table_entries[i];
8255 if (params->action_is_for_default_entry)
8256 action_is_for_default_entry = params->action_is_for_default_entry[i];
8257 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8260 CHECK_NAME(params->default_action_name, EINVAL);
8261 for (i = 0; i < p->n_actions; i++)
8262 if (!strcmp(params->action_names[i],
8263 params->default_action_name))
8265 CHECK(i < params->n_actions, EINVAL);
8266 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8269 default_action = action_find(p, params->default_action_name);
8270 CHECK((default_action->st && params->default_action_data) ||
8271 !params->default_action_data, EINVAL);
8273 /* Any other checks. */
8274 CHECK(size, EINVAL);
8275 CHECK(timeout, EINVAL);
8277 /* Memory allocation. */
8278 l = calloc(1, sizeof(struct learner));
8282 l->fields = calloc(params->n_fields, sizeof(struct field *));
8286 l->actions = calloc(params->n_actions, sizeof(struct action *));
8290 if (action_data_size_max) {
8291 l->default_action_data = calloc(1, action_data_size_max);
8292 if (!l->default_action_data)
8296 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8297 if (!l->action_is_for_table_entries)
8300 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8301 if (!l->action_is_for_default_entry)
8304 /* Node initialization. */
8305 strcpy(l->name, name);
8307 for (i = 0; i < params->n_fields; i++) {
8308 const char *field_name = params->field_names[i];
8310 l->fields[i] = header ?
8311 header_field_parse(p, field_name, NULL) :
8312 metadata_field_parse(p, field_name);
8315 l->n_fields = params->n_fields;
8319 for (i = 0; i < params->n_actions; i++) {
8320 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8322 if (params->action_is_for_table_entries)
8323 action_is_for_table_entries = params->action_is_for_table_entries[i];
8324 if (params->action_is_for_default_entry)
8325 action_is_for_default_entry = params->action_is_for_default_entry[i];
8327 l->actions[i] = action_find(p, params->action_names[i]);
8328 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8329 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8332 l->default_action = default_action;
8334 if (default_action->st)
8335 memcpy(l->default_action_data,
8336 params->default_action_data,
8337 default_action->st->n_bits / 8);
8339 l->n_actions = params->n_actions;
8341 l->default_action_is_const = params->default_action_is_const;
8343 l->action_data_size_max = action_data_size_max;
8347 l->timeout = timeout;
8349 l->id = p->n_learners;
8351 /* Node add to tailq. */
8352 TAILQ_INSERT_TAIL(&p->learners, l, node);
8361 free(l->action_is_for_default_entry);
8362 free(l->action_is_for_table_entries);
8363 free(l->default_action_data);
8372 learner_params_free(struct rte_swx_table_learner_params *params)
8377 free(params->key_mask0);
8382 static struct rte_swx_table_learner_params *
8383 learner_params_get(struct learner *l)
8385 struct rte_swx_table_learner_params *params = NULL;
8386 struct field *first, *last;
8389 /* Memory allocation. */
8390 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8394 /* Find first (smallest offset) and last (biggest offset) match fields. */
8395 first = l->fields[0];
8396 last = l->fields[0];
8398 for (i = 0; i < l->n_fields; i++) {
8399 struct field *f = l->fields[i];
8401 if (f->offset < first->offset)
8404 if (f->offset > last->offset)
8408 /* Key offset and size. */
8409 params->key_offset = first->offset / 8;
8410 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8412 /* Memory allocation. */
8413 params->key_mask0 = calloc(1, params->key_size);
8414 if (!params->key_mask0)
8418 for (i = 0; i < l->n_fields; i++) {
8419 struct field *f = l->fields[i];
8420 uint32_t start = (f->offset - first->offset) / 8;
8421 size_t size = f->n_bits / 8;
8423 memset(¶ms->key_mask0[start], 0xFF, size);
8426 /* Action data size. */
8427 params->action_data_size = l->action_data_size_max;
8429 /* Maximum number of keys. */
8430 params->n_keys_max = l->size;
8433 params->key_timeout = l->timeout;
8438 learner_params_free(params);
8443 learner_build_free(struct rte_swx_pipeline *p)
8447 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8448 struct thread *t = &p->threads[i];
8454 for (j = 0; j < p->n_learners; j++) {
8455 struct learner_runtime *r = &t->learners[j];
8464 if (p->learner_stats) {
8465 for (i = 0; i < p->n_learners; i++)
8466 free(p->learner_stats[i].n_pkts_action);
8468 free(p->learner_stats);
8473 learner_build(struct rte_swx_pipeline *p)
8478 /* Per pipeline: learner statistics. */
8479 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8480 CHECK(p->learner_stats, ENOMEM);
8482 for (i = 0; i < p->n_learners; i++) {
8483 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8484 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8487 /* Per thread: learner run-time. */
8488 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8489 struct thread *t = &p->threads[i];
8492 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8498 TAILQ_FOREACH(l, &p->learners, node) {
8499 struct learner_runtime *r = &t->learners[l->id];
8503 size = rte_swx_table_learner_mailbox_size_get();
8505 r->mailbox = calloc(1, size);
8513 r->key = l->header ?
8514 &t->structs[l->header->struct_id] :
8515 &t->structs[p->metadata_struct_id];
8522 learner_build_free(p);
8527 learner_free(struct rte_swx_pipeline *p)
8529 learner_build_free(p);
8531 /* Learner tables. */
8535 l = TAILQ_FIRST(&p->learners);
8539 TAILQ_REMOVE(&p->learners, l, node);
8542 free(l->default_action_data);
8551 table_state_build(struct rte_swx_pipeline *p)
8553 struct table *table;
8557 p->table_state = calloc(p->n_tables + p->n_selectors,
8558 sizeof(struct rte_swx_table_state));
8559 CHECK(p->table_state, ENOMEM);
8561 TAILQ_FOREACH(table, &p->tables, node) {
8562 struct rte_swx_table_state *ts = &p->table_state[table->id];
8565 struct rte_swx_table_params *params;
8568 params = table_params_get(table);
8569 CHECK(params, ENOMEM);
8571 ts->obj = table->type->ops.create(params,
8576 table_params_free(params);
8577 CHECK(ts->obj, ENODEV);
8580 /* ts->default_action_data. */
8581 if (table->action_data_size_max) {
8582 ts->default_action_data =
8583 malloc(table->action_data_size_max);
8584 CHECK(ts->default_action_data, ENOMEM);
8586 memcpy(ts->default_action_data,
8587 table->default_action_data,
8588 table->action_data_size_max);
8591 /* ts->default_action_id. */
8592 ts->default_action_id = table->default_action->id;
8595 TAILQ_FOREACH(s, &p->selectors, node) {
8596 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8597 struct rte_swx_table_selector_params *params;
8600 params = selector_table_params_get(s);
8601 CHECK(params, ENOMEM);
8603 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8605 selector_params_free(params);
8606 CHECK(ts->obj, ENODEV);
8609 TAILQ_FOREACH(l, &p->learners, node) {
8610 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8611 p->n_selectors + l->id];
8612 struct rte_swx_table_learner_params *params;
8615 params = learner_params_get(l);
8616 CHECK(params, ENOMEM);
8618 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8619 learner_params_free(params);
8620 CHECK(ts->obj, ENODEV);
8622 /* ts->default_action_data. */
8623 if (l->action_data_size_max) {
8624 ts->default_action_data = malloc(l->action_data_size_max);
8625 CHECK(ts->default_action_data, ENOMEM);
8627 memcpy(ts->default_action_data,
8628 l->default_action_data,
8629 l->action_data_size_max);
8632 /* ts->default_action_id. */
8633 ts->default_action_id = l->default_action->id;
8640 table_state_build_free(struct rte_swx_pipeline *p)
8644 if (!p->table_state)
8647 for (i = 0; i < p->n_tables; i++) {
8648 struct rte_swx_table_state *ts = &p->table_state[i];
8649 struct table *table = table_find_by_id(p, i);
8652 if (table->type && ts->obj)
8653 table->type->ops.free(ts->obj);
8655 /* ts->default_action_data. */
8656 free(ts->default_action_data);
8659 for (i = 0; i < p->n_selectors; i++) {
8660 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8664 rte_swx_table_selector_free(ts->obj);
8667 for (i = 0; i < p->n_learners; i++) {
8668 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8672 rte_swx_table_learner_free(ts->obj);
8674 /* ts->default_action_data. */
8675 free(ts->default_action_data);
8678 free(p->table_state);
8679 p->table_state = NULL;
8683 table_state_free(struct rte_swx_pipeline *p)
8685 table_state_build_free(p);
8691 static struct regarray *
8692 regarray_find(struct rte_swx_pipeline *p, const char *name)
8694 struct regarray *elem;
8696 TAILQ_FOREACH(elem, &p->regarrays, node)
8697 if (!strcmp(elem->name, name))
8703 static struct regarray *
8704 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8706 struct regarray *elem = NULL;
8708 TAILQ_FOREACH(elem, &p->regarrays, node)
8716 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8725 CHECK_NAME(name, EINVAL);
8726 CHECK(!regarray_find(p, name), EEXIST);
8728 CHECK(size, EINVAL);
8729 size = rte_align32pow2(size);
8731 /* Memory allocation. */
8732 r = calloc(1, sizeof(struct regarray));
8735 /* Node initialization. */
8736 strcpy(r->name, name);
8737 r->init_val = init_val;
8739 r->id = p->n_regarrays;
8741 /* Node add to tailq. */
8742 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8749 regarray_build(struct rte_swx_pipeline *p)
8751 struct regarray *regarray;
8753 if (!p->n_regarrays)
8756 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8757 CHECK(p->regarray_runtime, ENOMEM);
8759 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8760 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8763 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8764 RTE_CACHE_LINE_SIZE,
8766 CHECK(r->regarray, ENOMEM);
8768 if (regarray->init_val)
8769 for (i = 0; i < regarray->size; i++)
8770 r->regarray[i] = regarray->init_val;
8772 r->size_mask = regarray->size - 1;
8779 regarray_build_free(struct rte_swx_pipeline *p)
8783 if (!p->regarray_runtime)
8786 for (i = 0; i < p->n_regarrays; i++) {
8787 struct regarray *regarray = regarray_find_by_id(p, i);
8788 struct regarray_runtime *r = &p->regarray_runtime[i];
8790 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8793 free(p->regarray_runtime);
8794 p->regarray_runtime = NULL;
8798 regarray_free(struct rte_swx_pipeline *p)
8800 regarray_build_free(p);
8803 struct regarray *elem;
8805 elem = TAILQ_FIRST(&p->regarrays);
8809 TAILQ_REMOVE(&p->regarrays, elem, node);
8817 static struct meter_profile *
8818 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8820 struct meter_profile *elem;
8822 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8823 if (!strcmp(elem->name, name))
8829 static struct metarray *
8830 metarray_find(struct rte_swx_pipeline *p, const char *name)
8832 struct metarray *elem;
8834 TAILQ_FOREACH(elem, &p->metarrays, node)
8835 if (!strcmp(elem->name, name))
8841 static struct metarray *
8842 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8844 struct metarray *elem = NULL;
8846 TAILQ_FOREACH(elem, &p->metarrays, node)
8854 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8862 CHECK_NAME(name, EINVAL);
8863 CHECK(!metarray_find(p, name), EEXIST);
8865 CHECK(size, EINVAL);
8866 size = rte_align32pow2(size);
8868 /* Memory allocation. */
8869 m = calloc(1, sizeof(struct metarray));
8872 /* Node initialization. */
8873 strcpy(m->name, name);
8875 m->id = p->n_metarrays;
8877 /* Node add to tailq. */
8878 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8884 struct meter_profile meter_profile_default = {
8893 .cir_bytes_per_period = 1,
8895 .pir_bytes_per_period = 1,
8902 meter_init(struct meter *m)
8904 memset(m, 0, sizeof(struct meter));
8905 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8906 m->profile = &meter_profile_default;
8907 m->color_mask = RTE_COLOR_GREEN;
8909 meter_profile_default.n_users++;
8913 metarray_build(struct rte_swx_pipeline *p)
8917 if (!p->n_metarrays)
8920 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8921 CHECK(p->metarray_runtime, ENOMEM);
8923 TAILQ_FOREACH(m, &p->metarrays, node) {
8924 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8927 r->metarray = env_malloc(m->size * sizeof(struct meter),
8928 RTE_CACHE_LINE_SIZE,
8930 CHECK(r->metarray, ENOMEM);
8932 for (i = 0; i < m->size; i++)
8933 meter_init(&r->metarray[i]);
8935 r->size_mask = m->size - 1;
8942 metarray_build_free(struct rte_swx_pipeline *p)
8946 if (!p->metarray_runtime)
8949 for (i = 0; i < p->n_metarrays; i++) {
8950 struct metarray *m = metarray_find_by_id(p, i);
8951 struct metarray_runtime *r = &p->metarray_runtime[i];
8953 env_free(r->metarray, m->size * sizeof(struct meter));
8956 free(p->metarray_runtime);
8957 p->metarray_runtime = NULL;
8961 metarray_free(struct rte_swx_pipeline *p)
8963 metarray_build_free(p);
8967 struct metarray *elem;
8969 elem = TAILQ_FIRST(&p->metarrays);
8973 TAILQ_REMOVE(&p->metarrays, elem, node);
8977 /* Meter profiles. */
8979 struct meter_profile *elem;
8981 elem = TAILQ_FIRST(&p->meter_profiles);
8985 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8994 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9003 free(p->instruction_data);
9004 free(p->instructions);
9008 table_state_free(p);
9013 instruction_table_free(p);
9016 extern_func_free(p);
9029 port_in_types_register(struct rte_swx_pipeline *p)
9033 status = rte_swx_pipeline_port_in_type_register(p,
9035 &rte_swx_port_ethdev_reader_ops);
9039 status = rte_swx_pipeline_port_in_type_register(p,
9041 &rte_swx_port_ring_reader_ops);
9045 #ifdef RTE_PORT_PCAP
9046 status = rte_swx_pipeline_port_in_type_register(p,
9048 &rte_swx_port_source_ops);
9053 status = rte_swx_pipeline_port_in_type_register(p,
9055 &rte_swx_port_fd_reader_ops);
9063 port_out_types_register(struct rte_swx_pipeline *p)
9067 status = rte_swx_pipeline_port_out_type_register(p,
9069 &rte_swx_port_ethdev_writer_ops);
9073 status = rte_swx_pipeline_port_out_type_register(p,
9075 &rte_swx_port_ring_writer_ops);
9079 status = rte_swx_pipeline_port_out_type_register(p,
9081 &rte_swx_port_sink_ops);
9085 status = rte_swx_pipeline_port_out_type_register(p,
9087 &rte_swx_port_fd_writer_ops);
9095 table_types_register(struct rte_swx_pipeline *p)
9099 status = rte_swx_pipeline_table_type_register(p,
9101 RTE_SWX_TABLE_MATCH_EXACT,
9102 &rte_swx_table_exact_match_ops);
9106 status = rte_swx_pipeline_table_type_register(p,
9108 RTE_SWX_TABLE_MATCH_WILDCARD,
9109 &rte_swx_table_wildcard_match_ops);
9117 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9119 struct rte_swx_pipeline *pipeline = NULL;
9122 /* Check input parameters. */
9125 /* Memory allocation. */
9126 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9132 /* Initialization. */
9133 TAILQ_INIT(&pipeline->struct_types);
9134 TAILQ_INIT(&pipeline->port_in_types);
9135 TAILQ_INIT(&pipeline->ports_in);
9136 TAILQ_INIT(&pipeline->port_out_types);
9137 TAILQ_INIT(&pipeline->ports_out);
9138 TAILQ_INIT(&pipeline->extern_types);
9139 TAILQ_INIT(&pipeline->extern_objs);
9140 TAILQ_INIT(&pipeline->extern_funcs);
9141 TAILQ_INIT(&pipeline->headers);
9142 TAILQ_INIT(&pipeline->actions);
9143 TAILQ_INIT(&pipeline->table_types);
9144 TAILQ_INIT(&pipeline->tables);
9145 TAILQ_INIT(&pipeline->selectors);
9146 TAILQ_INIT(&pipeline->learners);
9147 TAILQ_INIT(&pipeline->regarrays);
9148 TAILQ_INIT(&pipeline->meter_profiles);
9149 TAILQ_INIT(&pipeline->metarrays);
9151 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9152 pipeline->numa_node = numa_node;
9154 status = port_in_types_register(pipeline);
9158 status = port_out_types_register(pipeline);
9162 status = table_types_register(pipeline);
9170 rte_swx_pipeline_free(pipeline);
9175 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9176 const char **instructions,
9177 uint32_t n_instructions)
9182 err = instruction_config(p, NULL, instructions, n_instructions);
9186 /* Thread instruction pointer reset. */
9187 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9188 struct thread *t = &p->threads[i];
9190 thread_ip_reset(p, t);
9197 pipeline_compile(struct rte_swx_pipeline *p);
9200 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9202 struct rte_swx_port_sink_params drop_port_params = {
9208 CHECK(p->build_done == 0, EEXIST);
9210 status = port_in_build(p);
9215 status = rte_swx_pipeline_port_out_config(p,
9222 status = port_out_build(p);
9226 status = struct_build(p);
9230 status = extern_obj_build(p);
9234 status = extern_func_build(p);
9238 status = header_build(p);
9242 status = metadata_build(p);
9246 status = instruction_table_build(p);
9250 status = action_build(p);
9254 status = table_build(p);
9258 status = selector_build(p);
9262 status = learner_build(p);
9266 status = table_state_build(p);
9270 status = regarray_build(p);
9274 status = metarray_build(p);
9280 pipeline_compile(p);
9285 metarray_build_free(p);
9286 regarray_build_free(p);
9287 table_state_build_free(p);
9288 learner_build_free(p);
9289 selector_build_free(p);
9290 table_build_free(p);
9291 action_build_free(p);
9292 instruction_table_build_free(p);
9293 metadata_build_free(p);
9294 header_build_free(p);
9295 extern_func_build_free(p);
9296 extern_obj_build_free(p);
9297 port_out_build_free(p);
9298 port_in_build_free(p);
9299 struct_build_free(p);
9305 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9309 for (i = 0; i < n_instructions; i++)
9314 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9318 for (i = 0; i < p->n_ports_out; i++) {
9319 struct port_out_runtime *port = &p->out[i];
9322 port->flush(port->obj);
9330 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9331 struct rte_swx_ctl_pipeline_info *pipeline)
9333 struct action *action;
9334 struct table *table;
9335 uint32_t n_actions = 0, n_tables = 0;
9337 if (!p || !pipeline)
9340 TAILQ_FOREACH(action, &p->actions, node)
9343 TAILQ_FOREACH(table, &p->tables, node)
9346 pipeline->n_ports_in = p->n_ports_in;
9347 pipeline->n_ports_out = p->n_ports_out;
9348 pipeline->n_actions = n_actions;
9349 pipeline->n_tables = n_tables;
9350 pipeline->n_selectors = p->n_selectors;
9351 pipeline->n_learners = p->n_learners;
9352 pipeline->n_regarrays = p->n_regarrays;
9353 pipeline->n_metarrays = p->n_metarrays;
9359 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9361 if (!p || !numa_node)
9364 *numa_node = p->numa_node;
9369 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9371 struct rte_swx_ctl_action_info *action)
9373 struct action *a = NULL;
9375 if (!p || (action_id >= p->n_actions) || !action)
9378 a = action_find_by_id(p, action_id);
9382 strcpy(action->name, a->name);
9383 action->n_args = a->st ? a->st->n_fields : 0;
9388 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9390 uint32_t action_arg_id,
9391 struct rte_swx_ctl_action_arg_info *action_arg)
9393 struct action *a = NULL;
9394 struct field *arg = NULL;
9396 if (!p || (action_id >= p->n_actions) || !action_arg)
9399 a = action_find_by_id(p, action_id);
9400 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9403 arg = &a->st->fields[action_arg_id];
9404 strcpy(action_arg->name, arg->name);
9405 action_arg->n_bits = arg->n_bits;
9406 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9412 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9414 struct rte_swx_ctl_table_info *table)
9416 struct table *t = NULL;
9421 t = table_find_by_id(p, table_id);
9425 strcpy(table->name, t->name);
9426 strcpy(table->args, t->args);
9427 table->n_match_fields = t->n_fields;
9428 table->n_actions = t->n_actions;
9429 table->default_action_is_const = t->default_action_is_const;
9430 table->size = t->size;
9435 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9437 uint32_t match_field_id,
9438 struct rte_swx_ctl_table_match_field_info *match_field)
9441 struct match_field *f;
9443 if (!p || (table_id >= p->n_tables) || !match_field)
9446 t = table_find_by_id(p, table_id);
9447 if (!t || (match_field_id >= t->n_fields))
9450 f = &t->fields[match_field_id];
9451 match_field->match_type = f->match_type;
9452 match_field->is_header = t->header ? 1 : 0;
9453 match_field->n_bits = f->field->n_bits;
9454 match_field->offset = f->field->offset;
9460 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9462 uint32_t table_action_id,
9463 struct rte_swx_ctl_table_action_info *table_action)
9467 if (!p || (table_id >= p->n_tables) || !table_action)
9470 t = table_find_by_id(p, table_id);
9471 if (!t || (table_action_id >= t->n_actions))
9474 table_action->action_id = t->actions[table_action_id]->id;
9476 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9477 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9483 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9485 struct rte_swx_table_ops *table_ops,
9490 if (!p || (table_id >= p->n_tables))
9493 t = table_find_by_id(p, table_id);
9499 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9509 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9510 uint32_t selector_id,
9511 struct rte_swx_ctl_selector_info *selector)
9513 struct selector *s = NULL;
9515 if (!p || !selector)
9518 s = selector_find_by_id(p, selector_id);
9522 strcpy(selector->name, s->name);
9524 selector->n_selector_fields = s->n_selector_fields;
9525 selector->n_groups_max = s->n_groups_max;
9526 selector->n_members_per_group_max = s->n_members_per_group_max;
9532 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9533 uint32_t selector_id,
9534 struct rte_swx_ctl_table_match_field_info *field)
9538 if (!p || (selector_id >= p->n_selectors) || !field)
9541 s = selector_find_by_id(p, selector_id);
9545 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9546 field->is_header = 0;
9547 field->n_bits = s->group_id_field->n_bits;
9548 field->offset = s->group_id_field->offset;
9554 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9555 uint32_t selector_id,
9556 uint32_t selector_field_id,
9557 struct rte_swx_ctl_table_match_field_info *field)
9562 if (!p || (selector_id >= p->n_selectors) || !field)
9565 s = selector_find_by_id(p, selector_id);
9566 if (!s || (selector_field_id >= s->n_selector_fields))
9569 f = s->selector_fields[selector_field_id];
9570 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9571 field->is_header = s->selector_header ? 1 : 0;
9572 field->n_bits = f->n_bits;
9573 field->offset = f->offset;
9579 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9580 uint32_t selector_id,
9581 struct rte_swx_ctl_table_match_field_info *field)
9585 if (!p || (selector_id >= p->n_selectors) || !field)
9588 s = selector_find_by_id(p, selector_id);
9592 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9593 field->is_header = 0;
9594 field->n_bits = s->member_id_field->n_bits;
9595 field->offset = s->member_id_field->offset;
9601 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9602 uint32_t learner_id,
9603 struct rte_swx_ctl_learner_info *learner)
9605 struct learner *l = NULL;
9610 l = learner_find_by_id(p, learner_id);
9614 strcpy(learner->name, l->name);
9616 learner->n_match_fields = l->n_fields;
9617 learner->n_actions = l->n_actions;
9618 learner->default_action_is_const = l->default_action_is_const;
9619 learner->size = l->size;
9625 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9626 uint32_t learner_id,
9627 uint32_t match_field_id,
9628 struct rte_swx_ctl_table_match_field_info *match_field)
9633 if (!p || (learner_id >= p->n_learners) || !match_field)
9636 l = learner_find_by_id(p, learner_id);
9637 if (!l || (match_field_id >= l->n_fields))
9640 f = l->fields[match_field_id];
9641 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9642 match_field->is_header = l->header ? 1 : 0;
9643 match_field->n_bits = f->n_bits;
9644 match_field->offset = f->offset;
9650 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9651 uint32_t learner_id,
9652 uint32_t learner_action_id,
9653 struct rte_swx_ctl_table_action_info *learner_action)
9657 if (!p || (learner_id >= p->n_learners) || !learner_action)
9660 l = learner_find_by_id(p, learner_id);
9661 if (!l || (learner_action_id >= l->n_actions))
9664 learner_action->action_id = l->actions[learner_action_id]->id;
9666 learner_action->action_is_for_table_entries =
9667 l->action_is_for_table_entries[learner_action_id];
9669 learner_action->action_is_for_default_entry =
9670 l->action_is_for_default_entry[learner_action_id];
9676 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9677 struct rte_swx_table_state **table_state)
9679 if (!p || !table_state || !p->build_done)
9682 *table_state = p->table_state;
9687 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9688 struct rte_swx_table_state *table_state)
9690 if (!p || !table_state || !p->build_done)
9693 p->table_state = table_state;
9698 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9700 struct rte_swx_port_in_stats *stats)
9702 struct port_in *port;
9707 port = port_in_find(p, port_id);
9711 port->type->ops.stats_read(port->obj, stats);
9716 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9718 struct rte_swx_port_out_stats *stats)
9720 struct port_out *port;
9725 port = port_out_find(p, port_id);
9729 port->type->ops.stats_read(port->obj, stats);
9734 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9735 const char *table_name,
9736 struct rte_swx_table_stats *stats)
9738 struct table *table;
9739 struct table_statistics *table_stats;
9741 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9744 table = table_find(p, table_name);
9748 table_stats = &p->table_stats[table->id];
9750 memcpy(stats->n_pkts_action,
9751 table_stats->n_pkts_action,
9752 p->n_actions * sizeof(uint64_t));
9754 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9755 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9761 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9762 const char *selector_name,
9763 struct rte_swx_pipeline_selector_stats *stats)
9767 if (!p || !selector_name || !selector_name[0] || !stats)
9770 s = selector_find(p, selector_name);
9774 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9780 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9781 const char *learner_name,
9782 struct rte_swx_learner_stats *stats)
9785 struct learner_statistics *learner_stats;
9787 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9790 l = learner_find(p, learner_name);
9794 learner_stats = &p->learner_stats[l->id];
9796 memcpy(stats->n_pkts_action,
9797 learner_stats->n_pkts_action,
9798 p->n_actions * sizeof(uint64_t));
9800 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9801 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9803 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9804 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9806 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9812 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9813 uint32_t regarray_id,
9814 struct rte_swx_ctl_regarray_info *regarray)
9818 if (!p || !regarray)
9821 r = regarray_find_by_id(p, regarray_id);
9825 strcpy(regarray->name, r->name);
9826 regarray->size = r->size;
9831 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9832 const char *regarray_name,
9833 uint32_t regarray_index,
9836 struct regarray *regarray;
9837 struct regarray_runtime *r;
9839 if (!p || !regarray_name || !value)
9842 regarray = regarray_find(p, regarray_name);
9843 if (!regarray || (regarray_index >= regarray->size))
9846 r = &p->regarray_runtime[regarray->id];
9847 *value = r->regarray[regarray_index];
9852 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9853 const char *regarray_name,
9854 uint32_t regarray_index,
9857 struct regarray *regarray;
9858 struct regarray_runtime *r;
9860 if (!p || !regarray_name)
9863 regarray = regarray_find(p, regarray_name);
9864 if (!regarray || (regarray_index >= regarray->size))
9867 r = &p->regarray_runtime[regarray->id];
9868 r->regarray[regarray_index] = value;
9873 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9874 uint32_t metarray_id,
9875 struct rte_swx_ctl_metarray_info *metarray)
9879 if (!p || !metarray)
9882 m = metarray_find_by_id(p, metarray_id);
9886 strcpy(metarray->name, m->name);
9887 metarray->size = m->size;
9892 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9894 struct rte_meter_trtcm_params *params)
9896 struct meter_profile *mp;
9900 CHECK_NAME(name, EINVAL);
9901 CHECK(params, EINVAL);
9902 CHECK(!meter_profile_find(p, name), EEXIST);
9904 /* Node allocation. */
9905 mp = calloc(1, sizeof(struct meter_profile));
9908 /* Node initialization. */
9909 strcpy(mp->name, name);
9910 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9911 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9917 /* Node add to tailq. */
9918 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9924 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9927 struct meter_profile *mp;
9930 CHECK_NAME(name, EINVAL);
9932 mp = meter_profile_find(p, name);
9934 CHECK(!mp->n_users, EBUSY);
9936 /* Remove node from tailq. */
9937 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9944 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9945 const char *metarray_name,
9946 uint32_t metarray_index)
9948 struct meter_profile *mp_old;
9949 struct metarray *metarray;
9950 struct metarray_runtime *metarray_runtime;
9954 CHECK_NAME(metarray_name, EINVAL);
9956 metarray = metarray_find(p, metarray_name);
9957 CHECK(metarray, EINVAL);
9958 CHECK(metarray_index < metarray->size, EINVAL);
9960 metarray_runtime = &p->metarray_runtime[metarray->id];
9961 m = &metarray_runtime->metarray[metarray_index];
9962 mp_old = m->profile;
9972 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9973 const char *metarray_name,
9974 uint32_t metarray_index,
9975 const char *profile_name)
9977 struct meter_profile *mp, *mp_old;
9978 struct metarray *metarray;
9979 struct metarray_runtime *metarray_runtime;
9983 CHECK_NAME(metarray_name, EINVAL);
9985 metarray = metarray_find(p, metarray_name);
9986 CHECK(metarray, EINVAL);
9987 CHECK(metarray_index < metarray->size, EINVAL);
9989 mp = meter_profile_find(p, profile_name);
9992 metarray_runtime = &p->metarray_runtime[metarray->id];
9993 m = &metarray_runtime->metarray[metarray_index];
9994 mp_old = m->profile;
9996 memset(m, 0, sizeof(struct meter));
9997 rte_meter_trtcm_config(&m->m, &mp->profile);
9999 m->color_mask = RTE_COLORS;
10008 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10009 const char *metarray_name,
10010 uint32_t metarray_index,
10011 struct rte_swx_ctl_meter_stats *stats)
10013 struct metarray *metarray;
10014 struct metarray_runtime *metarray_runtime;
10018 CHECK_NAME(metarray_name, EINVAL);
10020 metarray = metarray_find(p, metarray_name);
10021 CHECK(metarray, EINVAL);
10022 CHECK(metarray_index < metarray->size, EINVAL);
10024 CHECK(stats, EINVAL);
10026 metarray_runtime = &p->metarray_runtime[metarray->id];
10027 m = &metarray_runtime->metarray[metarray_index];
10029 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10030 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10036 * Pipeline compilation.
10038 static const char *
10039 instr_type_to_name(struct instruction *instr)
10041 switch (instr->type) {
10042 case INSTR_RX: return "INSTR_RX";
10044 case INSTR_TX: return "INSTR_TX";
10045 case INSTR_TX_I: return "INSTR_TX_I";
10046 case INSTR_DROP: return "INSTR_DROP";
10048 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10049 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10050 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10051 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10052 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10053 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10054 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10055 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10057 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10059 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10061 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10062 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10063 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10064 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10065 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10066 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10067 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10068 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10069 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10071 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10072 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10074 case INSTR_MOV: return "INSTR_MOV";
10075 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10076 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10077 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10078 case INSTR_MOV_I: return "INSTR_MOV_I";
10080 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10081 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10082 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10083 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10084 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10085 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10086 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10087 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10089 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10090 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10091 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10092 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10093 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10094 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10096 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10097 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10098 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10099 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10100 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10101 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10103 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10104 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10105 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10106 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10108 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10109 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10110 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10111 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10112 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10114 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10115 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10116 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10117 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10118 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10120 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10121 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10122 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10123 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10124 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10126 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10127 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10128 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10129 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10130 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10131 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10133 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10134 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10135 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10136 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10137 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10138 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10140 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10141 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10142 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10144 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10145 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10146 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10147 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10148 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10149 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10151 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10152 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10153 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10154 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10155 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10156 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10157 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10158 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10159 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10161 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10162 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10163 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10164 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10165 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10166 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10167 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10168 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10169 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10171 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10172 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10173 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10175 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10176 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10177 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10178 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10179 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10180 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10181 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10182 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10183 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10184 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10185 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10186 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10188 case INSTR_TABLE: return "INSTR_TABLE";
10189 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10190 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10191 case INSTR_LEARNER: return "INSTR_LEARNER";
10192 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10194 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10195 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10197 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10198 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10200 case INSTR_JMP: return "INSTR_JMP";
10201 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10202 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10203 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10204 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10205 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10206 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10207 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10208 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10209 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10210 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10211 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10212 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10213 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10214 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10215 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10216 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10217 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10218 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10219 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10220 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10221 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10222 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10223 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10224 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10225 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10226 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10227 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10228 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10230 case INSTR_RETURN: return "INSTR_RETURN";
10232 default: return "INSTR_UNKNOWN";
10237 (*instruction_export_t)(struct instruction *, FILE *);
10240 instr_io_export(struct instruction *instr, FILE *f)
10242 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10244 /* n_io, n_io_imm, n_hdrs. */
10245 if (instr->type == INSTR_RX ||
10246 instr->type == INSTR_TX ||
10247 instr->type == INSTR_HDR_EXTRACT_M ||
10248 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10251 if (instr->type == INSTR_TX_I)
10254 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10255 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10257 if (instr->type == INSTR_HDR_EXTRACT_M ||
10258 instr->type == INSTR_HDR_LOOKAHEAD ||
10259 instr->type == INSTR_HDR_EMIT)
10262 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10263 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10268 "\t\t.type = %s,\n",
10269 instr_type_to_name(instr));
10272 if (n_io || n_io_imm || n_hdrs)
10280 "\t\t\t\t.offset = %u,\n"
10281 "\t\t\t\t.n_bits = %u,\n"
10283 instr->io.io.offset,
10284 instr->io.io.n_bits);
10289 "\t\t\t\t.val = %u,\n"
10293 /* instr.io.hdr. */
10298 /* instr.io.hdr.header_id. */
10300 "\t\t\t.header_id = {");
10302 for (i = 0; i < n_hdrs; i++)
10305 instr->io.hdr.header_id[i]);
10310 /* instr.io.hdr.struct_id. */
10312 "\t\t\t.struct_id = {");
10314 for (i = 0; i < n_hdrs; i++)
10317 instr->io.hdr.struct_id[i]);
10322 /* instr.io.hdr.n_bytes. */
10324 "\t\t\t.n_bytes = {");
10326 for (i = 0; i < n_hdrs; i++)
10329 instr->io.hdr.n_bytes[i]);
10334 /* instr.io.hdr - closing curly brace. */
10339 /* instr.io - closing curly brace. */
10340 if (n_io || n_io_imm || n_hdrs)
10344 /* instr - closing curly brace. */
10350 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10354 "\t\t.type = %s,\n"
10356 "\t\t\t.header_id = %u,\n"
10359 instr_type_to_name(instr),
10360 instr->valid.header_id);
10364 instr_mov_export(struct instruction *instr, FILE *f)
10366 if (instr->type != INSTR_MOV_I)
10369 "\t\t.type = %s,\n"
10372 "\t\t\t\t.struct_id = %u,\n"
10373 "\t\t\t\t.n_bits = %u,\n"
10374 "\t\t\t\t.offset = %u,\n"
10377 "\t\t\t\t.struct_id = %u,\n"
10378 "\t\t\t\t.n_bits = %u,\n"
10379 "\t\t\t\t.offset = %u,\n"
10383 instr_type_to_name(instr),
10384 instr->mov.dst.struct_id,
10385 instr->mov.dst.n_bits,
10386 instr->mov.dst.offset,
10387 instr->mov.src.struct_id,
10388 instr->mov.src.n_bits,
10389 instr->mov.src.offset);
10393 "\t\t.type = %s,\n"
10396 "\t\t\t\t.struct_id = %u,\n"
10397 "\t\t\t\t.n_bits = %u,\n"
10398 "\t\t\t\t.offset = %u,\n"
10400 "\t\t\t.src_val = %" PRIu64 ",\n"
10403 instr_type_to_name(instr),
10404 instr->mov.dst.struct_id,
10405 instr->mov.dst.n_bits,
10406 instr->mov.dst.offset,
10407 instr->mov.src_val);
10411 instr_dma_ht_export(struct instruction *instr, FILE *f)
10413 uint32_t n_dma = 0, i;
10416 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10421 "\t\t.type = %s,\n",
10422 instr_type_to_name(instr));
10428 /* instr.dma.dst. */
10430 "\t\t\t.dst = {\n");
10432 /* instr.dma.dst.header_id. */
10434 "\t\t\t\t.header_id = {");
10436 for (i = 0; i < n_dma; i++)
10439 instr->dma.dst.header_id[i]);
10444 /* instr.dma.dst.struct_id. */
10446 "\t\t\t\t.struct_id = {");
10448 for (i = 0; i < n_dma; i++)
10451 instr->dma.dst.struct_id[i]);
10456 /* instr.dma.dst - closing curly brace. */
10460 /* instr.dma.src. */
10462 "\t\t\t.src = {\n");
10464 /* instr.dma.src.offset. */
10466 "\t\t\t\t.offset = {");
10468 for (i = 0; i < n_dma; i++)
10471 instr->dma.src.offset[i]);
10476 /* instr.dma.src - closing curly brace. */
10480 /* instr.dma.n_bytes. */
10482 "\t\t\t.n_bytes = {");
10484 for (i = 0; i < n_dma; i++)
10487 instr->dma.n_bytes[i]);
10492 /* instr.dma - closing curly brace. */
10496 /* instr - closing curly brace. */
10502 instr_alu_export(struct instruction *instr, FILE *f)
10506 if (instr->type == INSTR_ALU_ADD_MI ||
10507 instr->type == INSTR_ALU_ADD_HI ||
10508 instr->type == INSTR_ALU_SUB_MI ||
10509 instr->type == INSTR_ALU_SUB_HI ||
10510 instr->type == INSTR_ALU_SHL_MI ||
10511 instr->type == INSTR_ALU_SHL_HI ||
10512 instr->type == INSTR_ALU_SHR_MI ||
10513 instr->type == INSTR_ALU_SHR_HI ||
10514 instr->type == INSTR_ALU_AND_I ||
10515 instr->type == INSTR_ALU_OR_I ||
10516 instr->type == INSTR_ALU_XOR_I)
10522 "\t\t.type = %s,\n"
10525 "\t\t\t\t.struct_id = %u,\n"
10526 "\t\t\t\t.n_bits = %u,\n"
10527 "\t\t\t\t.offset = %u,\n"
10530 "\t\t\t\t.struct_id = %u,\n"
10531 "\t\t\t\t.n_bits = %u,\n"
10532 "\t\t\t\t.offset = %u,\n"
10536 instr_type_to_name(instr),
10537 instr->alu.dst.struct_id,
10538 instr->alu.dst.n_bits,
10539 instr->alu.dst.offset,
10540 instr->alu.src.struct_id,
10541 instr->alu.src.n_bits,
10542 instr->alu.src.offset);
10546 "\t\t.type = %s,\n"
10549 "\t\t\t\t.struct_id = %u,\n"
10550 "\t\t\t\t.n_bits = %u,\n"
10551 "\t\t\t\t.offset = %u,\n"
10553 "\t\t\t.src_val = %" PRIu64 ",\n"
10556 instr_type_to_name(instr),
10557 instr->alu.dst.struct_id,
10558 instr->alu.dst.n_bits,
10559 instr->alu.dst.offset,
10560 instr->alu.src_val);
10564 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10566 int prefetch = 0, idx_imm = 0, src_imm = 0;
10568 if (instr->type == INSTR_REGPREFETCH_RH ||
10569 instr->type == INSTR_REGPREFETCH_RM ||
10570 instr->type == INSTR_REGPREFETCH_RI)
10573 /* index is the 3rd operand for the regrd instruction and the 2nd
10574 * operand for the regwr and regadd instructions.
10576 if (instr->type == INSTR_REGPREFETCH_RI ||
10577 instr->type == INSTR_REGRD_HRI ||
10578 instr->type == INSTR_REGRD_MRI ||
10579 instr->type == INSTR_REGWR_RIH ||
10580 instr->type == INSTR_REGWR_RIM ||
10581 instr->type == INSTR_REGWR_RII ||
10582 instr->type == INSTR_REGADD_RIH ||
10583 instr->type == INSTR_REGADD_RIM ||
10584 instr->type == INSTR_REGADD_RII)
10587 /* src is the 3rd operand for the regwr and regadd instructions. */
10588 if (instr->type == INSTR_REGWR_RHI ||
10589 instr->type == INSTR_REGWR_RMI ||
10590 instr->type == INSTR_REGWR_RII ||
10591 instr->type == INSTR_REGADD_RHI ||
10592 instr->type == INSTR_REGADD_RMI ||
10593 instr->type == INSTR_REGADD_RII)
10596 /* instr.regarray.regarray_id. */
10599 "\t\t.type = %s,\n"
10600 "\t\t.regarray = {\n"
10601 "\t\t\t.regarray_id = %u,\n",
10602 instr_type_to_name(instr),
10603 instr->regarray.regarray_id);
10605 /* instr.regarray.idx / instr.regarray.idx_val. */
10608 "\t\t\t\t.idx = {\n"
10609 "\t\t\t\t\t.struct_id = %u,\n"
10610 "\t\t\t\t\t.n_bits = %u,\n"
10611 "\t\t\t\t\t.offset = %u,\n"
10613 instr->regarray.idx.struct_id,
10614 instr->regarray.idx.n_bits,
10615 instr->regarray.idx.offset);
10618 "\t\t\t\t.idx_val = %u,\n",
10619 instr->regarray.idx_val);
10621 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10625 "\t\t\t\t.dstsrc = {\n"
10626 "\t\t\t\t\t.struct_id = %u,\n"
10627 "\t\t\t\t\t.n_bits = %u,\n"
10628 "\t\t\t\t\t.offset = %u,\n"
10630 instr->regarray.dstsrc.struct_id,
10631 instr->regarray.dstsrc.n_bits,
10632 instr->regarray.dstsrc.offset);
10635 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10636 instr->regarray.dstsrc_val);
10639 /* instr.regarray and instr - closing curly braces. */
10646 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10648 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10650 if (instr->type == INSTR_METPREFETCH_H ||
10651 instr->type == INSTR_METPREFETCH_M ||
10652 instr->type == INSTR_METPREFETCH_I)
10656 if (instr->type == INSTR_METPREFETCH_I ||
10657 instr->type == INSTR_METER_IHM ||
10658 instr->type == INSTR_METER_IHI ||
10659 instr->type == INSTR_METER_IMM ||
10660 instr->type == INSTR_METER_IMI)
10663 /* color_in_imm. */
10664 if (instr->type == INSTR_METER_HHI ||
10665 instr->type == INSTR_METER_HMI ||
10666 instr->type == INSTR_METER_MHI ||
10667 instr->type == INSTR_METER_MMI ||
10668 instr->type == INSTR_METER_IHI ||
10669 instr->type == INSTR_METER_IMI)
10672 /* instr.meter.metarray_id. */
10675 "\t\t.type = %s,\n"
10677 "\t\t\t.metarray_id = %u,\n",
10678 instr_type_to_name(instr),
10679 instr->meter.metarray_id);
10681 /* instr.meter.idx / instr.meter.idx_val. */
10685 "\t\t\t\t.struct_id = %u,\n"
10686 "\t\t\t\t.n_bits = %u,\n"
10687 "\t\t\t\t.offset = %u,\n"
10689 instr->meter.idx.struct_id,
10690 instr->meter.idx.n_bits,
10691 instr->meter.idx.offset);
10694 "\t\t\t.idx_val = %u,\n",
10695 instr->meter.idx_val);
10698 /* instr.meter.length. */
10700 "\t\t\t.length = {\n"
10701 "\t\t\t\t.struct_id = %u,\n"
10702 "\t\t\t\t.n_bits = %u,\n"
10703 "\t\t\t\t.offset = %u,\n"
10705 instr->meter.length.struct_id,
10706 instr->meter.length.n_bits,
10707 instr->meter.length.offset);
10709 /* instr.meter.color_in / instr.meter.color_in_val. */
10712 "\t\t\t.color_in = {\n"
10713 "\t\t\t\t.struct_id = %u,\n"
10714 "\t\t\t\t.n_bits = %u,\n"
10715 "\t\t\t\t.offset = %u,\n"
10717 instr->meter.color_in.struct_id,
10718 instr->meter.color_in.n_bits,
10719 instr->meter.color_in.offset);
10722 "\t\t\t.color_in_val = %u,\n",
10723 (uint32_t)instr->meter.color_in_val);
10725 /* instr.meter.color_out. */
10727 "\t\t\t.color_out = {\n"
10728 "\t\t\t\t.struct_id = %u,\n"
10729 "\t\t\t\t.n_bits = %u,\n"
10730 "\t\t\t\t.offset = %u,\n"
10732 instr->meter.color_out.struct_id,
10733 instr->meter.color_out.n_bits,
10734 instr->meter.color_out.offset);
10737 /* instr.meter and instr - closing curly braces. */
10744 instr_table_export(struct instruction *instr,
10749 "\t\t.type = %s,\n"
10751 "\t\t\t.table_id = %u,\n"
10754 instr_type_to_name(instr),
10755 instr->table.table_id);
10759 instr_learn_export(struct instruction *instr, FILE *f)
10763 "\t\t.type = %s,\n"
10765 "\t\t\t\t.action_id = %u,\n"
10768 instr_type_to_name(instr),
10769 instr->learn.action_id);
10773 instr_forget_export(struct instruction *instr, FILE *f)
10777 "\t\t.type = %s,\n"
10779 instr_type_to_name(instr));
10783 instr_extern_export(struct instruction *instr, FILE *f)
10785 if (instr->type == INSTR_EXTERN_OBJ)
10788 "\t\t.type = %s,\n"
10789 "\t\t.ext_obj = {\n"
10790 "\t\t\t.ext_obj_id = %u,\n"
10791 "\t\t\t.func_id = %u,\n"
10794 instr_type_to_name(instr),
10795 instr->ext_obj.ext_obj_id,
10796 instr->ext_obj.func_id);
10800 "\t\t.type = %s,\n"
10801 "\t\t.ext_func = {\n"
10802 "\t\t\t.ext_func_id = %u,\n"
10805 instr_type_to_name(instr),
10806 instr->ext_func.ext_func_id);
10810 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10814 "\t\t.type = %s,\n"
10816 "\t\t\t.ip = NULL,\n",
10817 instr_type_to_name(instr));
10819 switch (instr->type) {
10820 case INSTR_JMP_VALID:
10821 case INSTR_JMP_INVALID:
10823 "\t\t\t.header_id = %u,\n",
10824 instr->jmp.header_id);
10827 case INSTR_JMP_ACTION_HIT:
10828 case INSTR_JMP_ACTION_MISS:
10830 "\t\t\t.action_id = %u,\n",
10831 instr->jmp.action_id);
10835 case INSTR_JMP_EQ_MH:
10836 case INSTR_JMP_EQ_HM:
10837 case INSTR_JMP_EQ_HH:
10838 case INSTR_JMP_NEQ:
10839 case INSTR_JMP_NEQ_MH:
10840 case INSTR_JMP_NEQ_HM:
10841 case INSTR_JMP_NEQ_HH:
10843 case INSTR_JMP_LT_MH:
10844 case INSTR_JMP_LT_HM:
10845 case INSTR_JMP_LT_HH:
10847 case INSTR_JMP_GT_MH:
10848 case INSTR_JMP_GT_HM:
10849 case INSTR_JMP_GT_HH:
10852 "\t\t\t\t.struct_id = %u,\n"
10853 "\t\t\t\t.n_bits = %u,\n"
10854 "\t\t\t\t.offset = %u,\n"
10857 "\t\t\t\t.struct_id = %u,\n"
10858 "\t\t\t\t.n_bits = %u,\n"
10859 "\t\t\t\t.offset = %u,\n"
10861 instr->jmp.a.struct_id,
10862 instr->jmp.a.n_bits,
10863 instr->jmp.a.offset,
10864 instr->jmp.b.struct_id,
10865 instr->jmp.b.n_bits,
10866 instr->jmp.b.offset);
10869 case INSTR_JMP_EQ_I:
10870 case INSTR_JMP_NEQ_I:
10871 case INSTR_JMP_LT_MI:
10872 case INSTR_JMP_LT_HI:
10873 case INSTR_JMP_GT_MI:
10874 case INSTR_JMP_GT_HI:
10877 "\t\t\t\t.struct_id = %u,\n"
10878 "\t\t\t\t.n_bits = %u,\n"
10879 "\t\t\t\t.offset = %u,\n"
10881 "\t\t\t.b_val = %" PRIu64 ",\n",
10882 instr->jmp.a.struct_id,
10883 instr->jmp.a.n_bits,
10884 instr->jmp.a.offset,
10898 instr_return_export(struct instruction *instr,
10903 "\t\t.type = %s,\n",
10904 instr_type_to_name(instr));
10910 static instruction_export_t export_table[] = {
10911 [INSTR_RX] = instr_io_export,
10913 [INSTR_TX] = instr_io_export,
10914 [INSTR_TX_I] = instr_io_export,
10915 [INSTR_DROP] = instr_io_export,
10917 [INSTR_HDR_EXTRACT] = instr_io_export,
10918 [INSTR_HDR_EXTRACT2] = instr_io_export,
10919 [INSTR_HDR_EXTRACT3] = instr_io_export,
10920 [INSTR_HDR_EXTRACT4] = instr_io_export,
10921 [INSTR_HDR_EXTRACT5] = instr_io_export,
10922 [INSTR_HDR_EXTRACT6] = instr_io_export,
10923 [INSTR_HDR_EXTRACT7] = instr_io_export,
10924 [INSTR_HDR_EXTRACT8] = instr_io_export,
10926 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10928 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10930 [INSTR_HDR_EMIT] = instr_io_export,
10931 [INSTR_HDR_EMIT_TX] = instr_io_export,
10932 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10933 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10934 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10935 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10936 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10937 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10938 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10940 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10941 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10943 [INSTR_MOV] = instr_mov_export,
10944 [INSTR_MOV_MH] = instr_mov_export,
10945 [INSTR_MOV_HM] = instr_mov_export,
10946 [INSTR_MOV_HH] = instr_mov_export,
10947 [INSTR_MOV_I] = instr_mov_export,
10949 [INSTR_DMA_HT] = instr_dma_ht_export,
10950 [INSTR_DMA_HT2] = instr_dma_ht_export,
10951 [INSTR_DMA_HT3] = instr_dma_ht_export,
10952 [INSTR_DMA_HT4] = instr_dma_ht_export,
10953 [INSTR_DMA_HT5] = instr_dma_ht_export,
10954 [INSTR_DMA_HT6] = instr_dma_ht_export,
10955 [INSTR_DMA_HT7] = instr_dma_ht_export,
10956 [INSTR_DMA_HT8] = instr_dma_ht_export,
10958 [INSTR_ALU_ADD] = instr_alu_export,
10959 [INSTR_ALU_ADD_MH] = instr_alu_export,
10960 [INSTR_ALU_ADD_HM] = instr_alu_export,
10961 [INSTR_ALU_ADD_HH] = instr_alu_export,
10962 [INSTR_ALU_ADD_MI] = instr_alu_export,
10963 [INSTR_ALU_ADD_HI] = instr_alu_export,
10965 [INSTR_ALU_SUB] = instr_alu_export,
10966 [INSTR_ALU_SUB_MH] = instr_alu_export,
10967 [INSTR_ALU_SUB_HM] = instr_alu_export,
10968 [INSTR_ALU_SUB_HH] = instr_alu_export,
10969 [INSTR_ALU_SUB_MI] = instr_alu_export,
10970 [INSTR_ALU_SUB_HI] = instr_alu_export,
10972 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10973 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10974 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10975 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10977 [INSTR_ALU_AND] = instr_alu_export,
10978 [INSTR_ALU_AND_MH] = instr_alu_export,
10979 [INSTR_ALU_AND_HM] = instr_alu_export,
10980 [INSTR_ALU_AND_HH] = instr_alu_export,
10981 [INSTR_ALU_AND_I] = instr_alu_export,
10983 [INSTR_ALU_OR] = instr_alu_export,
10984 [INSTR_ALU_OR_MH] = instr_alu_export,
10985 [INSTR_ALU_OR_HM] = instr_alu_export,
10986 [INSTR_ALU_OR_HH] = instr_alu_export,
10987 [INSTR_ALU_OR_I] = instr_alu_export,
10989 [INSTR_ALU_XOR] = instr_alu_export,
10990 [INSTR_ALU_XOR_MH] = instr_alu_export,
10991 [INSTR_ALU_XOR_HM] = instr_alu_export,
10992 [INSTR_ALU_XOR_HH] = instr_alu_export,
10993 [INSTR_ALU_XOR_I] = instr_alu_export,
10995 [INSTR_ALU_SHL] = instr_alu_export,
10996 [INSTR_ALU_SHL_MH] = instr_alu_export,
10997 [INSTR_ALU_SHL_HM] = instr_alu_export,
10998 [INSTR_ALU_SHL_HH] = instr_alu_export,
10999 [INSTR_ALU_SHL_MI] = instr_alu_export,
11000 [INSTR_ALU_SHL_HI] = instr_alu_export,
11002 [INSTR_ALU_SHR] = instr_alu_export,
11003 [INSTR_ALU_SHR_MH] = instr_alu_export,
11004 [INSTR_ALU_SHR_HM] = instr_alu_export,
11005 [INSTR_ALU_SHR_HH] = instr_alu_export,
11006 [INSTR_ALU_SHR_MI] = instr_alu_export,
11007 [INSTR_ALU_SHR_HI] = instr_alu_export,
11009 [INSTR_REGPREFETCH_RH] = instr_reg_export,
11010 [INSTR_REGPREFETCH_RM] = instr_reg_export,
11011 [INSTR_REGPREFETCH_RI] = instr_reg_export,
11013 [INSTR_REGRD_HRH] = instr_reg_export,
11014 [INSTR_REGRD_HRM] = instr_reg_export,
11015 [INSTR_REGRD_MRH] = instr_reg_export,
11016 [INSTR_REGRD_MRM] = instr_reg_export,
11017 [INSTR_REGRD_HRI] = instr_reg_export,
11018 [INSTR_REGRD_MRI] = instr_reg_export,
11020 [INSTR_REGWR_RHH] = instr_reg_export,
11021 [INSTR_REGWR_RHM] = instr_reg_export,
11022 [INSTR_REGWR_RMH] = instr_reg_export,
11023 [INSTR_REGWR_RMM] = instr_reg_export,
11024 [INSTR_REGWR_RHI] = instr_reg_export,
11025 [INSTR_REGWR_RMI] = instr_reg_export,
11026 [INSTR_REGWR_RIH] = instr_reg_export,
11027 [INSTR_REGWR_RIM] = instr_reg_export,
11028 [INSTR_REGWR_RII] = instr_reg_export,
11030 [INSTR_REGADD_RHH] = instr_reg_export,
11031 [INSTR_REGADD_RHM] = instr_reg_export,
11032 [INSTR_REGADD_RMH] = instr_reg_export,
11033 [INSTR_REGADD_RMM] = instr_reg_export,
11034 [INSTR_REGADD_RHI] = instr_reg_export,
11035 [INSTR_REGADD_RMI] = instr_reg_export,
11036 [INSTR_REGADD_RIH] = instr_reg_export,
11037 [INSTR_REGADD_RIM] = instr_reg_export,
11038 [INSTR_REGADD_RII] = instr_reg_export,
11040 [INSTR_METPREFETCH_H] = instr_meter_export,
11041 [INSTR_METPREFETCH_M] = instr_meter_export,
11042 [INSTR_METPREFETCH_I] = instr_meter_export,
11044 [INSTR_METER_HHM] = instr_meter_export,
11045 [INSTR_METER_HHI] = instr_meter_export,
11046 [INSTR_METER_HMM] = instr_meter_export,
11047 [INSTR_METER_HMI] = instr_meter_export,
11048 [INSTR_METER_MHM] = instr_meter_export,
11049 [INSTR_METER_MHI] = instr_meter_export,
11050 [INSTR_METER_MMM] = instr_meter_export,
11051 [INSTR_METER_MMI] = instr_meter_export,
11052 [INSTR_METER_IHM] = instr_meter_export,
11053 [INSTR_METER_IHI] = instr_meter_export,
11054 [INSTR_METER_IMM] = instr_meter_export,
11055 [INSTR_METER_IMI] = instr_meter_export,
11057 [INSTR_TABLE] = instr_table_export,
11058 [INSTR_TABLE_AF] = instr_table_export,
11059 [INSTR_SELECTOR] = instr_table_export,
11060 [INSTR_LEARNER] = instr_table_export,
11061 [INSTR_LEARNER_AF] = instr_table_export,
11063 [INSTR_LEARNER_LEARN] = instr_learn_export,
11064 [INSTR_LEARNER_FORGET] = instr_forget_export,
11066 [INSTR_EXTERN_OBJ] = instr_extern_export,
11067 [INSTR_EXTERN_FUNC] = instr_extern_export,
11069 [INSTR_JMP] = instr_jmp_export,
11070 [INSTR_JMP_VALID] = instr_jmp_export,
11071 [INSTR_JMP_INVALID] = instr_jmp_export,
11072 [INSTR_JMP_HIT] = instr_jmp_export,
11073 [INSTR_JMP_MISS] = instr_jmp_export,
11074 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11075 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11077 [INSTR_JMP_EQ] = instr_jmp_export,
11078 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11079 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11080 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11081 [INSTR_JMP_EQ_I] = instr_jmp_export,
11083 [INSTR_JMP_NEQ] = instr_jmp_export,
11084 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11085 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11086 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11087 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11089 [INSTR_JMP_LT] = instr_jmp_export,
11090 [INSTR_JMP_LT_MH] = instr_jmp_export,
11091 [INSTR_JMP_LT_HM] = instr_jmp_export,
11092 [INSTR_JMP_LT_HH] = instr_jmp_export,
11093 [INSTR_JMP_LT_MI] = instr_jmp_export,
11094 [INSTR_JMP_LT_HI] = instr_jmp_export,
11096 [INSTR_JMP_GT] = instr_jmp_export,
11097 [INSTR_JMP_GT_MH] = instr_jmp_export,
11098 [INSTR_JMP_GT_HM] = instr_jmp_export,
11099 [INSTR_JMP_GT_HH] = instr_jmp_export,
11100 [INSTR_JMP_GT_MI] = instr_jmp_export,
11101 [INSTR_JMP_GT_HI] = instr_jmp_export,
11103 [INSTR_RETURN] = instr_return_export,
11107 action_data_codegen(struct action *a, FILE *f)
11112 "static const struct instruction action_%s_instructions[] = {\n",
11115 for (i = 0; i < a->n_instructions; i++) {
11116 struct instruction *instr = &a->instructions[i];
11117 instruction_export_t func = export_table[instr->type];
11122 fprintf(f, "};\n");
11125 static const char *
11126 instr_type_to_func(struct instruction *instr)
11128 switch (instr->type) {
11129 case INSTR_RX: return NULL;
11131 case INSTR_TX: return "__instr_tx_exec";
11132 case INSTR_TX_I: return "__instr_tx_i_exec";
11133 case INSTR_DROP: return "__instr_drop_exec";
11135 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11136 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11137 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11138 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11139 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11140 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11141 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11142 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11144 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11146 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11148 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11149 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11150 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11151 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11152 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11153 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11154 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11155 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11156 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11158 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11159 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11161 case INSTR_MOV: return "__instr_mov_exec";
11162 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11163 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11164 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11165 case INSTR_MOV_I: return "__instr_mov_i_exec";
11167 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11168 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11169 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11170 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11171 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11172 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11173 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11174 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11176 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11177 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11178 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11179 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11180 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11181 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11183 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11184 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11185 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11186 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11187 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11188 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11190 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11191 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11192 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11193 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11195 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11196 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11197 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11198 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11199 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11201 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11202 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11203 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11204 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11205 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11207 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11208 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11209 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11210 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11211 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11213 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11214 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11215 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11216 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11217 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11218 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11220 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11221 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11222 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11223 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11224 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11225 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11227 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11228 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11229 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11231 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11232 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11233 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11234 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11235 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11236 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11238 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11239 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11240 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11241 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11242 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11243 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11244 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11245 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11246 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11248 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11249 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11250 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11251 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11252 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11253 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11254 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11255 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11256 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11258 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11259 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11260 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11262 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11263 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11264 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11265 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11266 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11267 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11268 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11269 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11270 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11271 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11272 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11273 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11275 case INSTR_TABLE: return NULL;
11276 case INSTR_TABLE_AF: return NULL;
11277 case INSTR_SELECTOR: return NULL;
11278 case INSTR_LEARNER: return NULL;
11279 case INSTR_LEARNER_AF: return NULL;
11281 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11282 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11284 case INSTR_EXTERN_OBJ: return NULL;
11285 case INSTR_EXTERN_FUNC: return NULL;
11287 case INSTR_JMP: return NULL;
11288 case INSTR_JMP_VALID: return NULL;
11289 case INSTR_JMP_INVALID: return NULL;
11290 case INSTR_JMP_HIT: return NULL;
11291 case INSTR_JMP_MISS: return NULL;
11292 case INSTR_JMP_ACTION_HIT: return NULL;
11293 case INSTR_JMP_ACTION_MISS: return NULL;
11294 case INSTR_JMP_EQ: return NULL;
11295 case INSTR_JMP_EQ_MH: return NULL;
11296 case INSTR_JMP_EQ_HM: return NULL;
11297 case INSTR_JMP_EQ_HH: return NULL;
11298 case INSTR_JMP_EQ_I: return NULL;
11299 case INSTR_JMP_NEQ: return NULL;
11300 case INSTR_JMP_NEQ_MH: return NULL;
11301 case INSTR_JMP_NEQ_HM: return NULL;
11302 case INSTR_JMP_NEQ_HH: return NULL;
11303 case INSTR_JMP_NEQ_I: return NULL;
11304 case INSTR_JMP_LT: return NULL;
11305 case INSTR_JMP_LT_MH: return NULL;
11306 case INSTR_JMP_LT_HM: return NULL;
11307 case INSTR_JMP_LT_HH: return NULL;
11308 case INSTR_JMP_LT_MI: return NULL;
11309 case INSTR_JMP_LT_HI: return NULL;
11310 case INSTR_JMP_GT: return NULL;
11311 case INSTR_JMP_GT_MH: return NULL;
11312 case INSTR_JMP_GT_HM: return NULL;
11313 case INSTR_JMP_GT_HH: return NULL;
11314 case INSTR_JMP_GT_MI: return NULL;
11315 case INSTR_JMP_GT_HI: return NULL;
11317 case INSTR_RETURN: return NULL;
11319 default: return NULL;
11324 action_instr_does_tx_codegen(struct action *a,
11325 uint32_t instr_pos,
11326 struct instruction *instr,
11330 "%s(p, t, &action_%s_instructions[%u]);\n"
11331 "\tthread_ip_reset(p, t);\n"
11332 "\tinstr_rx_exec(p);\n"
11334 instr_type_to_func(instr),
11340 action_instr_extern_obj_codegen(struct action *a,
11341 uint32_t instr_pos,
11345 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11351 action_instr_extern_func_codegen(struct action *a,
11352 uint32_t instr_pos,
11356 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11362 action_instr_jmp_codegen(struct action *a,
11363 uint32_t instr_pos,
11364 struct instruction *instr,
11365 struct instruction_data *data,
11368 switch (instr->type) {
11375 case INSTR_JMP_VALID:
11377 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11384 case INSTR_JMP_INVALID:
11386 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11393 case INSTR_JMP_HIT:
11400 case INSTR_JMP_MISS:
11407 case INSTR_JMP_ACTION_HIT:
11409 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11416 case INSTR_JMP_ACTION_MISS:
11418 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11427 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11428 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11437 case INSTR_JMP_EQ_MH:
11439 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11440 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11449 case INSTR_JMP_EQ_HM:
11451 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11452 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11461 case INSTR_JMP_EQ_HH:
11463 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11464 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11473 case INSTR_JMP_EQ_I:
11475 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11476 "action_%s_instructions[%u].jmp.b_val)\n"
11485 case INSTR_JMP_NEQ:
11487 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11488 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11497 case INSTR_JMP_NEQ_MH:
11499 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11500 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11509 case INSTR_JMP_NEQ_HM:
11511 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11512 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11521 case INSTR_JMP_NEQ_HH:
11523 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11524 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11533 case INSTR_JMP_NEQ_I:
11535 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11536 "action_%s_instructions[%u].jmp.b_val)\n"
11547 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11548 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11557 case INSTR_JMP_LT_MH:
11559 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11560 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11569 case INSTR_JMP_LT_HM:
11571 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11572 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11581 case INSTR_JMP_LT_HH:
11583 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11584 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11593 case INSTR_JMP_LT_MI:
11595 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11596 "action_%s_instructions[%u].jmp.b_val)\n"
11605 case INSTR_JMP_LT_HI:
11607 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11608 "action_%s_instructions[%u].jmp.b_val)\n"
11619 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11620 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11629 case INSTR_JMP_GT_MH:
11631 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11632 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11641 case INSTR_JMP_GT_HM:
11643 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11644 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11653 case INSTR_JMP_GT_HH:
11655 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11656 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11665 case INSTR_JMP_GT_MI:
11667 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11668 "action_%s_instructions[%u].jmp.b_val)\n"
11677 case INSTR_JMP_GT_HI:
11679 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11680 "action_%s_instructions[%u].jmp.b_val)\n"
11695 action_instr_return_codegen(FILE *f)
11702 action_instr_codegen(struct action *a, FILE *f)
11708 "action_%s_run(struct rte_swx_pipeline *p)\n"
11710 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11714 for (i = 0; i < a->n_instructions; i++) {
11715 struct instruction *instr = &a->instructions[i];
11716 struct instruction_data *data = &a->instruction_data[i];
11718 /* Label, if present. */
11719 if (data->label[0])
11720 fprintf(f, "\n%s : ", data->label);
11722 fprintf(f, "\n\t");
11724 /* TX instruction type. */
11725 if (instruction_does_tx(instr)) {
11726 action_instr_does_tx_codegen(a, i, instr, f);
11730 /* Extern object/function instruction type. */
11731 if (instr->type == INSTR_EXTERN_OBJ) {
11732 action_instr_extern_obj_codegen(a, i, f);
11736 if (instr->type == INSTR_EXTERN_FUNC) {
11737 action_instr_extern_func_codegen(a, i, f);
11741 /* Jump instruction type. */
11742 if (instruction_is_jmp(instr)) {
11743 action_instr_jmp_codegen(a, i, instr, data, f);
11747 /* Return instruction type. */
11748 if (instr->type == INSTR_RETURN) {
11749 action_instr_return_codegen(f);
11753 /* Any other instruction type. */
11755 "%s(p, t, &action_%s_instructions[%u]);\n",
11756 instr_type_to_func(instr),
11761 fprintf(f, "}\n\n");
11764 struct instruction_group {
11765 TAILQ_ENTRY(instruction_group) node;
11769 uint32_t first_instr_id;
11771 uint32_t last_instr_id;
11776 TAILQ_HEAD(instruction_group_list, instruction_group);
11778 static struct instruction_group *
11779 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11781 struct instruction_group *g;
11783 TAILQ_FOREACH(g, igl, node)
11784 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11791 instruction_group_list_free(struct instruction_group_list *igl)
11797 struct instruction_group *g;
11799 g = TAILQ_FIRST(igl);
11803 TAILQ_REMOVE(igl, g, node);
11810 static struct instruction_group_list *
11811 instruction_group_list_create(struct rte_swx_pipeline *p)
11813 struct instruction_group_list *igl = NULL;
11814 struct instruction_group *g = NULL;
11815 uint32_t n_groups = 0, i;
11817 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11821 igl = calloc(1, sizeof(struct instruction_group_list));
11827 /* Allocate the first group. */
11828 g = calloc(1, sizeof(struct instruction_group));
11832 /* Iteration 1: Separate the instructions into groups based on the thread yield
11833 * instructions. Do not worry about the jump instructions at this point.
11835 for (i = 0; i < p->n_instructions; i++) {
11836 struct instruction *instr = &p->instructions[i];
11838 /* Check for thread yield instructions. */
11839 if (!instruction_does_thread_yield(instr))
11842 /* If the current group contains at least one instruction, then finalize it (with
11843 * the previous instruction), add it to the list and allocate a new group (that
11844 * starts with the current instruction).
11846 if (i - g->first_instr_id) {
11847 /* Finalize the group. */
11848 g->last_instr_id = i - 1;
11850 /* Add the group to the list. Advance the number of groups. */
11851 TAILQ_INSERT_TAIL(igl, g, node);
11854 /* Allocate a new group. */
11855 g = calloc(1, sizeof(struct instruction_group));
11859 /* Initialize the new group. */
11860 g->group_id = n_groups;
11861 g->first_instr_id = i;
11864 /* Finalize the current group (with the current instruction, therefore this group
11865 * contains just the current thread yield instruction), add it to the list and
11866 * allocate a new group (that starts with the next instruction).
11869 /* Finalize the group. */
11870 g->last_instr_id = i;
11872 /* Add the group to the list. Advance the number of groups. */
11873 TAILQ_INSERT_TAIL(igl, g, node);
11876 /* Allocate a new group. */
11877 g = calloc(1, sizeof(struct instruction_group));
11881 /* Initialize the new group. */
11882 g->group_id = n_groups;
11883 g->first_instr_id = i + 1;
11886 /* Handle the last group. */
11887 if (i - g->first_instr_id) {
11888 /* Finalize the group. */
11889 g->last_instr_id = i - 1;
11891 /* Add the group to the list. Advance the number of groups. */
11892 TAILQ_INSERT_TAIL(igl, g, node);
11899 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11900 * the destination of a jump instruction located in a different group ("far jump"), then the
11901 * current group has to be split, so that the instruction representing the far jump
11902 * destination is at the start of its group.
11905 int is_modified = 0;
11907 for (i = 0; i < p->n_instructions; i++) {
11908 struct instruction_data *data = &p->instruction_data[i];
11909 struct instruction_group *g;
11912 /* Continue when the current instruction is not a jump destination. */
11913 if (!data->n_users)
11916 g = instruction_group_list_group_find(igl, i);
11920 /* Find out all the jump instructions with this destination. */
11921 for (j = 0; j < p->n_instructions; j++) {
11922 struct instruction *jmp_instr = &p->instructions[j];
11923 struct instruction_data *jmp_data = &p->instruction_data[j];
11924 struct instruction_group *jmp_g, *new_g;
11926 /* Continue when not a jump instruction. Even when jump instruction,
11927 * continue when the jump destination is not this instruction.
11929 if (!instruction_is_jmp(jmp_instr) ||
11930 strcmp(jmp_data->jmp_label, data->label))
11933 jmp_g = instruction_group_list_group_find(igl, j);
11937 /* Continue when both the jump instruction and the jump destination
11938 * instruction are in the same group. Even when in different groups,
11939 * still continue if the jump destination instruction is already the
11940 * first instruction of its group.
11942 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11945 /* Split the group of the current jump destination instruction to
11946 * make this instruction the first instruction of a new group.
11948 new_g = calloc(1, sizeof(struct instruction_group));
11952 new_g->group_id = n_groups;
11953 new_g->first_instr_id = i;
11954 new_g->last_instr_id = g->last_instr_id;
11956 g->last_instr_id = i - 1;
11958 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11962 /* The decision to split this group (to make the current instruction
11963 * the first instruction of a new group) is already taken and fully
11964 * implemented, so no need to search for more reasons to do it.
11970 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11971 * previously considered local (i.e. the jump destination is in the same group as
11972 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11973 * different group than the jump instruction). Wost case scenario: each instruction
11974 * that is a jump destination ends up as the first instruction of its group.
11980 /* Re-assign the group IDs to be in incremental order. */
11982 TAILQ_FOREACH(g, igl, node) {
11991 instruction_group_list_free(igl);
11999 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12000 uint32_t instr_pos,
12001 struct instruction *instr,
12005 "%s(p, t, &pipeline_instructions[%u]);\n"
12006 "\tthread_ip_reset(p, t);\n"
12007 "\tinstr_rx_exec(p);\n"
12009 instr_type_to_func(instr),
12014 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12015 struct instruction_group_list *igl,
12016 uint32_t jmp_instr_id,
12017 struct instruction *jmp_instr,
12018 struct instruction_data *jmp_data,
12021 struct instruction_group *jmp_g, *g;
12022 struct instruction_data *data;
12025 switch (jmp_instr->type) {
12029 case INSTR_JMP_VALID:
12031 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12035 case INSTR_JMP_INVALID:
12037 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12041 case INSTR_JMP_HIT:
12046 case INSTR_JMP_MISS:
12051 case INSTR_JMP_ACTION_HIT:
12053 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12057 case INSTR_JMP_ACTION_MISS:
12059 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12065 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12066 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12071 case INSTR_JMP_EQ_MH:
12073 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12074 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12079 case INSTR_JMP_EQ_HM:
12081 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12082 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12087 case INSTR_JMP_EQ_HH:
12089 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12090 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12095 case INSTR_JMP_EQ_I:
12097 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12098 "pipeline_instructions[%u].jmp.b_val)",
12103 case INSTR_JMP_NEQ:
12105 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12106 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12111 case INSTR_JMP_NEQ_MH:
12113 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12114 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12119 case INSTR_JMP_NEQ_HM:
12121 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12122 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12127 case INSTR_JMP_NEQ_HH:
12129 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12130 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12135 case INSTR_JMP_NEQ_I:
12137 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12138 "pipeline_instructions[%u].jmp.b_val)",
12145 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12146 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12151 case INSTR_JMP_LT_MH:
12153 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12154 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12159 case INSTR_JMP_LT_HM:
12161 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12162 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12167 case INSTR_JMP_LT_HH:
12169 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12170 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12175 case INSTR_JMP_LT_MI:
12177 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12178 "pipeline_instructions[%u].jmp.b_val)",
12183 case INSTR_JMP_LT_HI:
12185 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12186 "pipeline_instructions[%u].jmp.b_val)",
12193 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12194 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12199 case INSTR_JMP_GT_MH:
12201 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12202 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12207 case INSTR_JMP_GT_HM:
12209 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12210 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12215 case INSTR_JMP_GT_HH:
12217 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12218 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12223 case INSTR_JMP_GT_MI:
12225 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12226 "pipeline_instructions[%u].jmp.b_val)",
12231 case INSTR_JMP_GT_HI:
12233 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12234 "pipeline_instructions[%u].jmp.b_val)",
12243 /* Find the instruction group of the jump instruction. */
12244 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12248 /* Find the instruction group of the jump destination instruction. */
12249 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12253 instr_id = data - p->instruction_data;
12255 g = instruction_group_list_group_find(igl, instr_id);
12259 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12260 * instruction group).
12262 if (g->group_id == jmp_g->group_id)
12264 "\n\t\tgoto %s;\n",
12265 jmp_data->jmp_label);
12269 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12278 instruction_group_list_codegen(struct instruction_group_list *igl,
12279 struct rte_swx_pipeline *p,
12282 struct instruction_group *g;
12284 int is_required = 0;
12286 /* Check if code generation is required. */
12287 TAILQ_FOREACH(g, igl, node)
12288 if (g->first_instr_id < g->last_instr_id)
12294 /* Generate the code for the pipeline instruction array. */
12296 "static const struct instruction pipeline_instructions[] = {\n");
12298 for (i = 0; i < p->n_instructions; i++) {
12299 struct instruction *instr = &p->instructions[i];
12300 instruction_export_t func = export_table[instr->type];
12305 fprintf(f, "};\n\n");
12307 /* Generate the code for the pipeline functions: one function for each instruction group
12308 * that contains more than one instruction.
12310 TAILQ_FOREACH(g, igl, node) {
12311 struct instruction *last_instr;
12314 /* Skip if group contains a single instruction. */
12315 if (g->last_instr_id == g->first_instr_id)
12318 /* Generate new pipeline function. */
12321 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12323 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12327 /* Generate the code for each pipeline instruction. */
12328 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12329 struct instruction *instr = &p->instructions[j];
12330 struct instruction_data *data = &p->instruction_data[j];
12332 /* Label, if present. */
12333 if (data->label[0])
12334 fprintf(f, "\n%s : ", data->label);
12336 fprintf(f, "\n\t");
12338 /* TX instruction type. */
12339 if (instruction_does_tx(instr)) {
12340 pipeline_instr_does_tx_codegen(p, j, instr, f);
12344 /* Jump instruction type. */
12345 if (instruction_is_jmp(instr)) {
12346 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12350 /* Any other instruction type. */
12352 "%s(p, t, &pipeline_instructions[%u]);\n",
12353 instr_type_to_func(instr),
12357 /* Finalize the generated pipeline function. For some instructions such as TX,
12358 * emit-many-and-TX and unconditional jump, the next instruction has been already
12359 * decided unconditionally and the instruction pointer of the current thread set
12360 * accordingly; for all the other instructions, the instruction pointer must be
12363 last_instr = &p->instructions[g->last_instr_id];
12365 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12367 "thread_ip_inc(p);\n");
12376 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12378 struct instruction_group *g;
12379 uint32_t n_custom_instr = 0;
12381 /* Groups with a single instruction: no function is generated for this group, the group
12382 * keeps its current instruction. Groups with more than two instructions: one function and
12383 * the associated custom instruction get generated for each such group.
12385 TAILQ_FOREACH(g, igl, node) {
12386 if (g->first_instr_id == g->last_instr_id)
12392 return n_custom_instr;
12396 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12401 /* Create the .c file. */
12402 f = fopen("/tmp/pipeline.c", "w");
12406 /* Include the .h file. */
12407 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12409 /* Add the code for each action. */
12410 TAILQ_FOREACH(a, &p->actions, node) {
12411 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12413 action_data_codegen(a, f);
12417 action_instr_codegen(a, f);
12422 /* Add the pipeline code. */
12423 instruction_group_list_codegen(igl, p, f);
12425 /* Close the .c file. */
12431 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12432 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12436 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12439 struct instruction_group *g;
12440 char *dir_in, *buffer = NULL;
12441 const char *dir_out;
12444 /* Get the environment variables. */
12445 dir_in = getenv("RTE_INSTALL_DIR");
12453 /* Memory allocation for the command buffer. */
12454 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12461 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12462 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12463 "-I %s/lib/pipeline "
12464 "-I %s/lib/eal/include "
12465 "-I %s/lib/eal/x86/include "
12466 "-I %s/lib/eal/include/generic "
12470 "-I %s/lib/pipeline "
12473 "-I %s/lib/eal/linux/include "
12474 ">%s/pipeline.log 2>&1 "
12476 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12477 ">>%s/pipeline.log 2>&1",
12496 /* Build the shared object library. */
12497 status = system(buffer);
12501 /* Open library. */
12503 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12504 "%s/libpipeline.so",
12507 p->lib = dlopen(buffer, RTLD_LAZY);
12513 /* Get the action function symbols. */
12514 TAILQ_FOREACH(a, &p->actions, node) {
12515 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12517 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12518 if (!p->action_funcs[a->id]) {
12524 /* Get the pipeline function symbols. */
12525 TAILQ_FOREACH(g, igl, node) {
12526 if (g->first_instr_id == g->last_instr_id)
12529 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12531 g->func = dlsym(p->lib, buffer);
12539 if (status && p->lib) {
12550 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12551 struct instruction_group_list *igl)
12553 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12555 /* Check that enough space is available within the pipeline instruction table to store all
12556 * the custom instructions.
12558 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12565 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12567 struct instruction_group *g;
12570 /* Pipeline table instructions. */
12571 for (i = 0; i < p->n_instructions; i++) {
12572 struct instruction *instr = &p->instructions[i];
12574 if (instr->type == INSTR_TABLE)
12575 instr->type = INSTR_TABLE_AF;
12577 if (instr->type == INSTR_LEARNER)
12578 instr->type = INSTR_LEARNER_AF;
12581 /* Pipeline custom instructions. */
12583 TAILQ_FOREACH(g, igl, node) {
12584 struct instruction *instr = &p->instructions[g->first_instr_id];
12587 if (g->first_instr_id == g->last_instr_id)
12590 /* Install a new custom instruction. */
12591 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12593 /* First instruction of the group: change its type to the new custom instruction. */
12594 instr->type = INSTR_CUSTOM_0 + i;
12596 /* All the subsequent instructions of the group: invalidate. */
12597 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12598 struct instruction_data *data = &p->instruction_data[j];
12606 /* Remove the invalidated instructions. */
12607 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12609 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12610 * instructions that are the only instruction within their group, so they were left
12613 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12617 pipeline_compile(struct rte_swx_pipeline *p)
12619 struct instruction_group_list *igl = NULL;
12622 igl = instruction_group_list_create(p);
12628 /* Code generation. */
12629 status = pipeline_codegen(p, igl);
12633 /* Build and load the shared object library. */
12634 status = pipeline_libload(p, igl);
12638 /* Adjust instructions. */
12639 status = pipeline_adjust_check(p, igl);
12643 pipeline_adjust(p, igl);
12646 instruction_group_list_free(igl);