1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
9 #include <rte_swx_port_ethdev.h>
10 #include <rte_swx_port_fd.h>
11 #include <rte_swx_port_ring.h>
12 #include "rte_swx_port_source_sink.h"
14 #include <rte_swx_table_em.h>
15 #include <rte_swx_table_wm.h>
17 #include "rte_swx_pipeline_internal.h"
19 #define CHECK(condition, err_code) \
25 #define CHECK_NAME(name, err_code) \
28 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
31 #define CHECK_INSTRUCTION(instr, err_code) \
34 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
35 RTE_SWX_INSTRUCTION_SIZE), \
41 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
43 #include <rte_malloc.h>
46 env_malloc(size_t size, size_t alignment, int numa_node)
48 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
52 env_free(void *start, size_t size __rte_unused)
62 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
66 if (numa_available() == -1)
69 start = numa_alloc_onnode(size, numa_node);
73 memset(start, 0, size);
78 env_free(void *start, size_t size)
80 if (numa_available() == -1)
83 numa_free(start, size);
91 static struct struct_type *
92 struct_type_find(struct rte_swx_pipeline *p, const char *name)
94 struct struct_type *elem;
96 TAILQ_FOREACH(elem, &p->struct_types, node)
97 if (strcmp(elem->name, name) == 0)
103 static struct field *
104 struct_type_field_find(struct struct_type *st, const char *name)
108 for (i = 0; i < st->n_fields; i++) {
109 struct field *f = &st->fields[i];
111 if (strcmp(f->name, name) == 0)
119 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
121 struct rte_swx_field_params *fields,
123 int last_field_has_variable_size)
125 struct struct_type *st;
129 CHECK_NAME(name, EINVAL);
130 CHECK(fields, EINVAL);
131 CHECK(n_fields, EINVAL);
133 for (i = 0; i < n_fields; i++) {
134 struct rte_swx_field_params *f = &fields[i];
135 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
138 CHECK_NAME(f->name, EINVAL);
139 CHECK(f->n_bits, EINVAL);
140 CHECK((f->n_bits <= 64) || var_size, EINVAL);
141 CHECK((f->n_bits & 7) == 0, EINVAL);
143 for (j = 0; j < i; j++) {
144 struct rte_swx_field_params *f_prev = &fields[j];
146 CHECK(strcmp(f->name, f_prev->name), EINVAL);
150 CHECK(!struct_type_find(p, name), EEXIST);
152 /* Node allocation. */
153 st = calloc(1, sizeof(struct struct_type));
156 st->fields = calloc(n_fields, sizeof(struct field));
162 /* Node initialization. */
163 strcpy(st->name, name);
164 for (i = 0; i < n_fields; i++) {
165 struct field *dst = &st->fields[i];
166 struct rte_swx_field_params *src = &fields[i];
167 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
169 strcpy(dst->name, src->name);
170 dst->n_bits = src->n_bits;
171 dst->offset = st->n_bits;
172 dst->var_size = var_size;
174 st->n_bits += src->n_bits;
175 st->n_bits_min += var_size ? 0 : src->n_bits;
177 st->n_fields = n_fields;
178 st->var_size = last_field_has_variable_size;
180 /* Node add to tailq. */
181 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
187 struct_build(struct rte_swx_pipeline *p)
191 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
192 struct thread *t = &p->threads[i];
194 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
195 CHECK(t->structs, ENOMEM);
202 struct_build_free(struct rte_swx_pipeline *p)
206 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
207 struct thread *t = &p->threads[i];
215 struct_free(struct rte_swx_pipeline *p)
217 struct_build_free(p);
221 struct struct_type *elem;
223 elem = TAILQ_FIRST(&p->struct_types);
227 TAILQ_REMOVE(&p->struct_types, elem, node);
236 static struct port_in_type *
237 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
239 struct port_in_type *elem;
244 TAILQ_FOREACH(elem, &p->port_in_types, node)
245 if (strcmp(elem->name, name) == 0)
252 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
254 struct rte_swx_port_in_ops *ops)
256 struct port_in_type *elem;
259 CHECK_NAME(name, EINVAL);
261 CHECK(ops->create, EINVAL);
262 CHECK(ops->free, EINVAL);
263 CHECK(ops->pkt_rx, EINVAL);
264 CHECK(ops->stats_read, EINVAL);
266 CHECK(!port_in_type_find(p, name), EEXIST);
268 /* Node allocation. */
269 elem = calloc(1, sizeof(struct port_in_type));
272 /* Node initialization. */
273 strcpy(elem->name, name);
274 memcpy(&elem->ops, ops, sizeof(*ops));
276 /* Node add to tailq. */
277 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
282 static struct port_in *
283 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
285 struct port_in *port;
287 TAILQ_FOREACH(port, &p->ports_in, node)
288 if (port->id == port_id)
295 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
297 const char *port_type_name,
300 struct port_in_type *type = NULL;
301 struct port_in *port = NULL;
306 CHECK(!port_in_find(p, port_id), EINVAL);
308 CHECK_NAME(port_type_name, EINVAL);
309 type = port_in_type_find(p, port_type_name);
312 obj = type->ops.create(args);
315 /* Node allocation. */
316 port = calloc(1, sizeof(struct port_in));
319 /* Node initialization. */
324 /* Node add to tailq. */
325 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
326 if (p->n_ports_in < port_id + 1)
327 p->n_ports_in = port_id + 1;
333 port_in_build(struct rte_swx_pipeline *p)
335 struct port_in *port;
338 CHECK(p->n_ports_in, EINVAL);
339 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
341 for (i = 0; i < p->n_ports_in; i++)
342 CHECK(port_in_find(p, i), EINVAL);
344 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
345 CHECK(p->in, ENOMEM);
347 TAILQ_FOREACH(port, &p->ports_in, node) {
348 struct port_in_runtime *in = &p->in[port->id];
350 in->pkt_rx = port->type->ops.pkt_rx;
358 port_in_build_free(struct rte_swx_pipeline *p)
365 port_in_free(struct rte_swx_pipeline *p)
367 port_in_build_free(p);
371 struct port_in *port;
373 port = TAILQ_FIRST(&p->ports_in);
377 TAILQ_REMOVE(&p->ports_in, port, node);
378 port->type->ops.free(port->obj);
382 /* Input port types. */
384 struct port_in_type *elem;
386 elem = TAILQ_FIRST(&p->port_in_types);
390 TAILQ_REMOVE(&p->port_in_types, elem, node);
398 static struct port_out_type *
399 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
401 struct port_out_type *elem;
406 TAILQ_FOREACH(elem, &p->port_out_types, node)
407 if (!strcmp(elem->name, name))
414 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
416 struct rte_swx_port_out_ops *ops)
418 struct port_out_type *elem;
421 CHECK_NAME(name, EINVAL);
423 CHECK(ops->create, EINVAL);
424 CHECK(ops->free, EINVAL);
425 CHECK(ops->pkt_tx, EINVAL);
426 CHECK(ops->stats_read, EINVAL);
428 CHECK(!port_out_type_find(p, name), EEXIST);
430 /* Node allocation. */
431 elem = calloc(1, sizeof(struct port_out_type));
434 /* Node initialization. */
435 strcpy(elem->name, name);
436 memcpy(&elem->ops, ops, sizeof(*ops));
438 /* Node add to tailq. */
439 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
444 static struct port_out *
445 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
447 struct port_out *port;
449 TAILQ_FOREACH(port, &p->ports_out, node)
450 if (port->id == port_id)
457 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
459 const char *port_type_name,
462 struct port_out_type *type = NULL;
463 struct port_out *port = NULL;
468 CHECK(!port_out_find(p, port_id), EINVAL);
470 CHECK_NAME(port_type_name, EINVAL);
471 type = port_out_type_find(p, port_type_name);
474 obj = type->ops.create(args);
477 /* Node allocation. */
478 port = calloc(1, sizeof(struct port_out));
481 /* Node initialization. */
486 /* Node add to tailq. */
487 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
488 if (p->n_ports_out < port_id + 1)
489 p->n_ports_out = port_id + 1;
495 port_out_build(struct rte_swx_pipeline *p)
497 struct port_out *port;
500 CHECK(p->n_ports_out, EINVAL);
502 for (i = 0; i < p->n_ports_out; i++)
503 CHECK(port_out_find(p, i), EINVAL);
505 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
506 CHECK(p->out, ENOMEM);
508 TAILQ_FOREACH(port, &p->ports_out, node) {
509 struct port_out_runtime *out = &p->out[port->id];
511 out->pkt_tx = port->type->ops.pkt_tx;
512 out->flush = port->type->ops.flush;
513 out->obj = port->obj;
520 port_out_build_free(struct rte_swx_pipeline *p)
527 port_out_free(struct rte_swx_pipeline *p)
529 port_out_build_free(p);
533 struct port_out *port;
535 port = TAILQ_FIRST(&p->ports_out);
539 TAILQ_REMOVE(&p->ports_out, port, node);
540 port->type->ops.free(port->obj);
544 /* Output port types. */
546 struct port_out_type *elem;
548 elem = TAILQ_FIRST(&p->port_out_types);
552 TAILQ_REMOVE(&p->port_out_types, elem, node);
560 static struct extern_type *
561 extern_type_find(struct rte_swx_pipeline *p, const char *name)
563 struct extern_type *elem;
565 TAILQ_FOREACH(elem, &p->extern_types, node)
566 if (strcmp(elem->name, name) == 0)
572 static struct extern_type_member_func *
573 extern_type_member_func_find(struct extern_type *type, const char *name)
575 struct extern_type_member_func *elem;
577 TAILQ_FOREACH(elem, &type->funcs, node)
578 if (strcmp(elem->name, name) == 0)
584 static struct extern_obj *
585 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
587 struct extern_obj *elem;
589 TAILQ_FOREACH(elem, &p->extern_objs, node)
590 if (strcmp(elem->name, name) == 0)
596 static struct extern_type_member_func *
597 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
599 struct extern_obj **obj)
601 struct extern_obj *object;
602 struct extern_type_member_func *func;
603 char *object_name, *func_name;
605 if (name[0] != 'e' || name[1] != '.')
608 object_name = strdup(&name[2]);
612 func_name = strchr(object_name, '.');
621 object = extern_obj_find(p, object_name);
627 func = extern_type_member_func_find(object->type, func_name);
640 static struct field *
641 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
643 struct extern_obj **object)
645 struct extern_obj *obj;
647 char *obj_name, *field_name;
649 if ((name[0] != 'e') || (name[1] != '.'))
652 obj_name = strdup(&name[2]);
656 field_name = strchr(obj_name, '.');
665 obj = extern_obj_find(p, obj_name);
671 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
685 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
687 const char *mailbox_struct_type_name,
688 rte_swx_extern_type_constructor_t constructor,
689 rte_swx_extern_type_destructor_t destructor)
691 struct extern_type *elem;
692 struct struct_type *mailbox_struct_type;
696 CHECK_NAME(name, EINVAL);
697 CHECK(!extern_type_find(p, name), EEXIST);
699 CHECK_NAME(mailbox_struct_type_name, EINVAL);
700 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
701 CHECK(mailbox_struct_type, EINVAL);
702 CHECK(!mailbox_struct_type->var_size, EINVAL);
704 CHECK(constructor, EINVAL);
705 CHECK(destructor, EINVAL);
707 /* Node allocation. */
708 elem = calloc(1, sizeof(struct extern_type));
711 /* Node initialization. */
712 strcpy(elem->name, name);
713 elem->mailbox_struct_type = mailbox_struct_type;
714 elem->constructor = constructor;
715 elem->destructor = destructor;
716 TAILQ_INIT(&elem->funcs);
718 /* Node add to tailq. */
719 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
725 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
726 const char *extern_type_name,
728 rte_swx_extern_type_member_func_t member_func)
730 struct extern_type *type;
731 struct extern_type_member_func *type_member;
735 CHECK_NAME(extern_type_name, EINVAL);
736 type = extern_type_find(p, extern_type_name);
738 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
740 CHECK_NAME(name, EINVAL);
741 CHECK(!extern_type_member_func_find(type, name), EEXIST);
743 CHECK(member_func, EINVAL);
745 /* Node allocation. */
746 type_member = calloc(1, sizeof(struct extern_type_member_func));
747 CHECK(type_member, ENOMEM);
749 /* Node initialization. */
750 strcpy(type_member->name, name);
751 type_member->func = member_func;
752 type_member->id = type->n_funcs;
754 /* Node add to tailq. */
755 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
762 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
763 const char *extern_type_name,
767 struct extern_type *type;
768 struct extern_obj *obj;
773 CHECK_NAME(extern_type_name, EINVAL);
774 type = extern_type_find(p, extern_type_name);
777 CHECK_NAME(name, EINVAL);
778 CHECK(!extern_obj_find(p, name), EEXIST);
780 /* Node allocation. */
781 obj = calloc(1, sizeof(struct extern_obj));
784 /* Object construction. */
785 obj_handle = type->constructor(args);
791 /* Node initialization. */
792 strcpy(obj->name, name);
794 obj->obj = obj_handle;
795 obj->struct_id = p->n_structs;
796 obj->id = p->n_extern_objs;
798 /* Node add to tailq. */
799 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
807 extern_obj_build(struct rte_swx_pipeline *p)
811 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
812 struct thread *t = &p->threads[i];
813 struct extern_obj *obj;
815 t->extern_objs = calloc(p->n_extern_objs,
816 sizeof(struct extern_obj_runtime));
817 CHECK(t->extern_objs, ENOMEM);
819 TAILQ_FOREACH(obj, &p->extern_objs, node) {
820 struct extern_obj_runtime *r =
821 &t->extern_objs[obj->id];
822 struct extern_type_member_func *func;
823 uint32_t mailbox_size =
824 obj->type->mailbox_struct_type->n_bits / 8;
828 r->mailbox = calloc(1, mailbox_size);
829 CHECK(r->mailbox, ENOMEM);
831 TAILQ_FOREACH(func, &obj->type->funcs, node)
832 r->funcs[func->id] = func->func;
834 t->structs[obj->struct_id] = r->mailbox;
842 extern_obj_build_free(struct rte_swx_pipeline *p)
846 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
847 struct thread *t = &p->threads[i];
853 for (j = 0; j < p->n_extern_objs; j++) {
854 struct extern_obj_runtime *r = &t->extern_objs[j];
859 free(t->extern_objs);
860 t->extern_objs = NULL;
865 extern_obj_free(struct rte_swx_pipeline *p)
867 extern_obj_build_free(p);
869 /* Extern objects. */
871 struct extern_obj *elem;
873 elem = TAILQ_FIRST(&p->extern_objs);
877 TAILQ_REMOVE(&p->extern_objs, elem, node);
879 elem->type->destructor(elem->obj);
885 struct extern_type *elem;
887 elem = TAILQ_FIRST(&p->extern_types);
891 TAILQ_REMOVE(&p->extern_types, elem, node);
894 struct extern_type_member_func *func;
896 func = TAILQ_FIRST(&elem->funcs);
900 TAILQ_REMOVE(&elem->funcs, func, node);
911 static struct extern_func *
912 extern_func_find(struct rte_swx_pipeline *p, const char *name)
914 struct extern_func *elem;
916 TAILQ_FOREACH(elem, &p->extern_funcs, node)
917 if (strcmp(elem->name, name) == 0)
923 static struct extern_func *
924 extern_func_parse(struct rte_swx_pipeline *p,
927 if (name[0] != 'f' || name[1] != '.')
930 return extern_func_find(p, &name[2]);
933 static struct field *
934 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
936 struct extern_func **function)
938 struct extern_func *func;
940 char *func_name, *field_name;
942 if ((name[0] != 'f') || (name[1] != '.'))
945 func_name = strdup(&name[2]);
949 field_name = strchr(func_name, '.');
958 func = extern_func_find(p, func_name);
964 f = struct_type_field_find(func->mailbox_struct_type, field_name);
978 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
980 const char *mailbox_struct_type_name,
981 rte_swx_extern_func_t func)
983 struct extern_func *f;
984 struct struct_type *mailbox_struct_type;
988 CHECK_NAME(name, EINVAL);
989 CHECK(!extern_func_find(p, name), EEXIST);
991 CHECK_NAME(mailbox_struct_type_name, EINVAL);
992 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
993 CHECK(mailbox_struct_type, EINVAL);
994 CHECK(!mailbox_struct_type->var_size, EINVAL);
998 /* Node allocation. */
999 f = calloc(1, sizeof(struct extern_func));
1000 CHECK(func, ENOMEM);
1002 /* Node initialization. */
1003 strcpy(f->name, name);
1004 f->mailbox_struct_type = mailbox_struct_type;
1006 f->struct_id = p->n_structs;
1007 f->id = p->n_extern_funcs;
1009 /* Node add to tailq. */
1010 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1011 p->n_extern_funcs++;
1018 extern_func_build(struct rte_swx_pipeline *p)
1022 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1023 struct thread *t = &p->threads[i];
1024 struct extern_func *func;
1026 /* Memory allocation. */
1027 t->extern_funcs = calloc(p->n_extern_funcs,
1028 sizeof(struct extern_func_runtime));
1029 CHECK(t->extern_funcs, ENOMEM);
1031 /* Extern function. */
1032 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1033 struct extern_func_runtime *r =
1034 &t->extern_funcs[func->id];
1035 uint32_t mailbox_size =
1036 func->mailbox_struct_type->n_bits / 8;
1038 r->func = func->func;
1040 r->mailbox = calloc(1, mailbox_size);
1041 CHECK(r->mailbox, ENOMEM);
1043 t->structs[func->struct_id] = r->mailbox;
1051 extern_func_build_free(struct rte_swx_pipeline *p)
1055 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1056 struct thread *t = &p->threads[i];
1059 if (!t->extern_funcs)
1062 for (j = 0; j < p->n_extern_funcs; j++) {
1063 struct extern_func_runtime *r = &t->extern_funcs[j];
1068 free(t->extern_funcs);
1069 t->extern_funcs = NULL;
1074 extern_func_free(struct rte_swx_pipeline *p)
1076 extern_func_build_free(p);
1079 struct extern_func *elem;
1081 elem = TAILQ_FIRST(&p->extern_funcs);
1085 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1093 static struct header *
1094 header_find(struct rte_swx_pipeline *p, const char *name)
1096 struct header *elem;
1098 TAILQ_FOREACH(elem, &p->headers, node)
1099 if (strcmp(elem->name, name) == 0)
1105 static struct header *
1106 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1108 struct header *elem;
1110 TAILQ_FOREACH(elem, &p->headers, node)
1111 if (elem->struct_id == struct_id)
1117 static struct header *
1118 header_parse(struct rte_swx_pipeline *p,
1121 if (name[0] != 'h' || name[1] != '.')
1124 return header_find(p, &name[2]);
1127 static struct field *
1128 header_field_parse(struct rte_swx_pipeline *p,
1130 struct header **header)
1134 char *header_name, *field_name;
1136 if ((name[0] != 'h') || (name[1] != '.'))
1139 header_name = strdup(&name[2]);
1143 field_name = strchr(header_name, '.');
1152 h = header_find(p, header_name);
1158 f = struct_type_field_find(h->st, field_name);
1172 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1174 const char *struct_type_name)
1176 struct struct_type *st;
1178 size_t n_headers_max;
1181 CHECK_NAME(name, EINVAL);
1182 CHECK_NAME(struct_type_name, EINVAL);
1184 CHECK(!header_find(p, name), EEXIST);
1186 st = struct_type_find(p, struct_type_name);
1189 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1190 CHECK(p->n_headers < n_headers_max, ENOSPC);
1192 /* Node allocation. */
1193 h = calloc(1, sizeof(struct header));
1196 /* Node initialization. */
1197 strcpy(h->name, name);
1199 h->struct_id = p->n_structs;
1200 h->id = p->n_headers;
1202 /* Node add to tailq. */
1203 TAILQ_INSERT_TAIL(&p->headers, h, node);
1211 header_build(struct rte_swx_pipeline *p)
1214 uint32_t n_bytes = 0, i;
1216 TAILQ_FOREACH(h, &p->headers, node) {
1217 n_bytes += h->st->n_bits / 8;
1220 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1221 struct thread *t = &p->threads[i];
1222 uint32_t offset = 0;
1224 t->headers = calloc(p->n_headers,
1225 sizeof(struct header_runtime));
1226 CHECK(t->headers, ENOMEM);
1228 t->headers_out = calloc(p->n_headers,
1229 sizeof(struct header_out_runtime));
1230 CHECK(t->headers_out, ENOMEM);
1232 t->header_storage = calloc(1, n_bytes);
1233 CHECK(t->header_storage, ENOMEM);
1235 t->header_out_storage = calloc(1, n_bytes);
1236 CHECK(t->header_out_storage, ENOMEM);
1238 TAILQ_FOREACH(h, &p->headers, node) {
1239 uint8_t *header_storage;
1240 uint32_t n_bytes = h->st->n_bits / 8;
1242 header_storage = &t->header_storage[offset];
1245 t->headers[h->id].ptr0 = header_storage;
1246 t->headers[h->id].n_bytes = n_bytes;
1248 t->structs[h->struct_id] = header_storage;
1256 header_build_free(struct rte_swx_pipeline *p)
1260 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1261 struct thread *t = &p->threads[i];
1263 free(t->headers_out);
1264 t->headers_out = NULL;
1269 free(t->header_out_storage);
1270 t->header_out_storage = NULL;
1272 free(t->header_storage);
1273 t->header_storage = NULL;
1278 header_free(struct rte_swx_pipeline *p)
1280 header_build_free(p);
1283 struct header *elem;
1285 elem = TAILQ_FIRST(&p->headers);
1289 TAILQ_REMOVE(&p->headers, elem, node);
1297 static struct field *
1298 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1300 if (!p->metadata_st)
1303 if (name[0] != 'm' || name[1] != '.')
1306 return struct_type_field_find(p->metadata_st, &name[2]);
1310 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1311 const char *struct_type_name)
1313 struct struct_type *st = NULL;
1317 CHECK_NAME(struct_type_name, EINVAL);
1318 st = struct_type_find(p, struct_type_name);
1320 CHECK(!st->var_size, EINVAL);
1321 CHECK(!p->metadata_st, EINVAL);
1323 p->metadata_st = st;
1324 p->metadata_struct_id = p->n_structs;
1332 metadata_build(struct rte_swx_pipeline *p)
1334 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1337 /* Thread-level initialization. */
1338 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1339 struct thread *t = &p->threads[i];
1342 metadata = calloc(1, n_bytes);
1343 CHECK(metadata, ENOMEM);
1345 t->metadata = metadata;
1346 t->structs[p->metadata_struct_id] = metadata;
1353 metadata_build_free(struct rte_swx_pipeline *p)
1357 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1358 struct thread *t = &p->threads[i];
1366 metadata_free(struct rte_swx_pipeline *p)
1368 metadata_build_free(p);
1375 instruction_is_tx(enum instruction_type type)
1389 instruction_does_tx(struct instruction *instr)
1391 switch (instr->type) {
1395 case INSTR_HDR_EMIT_TX:
1396 case INSTR_HDR_EMIT2_TX:
1397 case INSTR_HDR_EMIT3_TX:
1398 case INSTR_HDR_EMIT4_TX:
1399 case INSTR_HDR_EMIT5_TX:
1400 case INSTR_HDR_EMIT6_TX:
1401 case INSTR_HDR_EMIT7_TX:
1402 case INSTR_HDR_EMIT8_TX:
1410 instruction_is_jmp(struct instruction *instr)
1412 switch (instr->type) {
1414 case INSTR_JMP_VALID:
1415 case INSTR_JMP_INVALID:
1417 case INSTR_JMP_MISS:
1418 case INSTR_JMP_ACTION_HIT:
1419 case INSTR_JMP_ACTION_MISS:
1421 case INSTR_JMP_EQ_MH:
1422 case INSTR_JMP_EQ_HM:
1423 case INSTR_JMP_EQ_HH:
1424 case INSTR_JMP_EQ_I:
1426 case INSTR_JMP_NEQ_MH:
1427 case INSTR_JMP_NEQ_HM:
1428 case INSTR_JMP_NEQ_HH:
1429 case INSTR_JMP_NEQ_I:
1431 case INSTR_JMP_LT_MH:
1432 case INSTR_JMP_LT_HM:
1433 case INSTR_JMP_LT_HH:
1434 case INSTR_JMP_LT_MI:
1435 case INSTR_JMP_LT_HI:
1437 case INSTR_JMP_GT_MH:
1438 case INSTR_JMP_GT_HM:
1439 case INSTR_JMP_GT_HH:
1440 case INSTR_JMP_GT_MI:
1441 case INSTR_JMP_GT_HI:
1450 instruction_does_thread_yield(struct instruction *instr)
1452 switch (instr->type) {
1455 case INSTR_TABLE_AF:
1456 case INSTR_SELECTOR:
1458 case INSTR_LEARNER_AF:
1459 case INSTR_EXTERN_OBJ:
1460 case INSTR_EXTERN_FUNC:
1467 static struct field *
1468 action_field_parse(struct action *action, const char *name);
1470 static struct field *
1471 struct_field_parse(struct rte_swx_pipeline *p,
1472 struct action *action,
1474 uint32_t *struct_id)
1481 struct header *header;
1483 f = header_field_parse(p, name, &header);
1487 *struct_id = header->struct_id;
1493 f = metadata_field_parse(p, name);
1497 *struct_id = p->metadata_struct_id;
1506 f = action_field_parse(action, name);
1516 struct extern_obj *obj;
1518 f = extern_obj_mailbox_field_parse(p, name, &obj);
1522 *struct_id = obj->struct_id;
1528 struct extern_func *func;
1530 f = extern_func_mailbox_field_parse(p, name, &func);
1534 *struct_id = func->struct_id;
1547 instr_rx_translate(struct rte_swx_pipeline *p,
1548 struct action *action,
1551 struct instruction *instr,
1552 struct instruction_data *data __rte_unused)
1556 CHECK(!action, EINVAL);
1557 CHECK(n_tokens == 2, EINVAL);
1559 f = metadata_field_parse(p, tokens[1]);
1562 instr->type = INSTR_RX;
1563 instr->io.io.offset = f->offset / 8;
1564 instr->io.io.n_bits = f->n_bits;
1572 instr_tx_translate(struct rte_swx_pipeline *p,
1573 struct action *action __rte_unused,
1576 struct instruction *instr,
1577 struct instruction_data *data __rte_unused)
1579 char *port = tokens[1];
1583 CHECK(n_tokens == 2, EINVAL);
1585 f = metadata_field_parse(p, port);
1587 instr->type = INSTR_TX;
1588 instr->io.io.offset = f->offset / 8;
1589 instr->io.io.n_bits = f->n_bits;
1594 port_val = strtoul(port, &port, 0);
1595 CHECK(!port[0], EINVAL);
1597 instr->type = INSTR_TX_I;
1598 instr->io.io.val = port_val;
1603 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1604 struct action *action __rte_unused,
1605 char **tokens __rte_unused,
1607 struct instruction *instr,
1608 struct instruction_data *data __rte_unused)
1610 CHECK(n_tokens == 1, EINVAL);
1613 instr->type = INSTR_DROP;
1618 instr_tx_exec(struct rte_swx_pipeline *p)
1620 struct thread *t = &p->threads[p->thread_id];
1621 struct instruction *ip = t->ip;
1623 __instr_tx_exec(p, t, ip);
1626 thread_ip_reset(p, t);
1631 instr_tx_i_exec(struct rte_swx_pipeline *p)
1633 struct thread *t = &p->threads[p->thread_id];
1634 struct instruction *ip = t->ip;
1636 __instr_tx_i_exec(p, t, ip);
1639 thread_ip_reset(p, t);
1644 instr_drop_exec(struct rte_swx_pipeline *p)
1646 struct thread *t = &p->threads[p->thread_id];
1647 struct instruction *ip = t->ip;
1649 __instr_drop_exec(p, t, ip);
1652 thread_ip_reset(p, t);
1660 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1661 struct action *action,
1664 struct instruction *instr,
1665 struct instruction_data *data __rte_unused)
1669 CHECK(!action, EINVAL);
1670 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1672 h = header_parse(p, tokens[1]);
1675 if (n_tokens == 2) {
1676 CHECK(!h->st->var_size, EINVAL);
1678 instr->type = INSTR_HDR_EXTRACT;
1679 instr->io.hdr.header_id[0] = h->id;
1680 instr->io.hdr.struct_id[0] = h->struct_id;
1681 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1685 CHECK(h->st->var_size, EINVAL);
1687 mf = metadata_field_parse(p, tokens[2]);
1689 CHECK(!mf->var_size, EINVAL);
1691 instr->type = INSTR_HDR_EXTRACT_M;
1692 instr->io.io.offset = mf->offset / 8;
1693 instr->io.io.n_bits = mf->n_bits;
1694 instr->io.hdr.header_id[0] = h->id;
1695 instr->io.hdr.struct_id[0] = h->struct_id;
1696 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1703 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1704 struct action *action,
1707 struct instruction *instr,
1708 struct instruction_data *data __rte_unused)
1712 CHECK(!action, EINVAL);
1713 CHECK(n_tokens == 2, EINVAL);
1715 h = header_parse(p, tokens[1]);
1717 CHECK(!h->st->var_size, EINVAL);
1719 instr->type = INSTR_HDR_LOOKAHEAD;
1720 instr->io.hdr.header_id[0] = h->id;
1721 instr->io.hdr.struct_id[0] = h->struct_id;
1722 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1728 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1730 struct thread *t = &p->threads[p->thread_id];
1731 struct instruction *ip = t->ip;
1733 __instr_hdr_extract_exec(p, t, ip);
1740 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1742 struct thread *t = &p->threads[p->thread_id];
1743 struct instruction *ip = t->ip;
1745 __instr_hdr_extract2_exec(p, t, ip);
1752 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1754 struct thread *t = &p->threads[p->thread_id];
1755 struct instruction *ip = t->ip;
1757 __instr_hdr_extract3_exec(p, t, ip);
1764 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1766 struct thread *t = &p->threads[p->thread_id];
1767 struct instruction *ip = t->ip;
1769 __instr_hdr_extract4_exec(p, t, ip);
1776 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1778 struct thread *t = &p->threads[p->thread_id];
1779 struct instruction *ip = t->ip;
1781 __instr_hdr_extract5_exec(p, t, ip);
1788 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1790 struct thread *t = &p->threads[p->thread_id];
1791 struct instruction *ip = t->ip;
1793 __instr_hdr_extract6_exec(p, t, ip);
1800 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1802 struct thread *t = &p->threads[p->thread_id];
1803 struct instruction *ip = t->ip;
1805 __instr_hdr_extract7_exec(p, t, ip);
1812 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1814 struct thread *t = &p->threads[p->thread_id];
1815 struct instruction *ip = t->ip;
1817 __instr_hdr_extract8_exec(p, t, ip);
1824 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1826 struct thread *t = &p->threads[p->thread_id];
1827 struct instruction *ip = t->ip;
1829 __instr_hdr_extract_m_exec(p, t, ip);
1836 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1838 struct thread *t = &p->threads[p->thread_id];
1839 struct instruction *ip = t->ip;
1841 __instr_hdr_lookahead_exec(p, t, ip);
1851 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1852 struct action *action __rte_unused,
1855 struct instruction *instr,
1856 struct instruction_data *data __rte_unused)
1860 CHECK(n_tokens == 2, EINVAL);
1862 h = header_parse(p, tokens[1]);
1865 instr->type = INSTR_HDR_EMIT;
1866 instr->io.hdr.header_id[0] = h->id;
1867 instr->io.hdr.struct_id[0] = h->struct_id;
1868 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1873 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1875 struct thread *t = &p->threads[p->thread_id];
1876 struct instruction *ip = t->ip;
1878 __instr_hdr_emit_exec(p, t, ip);
1885 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1887 struct thread *t = &p->threads[p->thread_id];
1888 struct instruction *ip = t->ip;
1890 __instr_hdr_emit_tx_exec(p, t, ip);
1893 thread_ip_reset(p, t);
1898 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1900 struct thread *t = &p->threads[p->thread_id];
1901 struct instruction *ip = t->ip;
1903 __instr_hdr_emit2_tx_exec(p, t, ip);
1906 thread_ip_reset(p, t);
1911 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1913 struct thread *t = &p->threads[p->thread_id];
1914 struct instruction *ip = t->ip;
1916 __instr_hdr_emit3_tx_exec(p, t, ip);
1919 thread_ip_reset(p, t);
1924 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1926 struct thread *t = &p->threads[p->thread_id];
1927 struct instruction *ip = t->ip;
1929 __instr_hdr_emit4_tx_exec(p, t, ip);
1932 thread_ip_reset(p, t);
1937 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1939 struct thread *t = &p->threads[p->thread_id];
1940 struct instruction *ip = t->ip;
1942 __instr_hdr_emit5_tx_exec(p, t, ip);
1945 thread_ip_reset(p, t);
1950 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1952 struct thread *t = &p->threads[p->thread_id];
1953 struct instruction *ip = t->ip;
1955 __instr_hdr_emit6_tx_exec(p, t, ip);
1958 thread_ip_reset(p, t);
1963 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1965 struct thread *t = &p->threads[p->thread_id];
1966 struct instruction *ip = t->ip;
1968 __instr_hdr_emit7_tx_exec(p, t, ip);
1971 thread_ip_reset(p, t);
1976 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1978 struct thread *t = &p->threads[p->thread_id];
1979 struct instruction *ip = t->ip;
1981 __instr_hdr_emit8_tx_exec(p, t, ip);
1984 thread_ip_reset(p, t);
1992 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1993 struct action *action __rte_unused,
1996 struct instruction *instr,
1997 struct instruction_data *data __rte_unused)
2001 CHECK(n_tokens == 2, EINVAL);
2003 h = header_parse(p, tokens[1]);
2006 instr->type = INSTR_HDR_VALIDATE;
2007 instr->valid.header_id = h->id;
2012 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2014 struct thread *t = &p->threads[p->thread_id];
2015 struct instruction *ip = t->ip;
2017 __instr_hdr_validate_exec(p, t, ip);
2027 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2028 struct action *action __rte_unused,
2031 struct instruction *instr,
2032 struct instruction_data *data __rte_unused)
2036 CHECK(n_tokens == 2, EINVAL);
2038 h = header_parse(p, tokens[1]);
2041 instr->type = INSTR_HDR_INVALIDATE;
2042 instr->valid.header_id = h->id;
2047 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2049 struct thread *t = &p->threads[p->thread_id];
2050 struct instruction *ip = t->ip;
2052 __instr_hdr_invalidate_exec(p, t, ip);
2061 static struct table *
2062 table_find(struct rte_swx_pipeline *p, const char *name);
2064 static struct selector *
2065 selector_find(struct rte_swx_pipeline *p, const char *name);
2067 static struct learner *
2068 learner_find(struct rte_swx_pipeline *p, const char *name);
2071 instr_table_translate(struct rte_swx_pipeline *p,
2072 struct action *action,
2075 struct instruction *instr,
2076 struct instruction_data *data __rte_unused)
2082 CHECK(!action, EINVAL);
2083 CHECK(n_tokens == 2, EINVAL);
2085 t = table_find(p, tokens[1]);
2087 instr->type = INSTR_TABLE;
2088 instr->table.table_id = t->id;
2092 s = selector_find(p, tokens[1]);
2094 instr->type = INSTR_SELECTOR;
2095 instr->table.table_id = s->id;
2099 l = learner_find(p, tokens[1]);
2101 instr->type = INSTR_LEARNER;
2102 instr->table.table_id = l->id;
2110 instr_table_exec(struct rte_swx_pipeline *p)
2112 struct thread *t = &p->threads[p->thread_id];
2113 struct instruction *ip = t->ip;
2114 uint32_t table_id = ip->table.table_id;
2115 struct rte_swx_table_state *ts = &t->table_state[table_id];
2116 struct table_runtime *table = &t->tables[table_id];
2117 struct table_statistics *stats = &p->table_stats[table_id];
2118 uint64_t action_id, n_pkts_hit, n_pkts_action;
2119 uint8_t *action_data;
2123 done = table->func(ts->obj,
2131 TRACE("[Thread %2u] table %u (not finalized)\n",
2139 action_id = hit ? action_id : ts->default_action_id;
2140 action_data = hit ? action_data : ts->default_action_data;
2141 n_pkts_hit = stats->n_pkts_hit[hit];
2142 n_pkts_action = stats->n_pkts_action[action_id];
2144 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2147 hit ? "hit" : "miss",
2148 (uint32_t)action_id);
2150 t->action_id = action_id;
2151 t->structs[0] = action_data;
2153 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2154 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2157 thread_ip_action_call(p, t, action_id);
2161 instr_table_af_exec(struct rte_swx_pipeline *p)
2163 struct thread *t = &p->threads[p->thread_id];
2164 struct instruction *ip = t->ip;
2165 uint32_t table_id = ip->table.table_id;
2166 struct rte_swx_table_state *ts = &t->table_state[table_id];
2167 struct table_runtime *table = &t->tables[table_id];
2168 struct table_statistics *stats = &p->table_stats[table_id];
2169 uint64_t action_id, n_pkts_hit, n_pkts_action;
2170 uint8_t *action_data;
2171 action_func_t action_func;
2175 done = table->func(ts->obj,
2183 TRACE("[Thread %2u] table %u (not finalized)\n",
2191 action_id = hit ? action_id : ts->default_action_id;
2192 action_data = hit ? action_data : ts->default_action_data;
2193 action_func = p->action_funcs[action_id];
2194 n_pkts_hit = stats->n_pkts_hit[hit];
2195 n_pkts_action = stats->n_pkts_action[action_id];
2197 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2200 hit ? "hit" : "miss",
2201 (uint32_t)action_id);
2203 t->action_id = action_id;
2204 t->structs[0] = action_data;
2206 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2207 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2217 instr_selector_exec(struct rte_swx_pipeline *p)
2219 struct thread *t = &p->threads[p->thread_id];
2220 struct instruction *ip = t->ip;
2221 uint32_t selector_id = ip->table.table_id;
2222 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2223 struct selector_runtime *selector = &t->selectors[selector_id];
2224 struct selector_statistics *stats = &p->selector_stats[selector_id];
2225 uint64_t n_pkts = stats->n_pkts;
2229 done = rte_swx_table_selector_select(ts->obj,
2231 selector->group_id_buffer,
2232 selector->selector_buffer,
2233 selector->member_id_buffer);
2236 TRACE("[Thread %2u] selector %u (not finalized)\n",
2245 TRACE("[Thread %2u] selector %u\n",
2249 stats->n_pkts = n_pkts + 1;
2256 instr_learner_exec(struct rte_swx_pipeline *p)
2258 struct thread *t = &p->threads[p->thread_id];
2259 struct instruction *ip = t->ip;
2260 uint32_t learner_id = ip->table.table_id;
2261 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2262 p->n_selectors + learner_id];
2263 struct learner_runtime *l = &t->learners[learner_id];
2264 struct learner_statistics *stats = &p->learner_stats[learner_id];
2265 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2266 uint8_t *action_data;
2270 time = rte_get_tsc_cycles();
2272 done = rte_swx_table_learner_lookup(ts->obj,
2281 TRACE("[Thread %2u] learner %u (not finalized)\n",
2289 action_id = hit ? action_id : ts->default_action_id;
2290 action_data = hit ? action_data : ts->default_action_data;
2291 n_pkts_hit = stats->n_pkts_hit[hit];
2292 n_pkts_action = stats->n_pkts_action[action_id];
2294 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2297 hit ? "hit" : "miss",
2298 (uint32_t)action_id);
2300 t->action_id = action_id;
2301 t->structs[0] = action_data;
2303 t->learner_id = learner_id;
2305 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2306 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2309 thread_ip_action_call(p, t, action_id);
2313 instr_learner_af_exec(struct rte_swx_pipeline *p)
2315 struct thread *t = &p->threads[p->thread_id];
2316 struct instruction *ip = t->ip;
2317 uint32_t learner_id = ip->table.table_id;
2318 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2319 p->n_selectors + learner_id];
2320 struct learner_runtime *l = &t->learners[learner_id];
2321 struct learner_statistics *stats = &p->learner_stats[learner_id];
2322 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2323 uint8_t *action_data;
2324 action_func_t action_func;
2328 time = rte_get_tsc_cycles();
2330 done = rte_swx_table_learner_lookup(ts->obj,
2339 TRACE("[Thread %2u] learner %u (not finalized)\n",
2347 action_id = hit ? action_id : ts->default_action_id;
2348 action_data = hit ? action_data : ts->default_action_data;
2349 action_func = p->action_funcs[action_id];
2350 n_pkts_hit = stats->n_pkts_hit[hit];
2351 n_pkts_action = stats->n_pkts_action[action_id];
2353 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2356 hit ? "hit" : "miss",
2357 (uint32_t)action_id);
2359 t->action_id = action_id;
2360 t->structs[0] = action_data;
2362 t->learner_id = learner_id;
2364 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2365 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2368 thread_ip_action_call(p, t, action_id);
2377 static struct action *
2378 action_find(struct rte_swx_pipeline *p, const char *name);
2381 action_has_nbo_args(struct action *a);
2384 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2387 instr_learn_translate(struct rte_swx_pipeline *p,
2388 struct action *action,
2391 struct instruction *instr,
2392 struct instruction_data *data __rte_unused)
2395 const char *mf_name;
2396 uint32_t mf_offset = 0;
2398 CHECK(action, EINVAL);
2399 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2401 a = action_find(p, tokens[1]);
2403 CHECK(!action_has_nbo_args(a), EINVAL);
2405 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2406 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2411 mf = metadata_field_parse(p, mf_name);
2414 mf_offset = mf->offset / 8;
2417 instr->type = INSTR_LEARNER_LEARN;
2418 instr->learn.action_id = a->id;
2419 instr->learn.mf_offset = mf_offset;
2425 instr_learn_exec(struct rte_swx_pipeline *p)
2427 struct thread *t = &p->threads[p->thread_id];
2428 struct instruction *ip = t->ip;
2430 __instr_learn_exec(p, t, ip);
2440 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2441 struct action *action,
2442 char **tokens __rte_unused,
2444 struct instruction *instr,
2445 struct instruction_data *data __rte_unused)
2447 CHECK(action, EINVAL);
2448 CHECK(n_tokens == 1, EINVAL);
2450 instr->type = INSTR_LEARNER_FORGET;
2456 instr_forget_exec(struct rte_swx_pipeline *p)
2458 struct thread *t = &p->threads[p->thread_id];
2459 struct instruction *ip = t->ip;
2461 __instr_forget_exec(p, t, ip);
2471 instr_extern_translate(struct rte_swx_pipeline *p,
2472 struct action *action __rte_unused,
2475 struct instruction *instr,
2476 struct instruction_data *data __rte_unused)
2478 char *token = tokens[1];
2480 CHECK(n_tokens == 2, EINVAL);
2482 if (token[0] == 'e') {
2483 struct extern_obj *obj;
2484 struct extern_type_member_func *func;
2486 func = extern_obj_member_func_parse(p, token, &obj);
2487 CHECK(func, EINVAL);
2489 instr->type = INSTR_EXTERN_OBJ;
2490 instr->ext_obj.ext_obj_id = obj->id;
2491 instr->ext_obj.func_id = func->id;
2496 if (token[0] == 'f') {
2497 struct extern_func *func;
2499 func = extern_func_parse(p, token);
2500 CHECK(func, EINVAL);
2502 instr->type = INSTR_EXTERN_FUNC;
2503 instr->ext_func.ext_func_id = func->id;
2512 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2514 struct thread *t = &p->threads[p->thread_id];
2515 struct instruction *ip = t->ip;
2518 /* Extern object member function execute. */
2519 done = __instr_extern_obj_exec(p, t, ip);
2522 thread_ip_inc_cond(t, done);
2523 thread_yield_cond(p, done ^ 1);
2527 instr_extern_func_exec(struct rte_swx_pipeline *p)
2529 struct thread *t = &p->threads[p->thread_id];
2530 struct instruction *ip = t->ip;
2533 /* Extern function execute. */
2534 done = __instr_extern_func_exec(p, t, ip);
2537 thread_ip_inc_cond(t, done);
2538 thread_yield_cond(p, done ^ 1);
2545 instr_mov_translate(struct rte_swx_pipeline *p,
2546 struct action *action,
2549 struct instruction *instr,
2550 struct instruction_data *data __rte_unused)
2552 char *dst = tokens[1], *src = tokens[2];
2553 struct field *fdst, *fsrc;
2555 uint32_t dst_struct_id = 0, src_struct_id = 0;
2557 CHECK(n_tokens == 3, EINVAL);
2559 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2560 CHECK(fdst, EINVAL);
2561 CHECK(!fdst->var_size, EINVAL);
2563 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2564 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2566 CHECK(!fsrc->var_size, EINVAL);
2568 instr->type = INSTR_MOV;
2569 if (dst[0] != 'h' && src[0] == 'h')
2570 instr->type = INSTR_MOV_MH;
2571 if (dst[0] == 'h' && src[0] != 'h')
2572 instr->type = INSTR_MOV_HM;
2573 if (dst[0] == 'h' && src[0] == 'h')
2574 instr->type = INSTR_MOV_HH;
2576 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2577 instr->mov.dst.n_bits = fdst->n_bits;
2578 instr->mov.dst.offset = fdst->offset / 8;
2579 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2580 instr->mov.src.n_bits = fsrc->n_bits;
2581 instr->mov.src.offset = fsrc->offset / 8;
2586 src_val = strtoull(src, &src, 0);
2587 CHECK(!src[0], EINVAL);
2590 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2592 instr->type = INSTR_MOV_I;
2593 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2594 instr->mov.dst.n_bits = fdst->n_bits;
2595 instr->mov.dst.offset = fdst->offset / 8;
2596 instr->mov.src_val = src_val;
2601 instr_mov_exec(struct rte_swx_pipeline *p)
2603 struct thread *t = &p->threads[p->thread_id];
2604 struct instruction *ip = t->ip;
2606 __instr_mov_exec(p, t, ip);
2613 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2615 struct thread *t = &p->threads[p->thread_id];
2616 struct instruction *ip = t->ip;
2618 __instr_mov_mh_exec(p, t, ip);
2625 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2627 struct thread *t = &p->threads[p->thread_id];
2628 struct instruction *ip = t->ip;
2630 __instr_mov_hm_exec(p, t, ip);
2637 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2639 struct thread *t = &p->threads[p->thread_id];
2640 struct instruction *ip = t->ip;
2642 __instr_mov_hh_exec(p, t, ip);
2649 instr_mov_i_exec(struct rte_swx_pipeline *p)
2651 struct thread *t = &p->threads[p->thread_id];
2652 struct instruction *ip = t->ip;
2654 __instr_mov_i_exec(p, t, ip);
2664 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2666 struct thread *t = &p->threads[p->thread_id];
2667 struct instruction *ip = t->ip;
2669 __instr_dma_ht_exec(p, t, ip);
2676 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2678 struct thread *t = &p->threads[p->thread_id];
2679 struct instruction *ip = t->ip;
2681 __instr_dma_ht2_exec(p, t, ip);
2688 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2690 struct thread *t = &p->threads[p->thread_id];
2691 struct instruction *ip = t->ip;
2693 __instr_dma_ht3_exec(p, t, ip);
2700 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2702 struct thread *t = &p->threads[p->thread_id];
2703 struct instruction *ip = t->ip;
2705 __instr_dma_ht4_exec(p, t, ip);
2712 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2714 struct thread *t = &p->threads[p->thread_id];
2715 struct instruction *ip = t->ip;
2717 __instr_dma_ht5_exec(p, t, ip);
2724 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2726 struct thread *t = &p->threads[p->thread_id];
2727 struct instruction *ip = t->ip;
2729 __instr_dma_ht6_exec(p, t, ip);
2736 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2738 struct thread *t = &p->threads[p->thread_id];
2739 struct instruction *ip = t->ip;
2741 __instr_dma_ht7_exec(p, t, ip);
2748 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2750 struct thread *t = &p->threads[p->thread_id];
2751 struct instruction *ip = t->ip;
2753 __instr_dma_ht8_exec(p, t, ip);
2763 instr_alu_add_translate(struct rte_swx_pipeline *p,
2764 struct action *action,
2767 struct instruction *instr,
2768 struct instruction_data *data __rte_unused)
2770 char *dst = tokens[1], *src = tokens[2];
2771 struct field *fdst, *fsrc;
2773 uint32_t dst_struct_id = 0, src_struct_id = 0;
2775 CHECK(n_tokens == 3, EINVAL);
2777 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2778 CHECK(fdst, EINVAL);
2779 CHECK(!fdst->var_size, EINVAL);
2781 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2782 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2784 CHECK(!fsrc->var_size, EINVAL);
2786 instr->type = INSTR_ALU_ADD;
2787 if (dst[0] == 'h' && src[0] != 'h')
2788 instr->type = INSTR_ALU_ADD_HM;
2789 if (dst[0] != 'h' && src[0] == 'h')
2790 instr->type = INSTR_ALU_ADD_MH;
2791 if (dst[0] == 'h' && src[0] == 'h')
2792 instr->type = INSTR_ALU_ADD_HH;
2794 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2795 instr->alu.dst.n_bits = fdst->n_bits;
2796 instr->alu.dst.offset = fdst->offset / 8;
2797 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2798 instr->alu.src.n_bits = fsrc->n_bits;
2799 instr->alu.src.offset = fsrc->offset / 8;
2803 /* ADD_MI, ADD_HI. */
2804 src_val = strtoull(src, &src, 0);
2805 CHECK(!src[0], EINVAL);
2807 instr->type = INSTR_ALU_ADD_MI;
2809 instr->type = INSTR_ALU_ADD_HI;
2811 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2812 instr->alu.dst.n_bits = fdst->n_bits;
2813 instr->alu.dst.offset = fdst->offset / 8;
2814 instr->alu.src_val = src_val;
2819 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2820 struct action *action,
2823 struct instruction *instr,
2824 struct instruction_data *data __rte_unused)
2826 char *dst = tokens[1], *src = tokens[2];
2827 struct field *fdst, *fsrc;
2829 uint32_t dst_struct_id = 0, src_struct_id = 0;
2831 CHECK(n_tokens == 3, EINVAL);
2833 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2834 CHECK(fdst, EINVAL);
2835 CHECK(!fdst->var_size, EINVAL);
2837 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2838 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2840 CHECK(!fsrc->var_size, EINVAL);
2842 instr->type = INSTR_ALU_SUB;
2843 if (dst[0] == 'h' && src[0] != 'h')
2844 instr->type = INSTR_ALU_SUB_HM;
2845 if (dst[0] != 'h' && src[0] == 'h')
2846 instr->type = INSTR_ALU_SUB_MH;
2847 if (dst[0] == 'h' && src[0] == 'h')
2848 instr->type = INSTR_ALU_SUB_HH;
2850 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2851 instr->alu.dst.n_bits = fdst->n_bits;
2852 instr->alu.dst.offset = fdst->offset / 8;
2853 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2854 instr->alu.src.n_bits = fsrc->n_bits;
2855 instr->alu.src.offset = fsrc->offset / 8;
2859 /* SUB_MI, SUB_HI. */
2860 src_val = strtoull(src, &src, 0);
2861 CHECK(!src[0], EINVAL);
2863 instr->type = INSTR_ALU_SUB_MI;
2865 instr->type = INSTR_ALU_SUB_HI;
2867 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2868 instr->alu.dst.n_bits = fdst->n_bits;
2869 instr->alu.dst.offset = fdst->offset / 8;
2870 instr->alu.src_val = src_val;
2875 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2876 struct action *action __rte_unused,
2879 struct instruction *instr,
2880 struct instruction_data *data __rte_unused)
2882 char *dst = tokens[1], *src = tokens[2];
2883 struct header *hdst, *hsrc;
2884 struct field *fdst, *fsrc;
2886 CHECK(n_tokens == 3, EINVAL);
2888 fdst = header_field_parse(p, dst, &hdst);
2889 CHECK(fdst, EINVAL);
2890 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
2893 fsrc = header_field_parse(p, src, &hsrc);
2895 CHECK(!fsrc->var_size, EINVAL);
2897 instr->type = INSTR_ALU_CKADD_FIELD;
2898 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2899 instr->alu.dst.n_bits = fdst->n_bits;
2900 instr->alu.dst.offset = fdst->offset / 8;
2901 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2902 instr->alu.src.n_bits = fsrc->n_bits;
2903 instr->alu.src.offset = fsrc->offset / 8;
2907 /* CKADD_STRUCT, CKADD_STRUCT20. */
2908 hsrc = header_parse(p, src);
2909 CHECK(hsrc, EINVAL);
2911 instr->type = INSTR_ALU_CKADD_STRUCT;
2912 if (!hsrc->st->var_size && ((hsrc->st->n_bits / 8) == 20))
2913 instr->type = INSTR_ALU_CKADD_STRUCT20;
2915 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2916 instr->alu.dst.n_bits = fdst->n_bits;
2917 instr->alu.dst.offset = fdst->offset / 8;
2918 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2919 instr->alu.src.n_bits = (uint8_t)hsrc->id; /* The src header ID is stored here. */
2920 instr->alu.src.offset = 0; /* Unused. */
2925 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2926 struct action *action __rte_unused,
2929 struct instruction *instr,
2930 struct instruction_data *data __rte_unused)
2932 char *dst = tokens[1], *src = tokens[2];
2933 struct header *hdst, *hsrc;
2934 struct field *fdst, *fsrc;
2936 CHECK(n_tokens == 3, EINVAL);
2938 fdst = header_field_parse(p, dst, &hdst);
2939 CHECK(fdst, EINVAL);
2940 CHECK(!fdst->var_size && (fdst->n_bits == 16), EINVAL);
2942 fsrc = header_field_parse(p, src, &hsrc);
2943 CHECK(fsrc, EINVAL);
2944 CHECK(!fsrc->var_size, EINVAL);
2946 instr->type = INSTR_ALU_CKSUB_FIELD;
2947 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2948 instr->alu.dst.n_bits = fdst->n_bits;
2949 instr->alu.dst.offset = fdst->offset / 8;
2950 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2951 instr->alu.src.n_bits = fsrc->n_bits;
2952 instr->alu.src.offset = fsrc->offset / 8;
2957 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2958 struct action *action,
2961 struct instruction *instr,
2962 struct instruction_data *data __rte_unused)
2964 char *dst = tokens[1], *src = tokens[2];
2965 struct field *fdst, *fsrc;
2967 uint32_t dst_struct_id = 0, src_struct_id = 0;
2969 CHECK(n_tokens == 3, EINVAL);
2971 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2972 CHECK(fdst, EINVAL);
2973 CHECK(!fdst->var_size, EINVAL);
2975 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2976 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2978 CHECK(!fsrc->var_size, EINVAL);
2980 instr->type = INSTR_ALU_SHL;
2981 if (dst[0] == 'h' && src[0] != 'h')
2982 instr->type = INSTR_ALU_SHL_HM;
2983 if (dst[0] != 'h' && src[0] == 'h')
2984 instr->type = INSTR_ALU_SHL_MH;
2985 if (dst[0] == 'h' && src[0] == 'h')
2986 instr->type = INSTR_ALU_SHL_HH;
2988 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2989 instr->alu.dst.n_bits = fdst->n_bits;
2990 instr->alu.dst.offset = fdst->offset / 8;
2991 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2992 instr->alu.src.n_bits = fsrc->n_bits;
2993 instr->alu.src.offset = fsrc->offset / 8;
2997 /* SHL_MI, SHL_HI. */
2998 src_val = strtoull(src, &src, 0);
2999 CHECK(!src[0], EINVAL);
3001 instr->type = INSTR_ALU_SHL_MI;
3003 instr->type = INSTR_ALU_SHL_HI;
3005 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3006 instr->alu.dst.n_bits = fdst->n_bits;
3007 instr->alu.dst.offset = fdst->offset / 8;
3008 instr->alu.src_val = src_val;
3013 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3014 struct action *action,
3017 struct instruction *instr,
3018 struct instruction_data *data __rte_unused)
3020 char *dst = tokens[1], *src = tokens[2];
3021 struct field *fdst, *fsrc;
3023 uint32_t dst_struct_id = 0, src_struct_id = 0;
3025 CHECK(n_tokens == 3, EINVAL);
3027 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3028 CHECK(fdst, EINVAL);
3029 CHECK(!fdst->var_size, EINVAL);
3031 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3032 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3034 CHECK(!fsrc->var_size, EINVAL);
3036 instr->type = INSTR_ALU_SHR;
3037 if (dst[0] == 'h' && src[0] != 'h')
3038 instr->type = INSTR_ALU_SHR_HM;
3039 if (dst[0] != 'h' && src[0] == 'h')
3040 instr->type = INSTR_ALU_SHR_MH;
3041 if (dst[0] == 'h' && src[0] == 'h')
3042 instr->type = INSTR_ALU_SHR_HH;
3044 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3045 instr->alu.dst.n_bits = fdst->n_bits;
3046 instr->alu.dst.offset = fdst->offset / 8;
3047 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3048 instr->alu.src.n_bits = fsrc->n_bits;
3049 instr->alu.src.offset = fsrc->offset / 8;
3053 /* SHR_MI, SHR_HI. */
3054 src_val = strtoull(src, &src, 0);
3055 CHECK(!src[0], EINVAL);
3057 instr->type = INSTR_ALU_SHR_MI;
3059 instr->type = INSTR_ALU_SHR_HI;
3061 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3062 instr->alu.dst.n_bits = fdst->n_bits;
3063 instr->alu.dst.offset = fdst->offset / 8;
3064 instr->alu.src_val = src_val;
3069 instr_alu_and_translate(struct rte_swx_pipeline *p,
3070 struct action *action,
3073 struct instruction *instr,
3074 struct instruction_data *data __rte_unused)
3076 char *dst = tokens[1], *src = tokens[2];
3077 struct field *fdst, *fsrc;
3079 uint32_t dst_struct_id = 0, src_struct_id = 0;
3081 CHECK(n_tokens == 3, EINVAL);
3083 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3084 CHECK(fdst, EINVAL);
3085 CHECK(!fdst->var_size, EINVAL);
3087 /* AND, AND_MH, AND_HM, AND_HH. */
3088 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3090 CHECK(!fsrc->var_size, EINVAL);
3092 instr->type = INSTR_ALU_AND;
3093 if (dst[0] != 'h' && src[0] == 'h')
3094 instr->type = INSTR_ALU_AND_MH;
3095 if (dst[0] == 'h' && src[0] != 'h')
3096 instr->type = INSTR_ALU_AND_HM;
3097 if (dst[0] == 'h' && src[0] == 'h')
3098 instr->type = INSTR_ALU_AND_HH;
3100 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3101 instr->alu.dst.n_bits = fdst->n_bits;
3102 instr->alu.dst.offset = fdst->offset / 8;
3103 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3104 instr->alu.src.n_bits = fsrc->n_bits;
3105 instr->alu.src.offset = fsrc->offset / 8;
3110 src_val = strtoull(src, &src, 0);
3111 CHECK(!src[0], EINVAL);
3114 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3116 instr->type = INSTR_ALU_AND_I;
3117 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3118 instr->alu.dst.n_bits = fdst->n_bits;
3119 instr->alu.dst.offset = fdst->offset / 8;
3120 instr->alu.src_val = src_val;
3125 instr_alu_or_translate(struct rte_swx_pipeline *p,
3126 struct action *action,
3129 struct instruction *instr,
3130 struct instruction_data *data __rte_unused)
3132 char *dst = tokens[1], *src = tokens[2];
3133 struct field *fdst, *fsrc;
3135 uint32_t dst_struct_id = 0, src_struct_id = 0;
3137 CHECK(n_tokens == 3, EINVAL);
3139 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3140 CHECK(fdst, EINVAL);
3141 CHECK(!fdst->var_size, EINVAL);
3143 /* OR, OR_MH, OR_HM, OR_HH. */
3144 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3146 CHECK(!fsrc->var_size, EINVAL);
3148 instr->type = INSTR_ALU_OR;
3149 if (dst[0] != 'h' && src[0] == 'h')
3150 instr->type = INSTR_ALU_OR_MH;
3151 if (dst[0] == 'h' && src[0] != 'h')
3152 instr->type = INSTR_ALU_OR_HM;
3153 if (dst[0] == 'h' && src[0] == 'h')
3154 instr->type = INSTR_ALU_OR_HH;
3156 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3157 instr->alu.dst.n_bits = fdst->n_bits;
3158 instr->alu.dst.offset = fdst->offset / 8;
3159 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3160 instr->alu.src.n_bits = fsrc->n_bits;
3161 instr->alu.src.offset = fsrc->offset / 8;
3166 src_val = strtoull(src, &src, 0);
3167 CHECK(!src[0], EINVAL);
3170 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3172 instr->type = INSTR_ALU_OR_I;
3173 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3174 instr->alu.dst.n_bits = fdst->n_bits;
3175 instr->alu.dst.offset = fdst->offset / 8;
3176 instr->alu.src_val = src_val;
3181 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3182 struct action *action,
3185 struct instruction *instr,
3186 struct instruction_data *data __rte_unused)
3188 char *dst = tokens[1], *src = tokens[2];
3189 struct field *fdst, *fsrc;
3191 uint32_t dst_struct_id = 0, src_struct_id = 0;
3193 CHECK(n_tokens == 3, EINVAL);
3195 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3196 CHECK(fdst, EINVAL);
3197 CHECK(!fdst->var_size, EINVAL);
3199 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3200 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3202 CHECK(!fsrc->var_size, EINVAL);
3204 instr->type = INSTR_ALU_XOR;
3205 if (dst[0] != 'h' && src[0] == 'h')
3206 instr->type = INSTR_ALU_XOR_MH;
3207 if (dst[0] == 'h' && src[0] != 'h')
3208 instr->type = INSTR_ALU_XOR_HM;
3209 if (dst[0] == 'h' && src[0] == 'h')
3210 instr->type = INSTR_ALU_XOR_HH;
3212 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3213 instr->alu.dst.n_bits = fdst->n_bits;
3214 instr->alu.dst.offset = fdst->offset / 8;
3215 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3216 instr->alu.src.n_bits = fsrc->n_bits;
3217 instr->alu.src.offset = fsrc->offset / 8;
3222 src_val = strtoull(src, &src, 0);
3223 CHECK(!src[0], EINVAL);
3226 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3228 instr->type = INSTR_ALU_XOR_I;
3229 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3230 instr->alu.dst.n_bits = fdst->n_bits;
3231 instr->alu.dst.offset = fdst->offset / 8;
3232 instr->alu.src_val = src_val;
3237 instr_alu_add_exec(struct rte_swx_pipeline *p)
3239 struct thread *t = &p->threads[p->thread_id];
3240 struct instruction *ip = t->ip;
3243 __instr_alu_add_exec(p, t, ip);
3250 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3252 struct thread *t = &p->threads[p->thread_id];
3253 struct instruction *ip = t->ip;
3256 __instr_alu_add_mh_exec(p, t, ip);
3263 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3265 struct thread *t = &p->threads[p->thread_id];
3266 struct instruction *ip = t->ip;
3269 __instr_alu_add_hm_exec(p, t, ip);
3276 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3278 struct thread *t = &p->threads[p->thread_id];
3279 struct instruction *ip = t->ip;
3282 __instr_alu_add_hh_exec(p, t, ip);
3289 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3291 struct thread *t = &p->threads[p->thread_id];
3292 struct instruction *ip = t->ip;
3295 __instr_alu_add_mi_exec(p, t, ip);
3302 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3304 struct thread *t = &p->threads[p->thread_id];
3305 struct instruction *ip = t->ip;
3308 __instr_alu_add_hi_exec(p, t, ip);
3315 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3317 struct thread *t = &p->threads[p->thread_id];
3318 struct instruction *ip = t->ip;
3321 __instr_alu_sub_exec(p, t, ip);
3328 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3330 struct thread *t = &p->threads[p->thread_id];
3331 struct instruction *ip = t->ip;
3334 __instr_alu_sub_mh_exec(p, t, ip);
3341 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3343 struct thread *t = &p->threads[p->thread_id];
3344 struct instruction *ip = t->ip;
3347 __instr_alu_sub_hm_exec(p, t, ip);
3354 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3356 struct thread *t = &p->threads[p->thread_id];
3357 struct instruction *ip = t->ip;
3360 __instr_alu_sub_hh_exec(p, t, ip);
3367 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3369 struct thread *t = &p->threads[p->thread_id];
3370 struct instruction *ip = t->ip;
3373 __instr_alu_sub_mi_exec(p, t, ip);
3380 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3382 struct thread *t = &p->threads[p->thread_id];
3383 struct instruction *ip = t->ip;
3386 __instr_alu_sub_hi_exec(p, t, ip);
3393 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3395 struct thread *t = &p->threads[p->thread_id];
3396 struct instruction *ip = t->ip;
3399 __instr_alu_shl_exec(p, t, ip);
3406 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3408 struct thread *t = &p->threads[p->thread_id];
3409 struct instruction *ip = t->ip;
3412 __instr_alu_shl_mh_exec(p, t, ip);
3419 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3421 struct thread *t = &p->threads[p->thread_id];
3422 struct instruction *ip = t->ip;
3425 __instr_alu_shl_hm_exec(p, t, ip);
3432 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3434 struct thread *t = &p->threads[p->thread_id];
3435 struct instruction *ip = t->ip;
3438 __instr_alu_shl_hh_exec(p, t, ip);
3445 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3447 struct thread *t = &p->threads[p->thread_id];
3448 struct instruction *ip = t->ip;
3451 __instr_alu_shl_mi_exec(p, t, ip);
3458 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3460 struct thread *t = &p->threads[p->thread_id];
3461 struct instruction *ip = t->ip;
3464 __instr_alu_shl_hi_exec(p, t, ip);
3471 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3473 struct thread *t = &p->threads[p->thread_id];
3474 struct instruction *ip = t->ip;
3477 __instr_alu_shr_exec(p, t, ip);
3484 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3486 struct thread *t = &p->threads[p->thread_id];
3487 struct instruction *ip = t->ip;
3490 __instr_alu_shr_mh_exec(p, t, ip);
3497 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3499 struct thread *t = &p->threads[p->thread_id];
3500 struct instruction *ip = t->ip;
3503 __instr_alu_shr_hm_exec(p, t, ip);
3510 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3512 struct thread *t = &p->threads[p->thread_id];
3513 struct instruction *ip = t->ip;
3516 __instr_alu_shr_hh_exec(p, t, ip);
3523 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3525 struct thread *t = &p->threads[p->thread_id];
3526 struct instruction *ip = t->ip;
3529 __instr_alu_shr_mi_exec(p, t, ip);
3536 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3538 struct thread *t = &p->threads[p->thread_id];
3539 struct instruction *ip = t->ip;
3542 __instr_alu_shr_hi_exec(p, t, ip);
3549 instr_alu_and_exec(struct rte_swx_pipeline *p)
3551 struct thread *t = &p->threads[p->thread_id];
3552 struct instruction *ip = t->ip;
3555 __instr_alu_and_exec(p, t, ip);
3562 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3564 struct thread *t = &p->threads[p->thread_id];
3565 struct instruction *ip = t->ip;
3568 __instr_alu_and_mh_exec(p, t, ip);
3575 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3577 struct thread *t = &p->threads[p->thread_id];
3578 struct instruction *ip = t->ip;
3581 __instr_alu_and_hm_exec(p, t, ip);
3588 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3590 struct thread *t = &p->threads[p->thread_id];
3591 struct instruction *ip = t->ip;
3594 __instr_alu_and_hh_exec(p, t, ip);
3601 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3603 struct thread *t = &p->threads[p->thread_id];
3604 struct instruction *ip = t->ip;
3607 __instr_alu_and_i_exec(p, t, ip);
3614 instr_alu_or_exec(struct rte_swx_pipeline *p)
3616 struct thread *t = &p->threads[p->thread_id];
3617 struct instruction *ip = t->ip;
3620 __instr_alu_or_exec(p, t, ip);
3627 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3629 struct thread *t = &p->threads[p->thread_id];
3630 struct instruction *ip = t->ip;
3633 __instr_alu_or_mh_exec(p, t, ip);
3640 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3642 struct thread *t = &p->threads[p->thread_id];
3643 struct instruction *ip = t->ip;
3646 __instr_alu_or_hm_exec(p, t, ip);
3653 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3655 struct thread *t = &p->threads[p->thread_id];
3656 struct instruction *ip = t->ip;
3659 __instr_alu_or_hh_exec(p, t, ip);
3666 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3668 struct thread *t = &p->threads[p->thread_id];
3669 struct instruction *ip = t->ip;
3672 __instr_alu_or_i_exec(p, t, ip);
3679 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3681 struct thread *t = &p->threads[p->thread_id];
3682 struct instruction *ip = t->ip;
3685 __instr_alu_xor_exec(p, t, ip);
3692 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3694 struct thread *t = &p->threads[p->thread_id];
3695 struct instruction *ip = t->ip;
3698 __instr_alu_xor_mh_exec(p, t, ip);
3705 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3707 struct thread *t = &p->threads[p->thread_id];
3708 struct instruction *ip = t->ip;
3711 __instr_alu_xor_hm_exec(p, t, ip);
3718 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3720 struct thread *t = &p->threads[p->thread_id];
3721 struct instruction *ip = t->ip;
3724 __instr_alu_xor_hh_exec(p, t, ip);
3731 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3733 struct thread *t = &p->threads[p->thread_id];
3734 struct instruction *ip = t->ip;
3737 __instr_alu_xor_i_exec(p, t, ip);
3744 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3746 struct thread *t = &p->threads[p->thread_id];
3747 struct instruction *ip = t->ip;
3750 __instr_alu_ckadd_field_exec(p, t, ip);
3757 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3759 struct thread *t = &p->threads[p->thread_id];
3760 struct instruction *ip = t->ip;
3763 __instr_alu_cksub_field_exec(p, t, ip);
3770 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3772 struct thread *t = &p->threads[p->thread_id];
3773 struct instruction *ip = t->ip;
3776 __instr_alu_ckadd_struct20_exec(p, t, ip);
3783 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3785 struct thread *t = &p->threads[p->thread_id];
3786 struct instruction *ip = t->ip;
3789 __instr_alu_ckadd_struct_exec(p, t, ip);
3798 static struct regarray *
3799 regarray_find(struct rte_swx_pipeline *p, const char *name);
3802 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3803 struct action *action,
3806 struct instruction *instr,
3807 struct instruction_data *data __rte_unused)
3809 char *regarray = tokens[1], *idx = tokens[2];
3812 uint32_t idx_struct_id, idx_val;
3814 CHECK(n_tokens == 3, EINVAL);
3816 r = regarray_find(p, regarray);
3819 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3820 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3822 CHECK(!fidx->var_size, EINVAL);
3824 instr->type = INSTR_REGPREFETCH_RM;
3826 instr->type = INSTR_REGPREFETCH_RH;
3828 instr->regarray.regarray_id = r->id;
3829 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3830 instr->regarray.idx.n_bits = fidx->n_bits;
3831 instr->regarray.idx.offset = fidx->offset / 8;
3832 instr->regarray.dstsrc_val = 0; /* Unused. */
3836 /* REGPREFETCH_RI. */
3837 idx_val = strtoul(idx, &idx, 0);
3838 CHECK(!idx[0], EINVAL);
3840 instr->type = INSTR_REGPREFETCH_RI;
3841 instr->regarray.regarray_id = r->id;
3842 instr->regarray.idx_val = idx_val;
3843 instr->regarray.dstsrc_val = 0; /* Unused. */
3848 instr_regrd_translate(struct rte_swx_pipeline *p,
3849 struct action *action,
3852 struct instruction *instr,
3853 struct instruction_data *data __rte_unused)
3855 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3857 struct field *fdst, *fidx;
3858 uint32_t dst_struct_id, idx_struct_id, idx_val;
3860 CHECK(n_tokens == 4, EINVAL);
3862 r = regarray_find(p, regarray);
3865 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3866 CHECK(fdst, EINVAL);
3867 CHECK(!fdst->var_size, EINVAL);
3869 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3870 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3872 CHECK(!fidx->var_size, EINVAL);
3874 instr->type = INSTR_REGRD_MRM;
3875 if (dst[0] == 'h' && idx[0] != 'h')
3876 instr->type = INSTR_REGRD_HRM;
3877 if (dst[0] != 'h' && idx[0] == 'h')
3878 instr->type = INSTR_REGRD_MRH;
3879 if (dst[0] == 'h' && idx[0] == 'h')
3880 instr->type = INSTR_REGRD_HRH;
3882 instr->regarray.regarray_id = r->id;
3883 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3884 instr->regarray.idx.n_bits = fidx->n_bits;
3885 instr->regarray.idx.offset = fidx->offset / 8;
3886 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3887 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3888 instr->regarray.dstsrc.offset = fdst->offset / 8;
3892 /* REGRD_MRI, REGRD_HRI. */
3893 idx_val = strtoul(idx, &idx, 0);
3894 CHECK(!idx[0], EINVAL);
3896 instr->type = INSTR_REGRD_MRI;
3898 instr->type = INSTR_REGRD_HRI;
3900 instr->regarray.regarray_id = r->id;
3901 instr->regarray.idx_val = idx_val;
3902 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3903 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3904 instr->regarray.dstsrc.offset = fdst->offset / 8;
3909 instr_regwr_translate(struct rte_swx_pipeline *p,
3910 struct action *action,
3913 struct instruction *instr,
3914 struct instruction_data *data __rte_unused)
3916 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3918 struct field *fidx, *fsrc;
3920 uint32_t idx_struct_id, idx_val, src_struct_id;
3922 CHECK(n_tokens == 4, EINVAL);
3924 r = regarray_find(p, regarray);
3927 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3928 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3929 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3931 CHECK(!fidx->var_size, EINVAL);
3932 CHECK(!fsrc->var_size, EINVAL);
3934 instr->type = INSTR_REGWR_RMM;
3935 if (idx[0] == 'h' && src[0] != 'h')
3936 instr->type = INSTR_REGWR_RHM;
3937 if (idx[0] != 'h' && src[0] == 'h')
3938 instr->type = INSTR_REGWR_RMH;
3939 if (idx[0] == 'h' && src[0] == 'h')
3940 instr->type = INSTR_REGWR_RHH;
3942 instr->regarray.regarray_id = r->id;
3943 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3944 instr->regarray.idx.n_bits = fidx->n_bits;
3945 instr->regarray.idx.offset = fidx->offset / 8;
3946 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3947 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3948 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3952 /* REGWR_RHI, REGWR_RMI. */
3953 if (fidx && !fsrc) {
3954 CHECK(!fidx->var_size, EINVAL);
3956 src_val = strtoull(src, &src, 0);
3957 CHECK(!src[0], EINVAL);
3959 instr->type = INSTR_REGWR_RMI;
3961 instr->type = INSTR_REGWR_RHI;
3963 instr->regarray.regarray_id = r->id;
3964 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3965 instr->regarray.idx.n_bits = fidx->n_bits;
3966 instr->regarray.idx.offset = fidx->offset / 8;
3967 instr->regarray.dstsrc_val = src_val;
3971 /* REGWR_RIH, REGWR_RIM. */
3972 if (!fidx && fsrc) {
3973 idx_val = strtoul(idx, &idx, 0);
3974 CHECK(!idx[0], EINVAL);
3976 CHECK(!fsrc->var_size, EINVAL);
3978 instr->type = INSTR_REGWR_RIM;
3980 instr->type = INSTR_REGWR_RIH;
3982 instr->regarray.regarray_id = r->id;
3983 instr->regarray.idx_val = idx_val;
3984 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3985 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3986 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3991 src_val = strtoull(src, &src, 0);
3992 CHECK(!src[0], EINVAL);
3994 idx_val = strtoul(idx, &idx, 0);
3995 CHECK(!idx[0], EINVAL);
3997 instr->type = INSTR_REGWR_RII;
3998 instr->regarray.idx_val = idx_val;
3999 instr->regarray.dstsrc_val = src_val;
4005 instr_regadd_translate(struct rte_swx_pipeline *p,
4006 struct action *action,
4009 struct instruction *instr,
4010 struct instruction_data *data __rte_unused)
4012 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4014 struct field *fidx, *fsrc;
4016 uint32_t idx_struct_id, idx_val, src_struct_id;
4018 CHECK(n_tokens == 4, EINVAL);
4020 r = regarray_find(p, regarray);
4023 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4024 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4025 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4027 CHECK(!fidx->var_size, EINVAL);
4028 CHECK(!fsrc->var_size, EINVAL);
4030 instr->type = INSTR_REGADD_RMM;
4031 if (idx[0] == 'h' && src[0] != 'h')
4032 instr->type = INSTR_REGADD_RHM;
4033 if (idx[0] != 'h' && src[0] == 'h')
4034 instr->type = INSTR_REGADD_RMH;
4035 if (idx[0] == 'h' && src[0] == 'h')
4036 instr->type = INSTR_REGADD_RHH;
4038 instr->regarray.regarray_id = r->id;
4039 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4040 instr->regarray.idx.n_bits = fidx->n_bits;
4041 instr->regarray.idx.offset = fidx->offset / 8;
4042 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4043 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4044 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4048 /* REGADD_RHI, REGADD_RMI. */
4049 if (fidx && !fsrc) {
4050 CHECK(!fidx->var_size, EINVAL);
4052 src_val = strtoull(src, &src, 0);
4053 CHECK(!src[0], EINVAL);
4055 instr->type = INSTR_REGADD_RMI;
4057 instr->type = INSTR_REGADD_RHI;
4059 instr->regarray.regarray_id = r->id;
4060 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4061 instr->regarray.idx.n_bits = fidx->n_bits;
4062 instr->regarray.idx.offset = fidx->offset / 8;
4063 instr->regarray.dstsrc_val = src_val;
4067 /* REGADD_RIH, REGADD_RIM. */
4068 if (!fidx && fsrc) {
4069 idx_val = strtoul(idx, &idx, 0);
4070 CHECK(!idx[0], EINVAL);
4072 CHECK(!fsrc->var_size, EINVAL);
4074 instr->type = INSTR_REGADD_RIM;
4076 instr->type = INSTR_REGADD_RIH;
4078 instr->regarray.regarray_id = r->id;
4079 instr->regarray.idx_val = idx_val;
4080 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4081 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4082 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4087 src_val = strtoull(src, &src, 0);
4088 CHECK(!src[0], EINVAL);
4090 idx_val = strtoul(idx, &idx, 0);
4091 CHECK(!idx[0], EINVAL);
4093 instr->type = INSTR_REGADD_RII;
4094 instr->regarray.idx_val = idx_val;
4095 instr->regarray.dstsrc_val = src_val;
4100 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4102 struct thread *t = &p->threads[p->thread_id];
4103 struct instruction *ip = t->ip;
4106 __instr_regprefetch_rh_exec(p, t, ip);
4113 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4115 struct thread *t = &p->threads[p->thread_id];
4116 struct instruction *ip = t->ip;
4119 __instr_regprefetch_rm_exec(p, t, ip);
4126 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4128 struct thread *t = &p->threads[p->thread_id];
4129 struct instruction *ip = t->ip;
4132 __instr_regprefetch_ri_exec(p, t, ip);
4139 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4141 struct thread *t = &p->threads[p->thread_id];
4142 struct instruction *ip = t->ip;
4145 __instr_regrd_hrh_exec(p, t, ip);
4152 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4154 struct thread *t = &p->threads[p->thread_id];
4155 struct instruction *ip = t->ip;
4158 __instr_regrd_hrm_exec(p, t, ip);
4165 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4167 struct thread *t = &p->threads[p->thread_id];
4168 struct instruction *ip = t->ip;
4171 __instr_regrd_mrh_exec(p, t, ip);
4178 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4180 struct thread *t = &p->threads[p->thread_id];
4181 struct instruction *ip = t->ip;
4184 __instr_regrd_mrm_exec(p, t, ip);
4191 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4193 struct thread *t = &p->threads[p->thread_id];
4194 struct instruction *ip = t->ip;
4197 __instr_regrd_hri_exec(p, t, ip);
4204 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4206 struct thread *t = &p->threads[p->thread_id];
4207 struct instruction *ip = t->ip;
4210 __instr_regrd_mri_exec(p, t, ip);
4217 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4219 struct thread *t = &p->threads[p->thread_id];
4220 struct instruction *ip = t->ip;
4223 __instr_regwr_rhh_exec(p, t, ip);
4230 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4232 struct thread *t = &p->threads[p->thread_id];
4233 struct instruction *ip = t->ip;
4236 __instr_regwr_rhm_exec(p, t, ip);
4243 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4245 struct thread *t = &p->threads[p->thread_id];
4246 struct instruction *ip = t->ip;
4249 __instr_regwr_rmh_exec(p, t, ip);
4256 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4258 struct thread *t = &p->threads[p->thread_id];
4259 struct instruction *ip = t->ip;
4262 __instr_regwr_rmm_exec(p, t, ip);
4269 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4271 struct thread *t = &p->threads[p->thread_id];
4272 struct instruction *ip = t->ip;
4275 __instr_regwr_rhi_exec(p, t, ip);
4282 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4284 struct thread *t = &p->threads[p->thread_id];
4285 struct instruction *ip = t->ip;
4288 __instr_regwr_rmi_exec(p, t, ip);
4295 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4297 struct thread *t = &p->threads[p->thread_id];
4298 struct instruction *ip = t->ip;
4301 __instr_regwr_rih_exec(p, t, ip);
4308 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4310 struct thread *t = &p->threads[p->thread_id];
4311 struct instruction *ip = t->ip;
4314 __instr_regwr_rim_exec(p, t, ip);
4321 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4323 struct thread *t = &p->threads[p->thread_id];
4324 struct instruction *ip = t->ip;
4327 __instr_regwr_rii_exec(p, t, ip);
4334 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4336 struct thread *t = &p->threads[p->thread_id];
4337 struct instruction *ip = t->ip;
4340 __instr_regadd_rhh_exec(p, t, ip);
4347 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4349 struct thread *t = &p->threads[p->thread_id];
4350 struct instruction *ip = t->ip;
4353 __instr_regadd_rhm_exec(p, t, ip);
4360 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4362 struct thread *t = &p->threads[p->thread_id];
4363 struct instruction *ip = t->ip;
4366 __instr_regadd_rmh_exec(p, t, ip);
4373 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4375 struct thread *t = &p->threads[p->thread_id];
4376 struct instruction *ip = t->ip;
4379 __instr_regadd_rmm_exec(p, t, ip);
4386 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4388 struct thread *t = &p->threads[p->thread_id];
4389 struct instruction *ip = t->ip;
4392 __instr_regadd_rhi_exec(p, t, ip);
4399 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4401 struct thread *t = &p->threads[p->thread_id];
4402 struct instruction *ip = t->ip;
4405 __instr_regadd_rmi_exec(p, t, ip);
4412 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4414 struct thread *t = &p->threads[p->thread_id];
4415 struct instruction *ip = t->ip;
4418 __instr_regadd_rih_exec(p, t, ip);
4425 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4427 struct thread *t = &p->threads[p->thread_id];
4428 struct instruction *ip = t->ip;
4431 __instr_regadd_rim_exec(p, t, ip);
4438 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4440 struct thread *t = &p->threads[p->thread_id];
4441 struct instruction *ip = t->ip;
4444 __instr_regadd_rii_exec(p, t, ip);
4453 static struct metarray *
4454 metarray_find(struct rte_swx_pipeline *p, const char *name);
4457 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4458 struct action *action,
4461 struct instruction *instr,
4462 struct instruction_data *data __rte_unused)
4464 char *metarray = tokens[1], *idx = tokens[2];
4467 uint32_t idx_struct_id, idx_val;
4469 CHECK(n_tokens == 3, EINVAL);
4471 m = metarray_find(p, metarray);
4474 /* METPREFETCH_H, METPREFETCH_M. */
4475 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4477 CHECK(!fidx->var_size, EINVAL);
4479 instr->type = INSTR_METPREFETCH_M;
4481 instr->type = INSTR_METPREFETCH_H;
4483 instr->meter.metarray_id = m->id;
4484 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4485 instr->meter.idx.n_bits = fidx->n_bits;
4486 instr->meter.idx.offset = fidx->offset / 8;
4490 /* METPREFETCH_I. */
4491 idx_val = strtoul(idx, &idx, 0);
4492 CHECK(!idx[0], EINVAL);
4494 instr->type = INSTR_METPREFETCH_I;
4495 instr->meter.metarray_id = m->id;
4496 instr->meter.idx_val = idx_val;
4501 instr_meter_translate(struct rte_swx_pipeline *p,
4502 struct action *action,
4505 struct instruction *instr,
4506 struct instruction_data *data __rte_unused)
4508 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4509 char *color_in = tokens[4], *color_out = tokens[5];
4511 struct field *fidx, *flength, *fcin, *fcout;
4512 uint32_t idx_struct_id, length_struct_id;
4513 uint32_t color_in_struct_id, color_out_struct_id;
4515 CHECK(n_tokens == 6, EINVAL);
4517 m = metarray_find(p, metarray);
4520 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4522 flength = struct_field_parse(p, action, length, &length_struct_id);
4523 CHECK(flength, EINVAL);
4524 CHECK(!flength->var_size, EINVAL);
4526 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4528 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4529 CHECK(fcout, EINVAL);
4530 CHECK(!fcout->var_size, EINVAL);
4532 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4534 CHECK(!fidx->var_size, EINVAL);
4535 CHECK(!fcin->var_size, EINVAL);
4537 instr->type = INSTR_METER_MMM;
4538 if (idx[0] == 'h' && length[0] == 'h')
4539 instr->type = INSTR_METER_HHM;
4540 if (idx[0] == 'h' && length[0] != 'h')
4541 instr->type = INSTR_METER_HMM;
4542 if (idx[0] != 'h' && length[0] == 'h')
4543 instr->type = INSTR_METER_MHM;
4545 instr->meter.metarray_id = m->id;
4547 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4548 instr->meter.idx.n_bits = fidx->n_bits;
4549 instr->meter.idx.offset = fidx->offset / 8;
4551 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4552 instr->meter.length.n_bits = flength->n_bits;
4553 instr->meter.length.offset = flength->offset / 8;
4555 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4556 instr->meter.color_in.n_bits = fcin->n_bits;
4557 instr->meter.color_in.offset = fcin->offset / 8;
4559 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4560 instr->meter.color_out.n_bits = fcout->n_bits;
4561 instr->meter.color_out.offset = fcout->offset / 8;
4564 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4565 if (fidx && !fcin) {
4566 uint32_t color_in_val;
4568 CHECK(!fidx->var_size, EINVAL);
4570 color_in_val = strtoul(color_in, &color_in, 0);
4571 CHECK(!color_in[0], EINVAL);
4573 instr->type = INSTR_METER_MMI;
4574 if (idx[0] == 'h' && length[0] == 'h')
4575 instr->type = INSTR_METER_HHI;
4576 if (idx[0] == 'h' && length[0] != 'h')
4577 instr->type = INSTR_METER_HMI;
4578 if (idx[0] != 'h' && length[0] == 'h')
4579 instr->type = INSTR_METER_MHI;
4581 instr->meter.metarray_id = m->id;
4583 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4584 instr->meter.idx.n_bits = fidx->n_bits;
4585 instr->meter.idx.offset = fidx->offset / 8;
4587 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4588 instr->meter.length.n_bits = flength->n_bits;
4589 instr->meter.length.offset = flength->offset / 8;
4591 instr->meter.color_in_val = color_in_val;
4593 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4594 instr->meter.color_out.n_bits = fcout->n_bits;
4595 instr->meter.color_out.offset = fcout->offset / 8;
4598 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4599 if (!fidx && fcin) {
4602 idx_val = strtoul(idx, &idx, 0);
4603 CHECK(!idx[0], EINVAL);
4605 CHECK(!fcin->var_size, EINVAL);
4607 instr->type = INSTR_METER_IMM;
4608 if (length[0] == 'h')
4609 instr->type = INSTR_METER_IHM;
4611 instr->meter.metarray_id = m->id;
4613 instr->meter.idx_val = idx_val;
4615 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4616 instr->meter.length.n_bits = flength->n_bits;
4617 instr->meter.length.offset = flength->offset / 8;
4619 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4620 instr->meter.color_in.n_bits = fcin->n_bits;
4621 instr->meter.color_in.offset = fcin->offset / 8;
4623 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4624 instr->meter.color_out.n_bits = fcout->n_bits;
4625 instr->meter.color_out.offset = fcout->offset / 8;
4628 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4629 if (!fidx && !fcin) {
4630 uint32_t idx_val, color_in_val;
4632 idx_val = strtoul(idx, &idx, 0);
4633 CHECK(!idx[0], EINVAL);
4635 color_in_val = strtoul(color_in, &color_in, 0);
4636 CHECK(!color_in[0], EINVAL);
4638 instr->type = INSTR_METER_IMI;
4639 if (length[0] == 'h')
4640 instr->type = INSTR_METER_IHI;
4642 instr->meter.metarray_id = m->id;
4644 instr->meter.idx_val = idx_val;
4646 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4647 instr->meter.length.n_bits = flength->n_bits;
4648 instr->meter.length.offset = flength->offset / 8;
4650 instr->meter.color_in_val = color_in_val;
4652 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4653 instr->meter.color_out.n_bits = fcout->n_bits;
4654 instr->meter.color_out.offset = fcout->offset / 8;
4661 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4663 struct thread *t = &p->threads[p->thread_id];
4664 struct instruction *ip = t->ip;
4667 __instr_metprefetch_h_exec(p, t, ip);
4674 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4676 struct thread *t = &p->threads[p->thread_id];
4677 struct instruction *ip = t->ip;
4680 __instr_metprefetch_m_exec(p, t, ip);
4687 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4689 struct thread *t = &p->threads[p->thread_id];
4690 struct instruction *ip = t->ip;
4693 __instr_metprefetch_i_exec(p, t, ip);
4700 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4702 struct thread *t = &p->threads[p->thread_id];
4703 struct instruction *ip = t->ip;
4706 __instr_meter_hhm_exec(p, t, ip);
4713 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4715 struct thread *t = &p->threads[p->thread_id];
4716 struct instruction *ip = t->ip;
4719 __instr_meter_hhi_exec(p, t, ip);
4726 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4728 struct thread *t = &p->threads[p->thread_id];
4729 struct instruction *ip = t->ip;
4732 __instr_meter_hmm_exec(p, t, ip);
4739 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4741 struct thread *t = &p->threads[p->thread_id];
4742 struct instruction *ip = t->ip;
4745 __instr_meter_hmi_exec(p, t, ip);
4752 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4754 struct thread *t = &p->threads[p->thread_id];
4755 struct instruction *ip = t->ip;
4758 __instr_meter_mhm_exec(p, t, ip);
4765 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4767 struct thread *t = &p->threads[p->thread_id];
4768 struct instruction *ip = t->ip;
4771 __instr_meter_mhi_exec(p, t, ip);
4778 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4780 struct thread *t = &p->threads[p->thread_id];
4781 struct instruction *ip = t->ip;
4784 __instr_meter_mmm_exec(p, t, ip);
4791 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4793 struct thread *t = &p->threads[p->thread_id];
4794 struct instruction *ip = t->ip;
4797 __instr_meter_mmi_exec(p, t, ip);
4804 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4806 struct thread *t = &p->threads[p->thread_id];
4807 struct instruction *ip = t->ip;
4810 __instr_meter_ihm_exec(p, t, ip);
4817 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4819 struct thread *t = &p->threads[p->thread_id];
4820 struct instruction *ip = t->ip;
4823 __instr_meter_ihi_exec(p, t, ip);
4830 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4832 struct thread *t = &p->threads[p->thread_id];
4833 struct instruction *ip = t->ip;
4836 __instr_meter_imm_exec(p, t, ip);
4843 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4845 struct thread *t = &p->threads[p->thread_id];
4846 struct instruction *ip = t->ip;
4849 __instr_meter_imi_exec(p, t, ip);
4859 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4860 struct action *action __rte_unused,
4863 struct instruction *instr,
4864 struct instruction_data *data)
4866 CHECK(n_tokens == 2, EINVAL);
4868 strcpy(data->jmp_label, tokens[1]);
4870 instr->type = INSTR_JMP;
4871 instr->jmp.ip = NULL; /* Resolved later. */
4876 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4877 struct action *action __rte_unused,
4880 struct instruction *instr,
4881 struct instruction_data *data)
4885 CHECK(n_tokens == 3, EINVAL);
4887 strcpy(data->jmp_label, tokens[1]);
4889 h = header_parse(p, tokens[2]);
4892 instr->type = INSTR_JMP_VALID;
4893 instr->jmp.ip = NULL; /* Resolved later. */
4894 instr->jmp.header_id = h->id;
4899 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4900 struct action *action __rte_unused,
4903 struct instruction *instr,
4904 struct instruction_data *data)
4908 CHECK(n_tokens == 3, EINVAL);
4910 strcpy(data->jmp_label, tokens[1]);
4912 h = header_parse(p, tokens[2]);
4915 instr->type = INSTR_JMP_INVALID;
4916 instr->jmp.ip = NULL; /* Resolved later. */
4917 instr->jmp.header_id = h->id;
4922 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4923 struct action *action,
4926 struct instruction *instr,
4927 struct instruction_data *data)
4929 CHECK(!action, EINVAL);
4930 CHECK(n_tokens == 2, EINVAL);
4932 strcpy(data->jmp_label, tokens[1]);
4934 instr->type = INSTR_JMP_HIT;
4935 instr->jmp.ip = NULL; /* Resolved later. */
4940 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4941 struct action *action,
4944 struct instruction *instr,
4945 struct instruction_data *data)
4947 CHECK(!action, EINVAL);
4948 CHECK(n_tokens == 2, EINVAL);
4950 strcpy(data->jmp_label, tokens[1]);
4952 instr->type = INSTR_JMP_MISS;
4953 instr->jmp.ip = NULL; /* Resolved later. */
4958 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4959 struct action *action,
4962 struct instruction *instr,
4963 struct instruction_data *data)
4967 CHECK(!action, EINVAL);
4968 CHECK(n_tokens == 3, EINVAL);
4970 strcpy(data->jmp_label, tokens[1]);
4972 a = action_find(p, tokens[2]);
4975 instr->type = INSTR_JMP_ACTION_HIT;
4976 instr->jmp.ip = NULL; /* Resolved later. */
4977 instr->jmp.action_id = a->id;
4982 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4983 struct action *action,
4986 struct instruction *instr,
4987 struct instruction_data *data)
4991 CHECK(!action, EINVAL);
4992 CHECK(n_tokens == 3, EINVAL);
4994 strcpy(data->jmp_label, tokens[1]);
4996 a = action_find(p, tokens[2]);
4999 instr->type = INSTR_JMP_ACTION_MISS;
5000 instr->jmp.ip = NULL; /* Resolved later. */
5001 instr->jmp.action_id = a->id;
5006 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5007 struct action *action,
5010 struct instruction *instr,
5011 struct instruction_data *data)
5013 char *a = tokens[2], *b = tokens[3];
5014 struct field *fa, *fb;
5016 uint32_t a_struct_id, b_struct_id;
5018 CHECK(n_tokens == 4, EINVAL);
5020 strcpy(data->jmp_label, tokens[1]);
5022 fa = struct_field_parse(p, action, a, &a_struct_id);
5024 CHECK(!fa->var_size, EINVAL);
5026 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5027 fb = struct_field_parse(p, action, b, &b_struct_id);
5029 CHECK(!fb->var_size, EINVAL);
5031 instr->type = INSTR_JMP_EQ;
5032 if (a[0] != 'h' && b[0] == 'h')
5033 instr->type = INSTR_JMP_EQ_MH;
5034 if (a[0] == 'h' && b[0] != 'h')
5035 instr->type = INSTR_JMP_EQ_HM;
5036 if (a[0] == 'h' && b[0] == 'h')
5037 instr->type = INSTR_JMP_EQ_HH;
5038 instr->jmp.ip = NULL; /* Resolved later. */
5040 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5041 instr->jmp.a.n_bits = fa->n_bits;
5042 instr->jmp.a.offset = fa->offset / 8;
5043 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5044 instr->jmp.b.n_bits = fb->n_bits;
5045 instr->jmp.b.offset = fb->offset / 8;
5050 b_val = strtoull(b, &b, 0);
5051 CHECK(!b[0], EINVAL);
5054 b_val = hton64(b_val) >> (64 - fa->n_bits);
5056 instr->type = INSTR_JMP_EQ_I;
5057 instr->jmp.ip = NULL; /* Resolved later. */
5058 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5059 instr->jmp.a.n_bits = fa->n_bits;
5060 instr->jmp.a.offset = fa->offset / 8;
5061 instr->jmp.b_val = b_val;
5066 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5067 struct action *action,
5070 struct instruction *instr,
5071 struct instruction_data *data)
5073 char *a = tokens[2], *b = tokens[3];
5074 struct field *fa, *fb;
5076 uint32_t a_struct_id, b_struct_id;
5078 CHECK(n_tokens == 4, EINVAL);
5080 strcpy(data->jmp_label, tokens[1]);
5082 fa = struct_field_parse(p, action, a, &a_struct_id);
5084 CHECK(!fa->var_size, EINVAL);
5086 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5087 fb = struct_field_parse(p, action, b, &b_struct_id);
5089 CHECK(!fb->var_size, EINVAL);
5091 instr->type = INSTR_JMP_NEQ;
5092 if (a[0] != 'h' && b[0] == 'h')
5093 instr->type = INSTR_JMP_NEQ_MH;
5094 if (a[0] == 'h' && b[0] != 'h')
5095 instr->type = INSTR_JMP_NEQ_HM;
5096 if (a[0] == 'h' && b[0] == 'h')
5097 instr->type = INSTR_JMP_NEQ_HH;
5098 instr->jmp.ip = NULL; /* Resolved later. */
5100 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5101 instr->jmp.a.n_bits = fa->n_bits;
5102 instr->jmp.a.offset = fa->offset / 8;
5103 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5104 instr->jmp.b.n_bits = fb->n_bits;
5105 instr->jmp.b.offset = fb->offset / 8;
5110 b_val = strtoull(b, &b, 0);
5111 CHECK(!b[0], EINVAL);
5114 b_val = hton64(b_val) >> (64 - fa->n_bits);
5116 instr->type = INSTR_JMP_NEQ_I;
5117 instr->jmp.ip = NULL; /* Resolved later. */
5118 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5119 instr->jmp.a.n_bits = fa->n_bits;
5120 instr->jmp.a.offset = fa->offset / 8;
5121 instr->jmp.b_val = b_val;
5126 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5127 struct action *action,
5130 struct instruction *instr,
5131 struct instruction_data *data)
5133 char *a = tokens[2], *b = tokens[3];
5134 struct field *fa, *fb;
5136 uint32_t a_struct_id, b_struct_id;
5138 CHECK(n_tokens == 4, EINVAL);
5140 strcpy(data->jmp_label, tokens[1]);
5142 fa = struct_field_parse(p, action, a, &a_struct_id);
5144 CHECK(!fa->var_size, EINVAL);
5146 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5147 fb = struct_field_parse(p, action, b, &b_struct_id);
5149 CHECK(!fb->var_size, EINVAL);
5151 instr->type = INSTR_JMP_LT;
5152 if (a[0] == 'h' && b[0] != 'h')
5153 instr->type = INSTR_JMP_LT_HM;
5154 if (a[0] != 'h' && b[0] == 'h')
5155 instr->type = INSTR_JMP_LT_MH;
5156 if (a[0] == 'h' && b[0] == 'h')
5157 instr->type = INSTR_JMP_LT_HH;
5158 instr->jmp.ip = NULL; /* Resolved later. */
5160 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5161 instr->jmp.a.n_bits = fa->n_bits;
5162 instr->jmp.a.offset = fa->offset / 8;
5163 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5164 instr->jmp.b.n_bits = fb->n_bits;
5165 instr->jmp.b.offset = fb->offset / 8;
5169 /* JMP_LT_MI, JMP_LT_HI. */
5170 b_val = strtoull(b, &b, 0);
5171 CHECK(!b[0], EINVAL);
5173 instr->type = INSTR_JMP_LT_MI;
5175 instr->type = INSTR_JMP_LT_HI;
5176 instr->jmp.ip = NULL; /* Resolved later. */
5178 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5179 instr->jmp.a.n_bits = fa->n_bits;
5180 instr->jmp.a.offset = fa->offset / 8;
5181 instr->jmp.b_val = b_val;
5186 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5187 struct action *action,
5190 struct instruction *instr,
5191 struct instruction_data *data)
5193 char *a = tokens[2], *b = tokens[3];
5194 struct field *fa, *fb;
5196 uint32_t a_struct_id, b_struct_id;
5198 CHECK(n_tokens == 4, EINVAL);
5200 strcpy(data->jmp_label, tokens[1]);
5202 fa = struct_field_parse(p, action, a, &a_struct_id);
5204 CHECK(!fa->var_size, EINVAL);
5206 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5207 fb = struct_field_parse(p, action, b, &b_struct_id);
5209 CHECK(!fb->var_size, EINVAL);
5211 instr->type = INSTR_JMP_GT;
5212 if (a[0] == 'h' && b[0] != 'h')
5213 instr->type = INSTR_JMP_GT_HM;
5214 if (a[0] != 'h' && b[0] == 'h')
5215 instr->type = INSTR_JMP_GT_MH;
5216 if (a[0] == 'h' && b[0] == 'h')
5217 instr->type = INSTR_JMP_GT_HH;
5218 instr->jmp.ip = NULL; /* Resolved later. */
5220 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5221 instr->jmp.a.n_bits = fa->n_bits;
5222 instr->jmp.a.offset = fa->offset / 8;
5223 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5224 instr->jmp.b.n_bits = fb->n_bits;
5225 instr->jmp.b.offset = fb->offset / 8;
5229 /* JMP_GT_MI, JMP_GT_HI. */
5230 b_val = strtoull(b, &b, 0);
5231 CHECK(!b[0], EINVAL);
5233 instr->type = INSTR_JMP_GT_MI;
5235 instr->type = INSTR_JMP_GT_HI;
5236 instr->jmp.ip = NULL; /* Resolved later. */
5238 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5239 instr->jmp.a.n_bits = fa->n_bits;
5240 instr->jmp.a.offset = fa->offset / 8;
5241 instr->jmp.b_val = b_val;
5246 instr_jmp_exec(struct rte_swx_pipeline *p)
5248 struct thread *t = &p->threads[p->thread_id];
5249 struct instruction *ip = t->ip;
5251 TRACE("[Thread %2u] jmp\n", p->thread_id);
5253 thread_ip_set(t, ip->jmp.ip);
5257 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5259 struct thread *t = &p->threads[p->thread_id];
5260 struct instruction *ip = t->ip;
5261 uint32_t header_id = ip->jmp.header_id;
5263 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5265 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5269 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5271 struct thread *t = &p->threads[p->thread_id];
5272 struct instruction *ip = t->ip;
5273 uint32_t header_id = ip->jmp.header_id;
5275 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5277 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5281 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5283 struct thread *t = &p->threads[p->thread_id];
5284 struct instruction *ip = t->ip;
5285 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5287 TRACE("[Thread %2u] jmph\n", p->thread_id);
5289 t->ip = ip_next[t->hit];
5293 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5295 struct thread *t = &p->threads[p->thread_id];
5296 struct instruction *ip = t->ip;
5297 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5299 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5301 t->ip = ip_next[t->hit];
5305 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5307 struct thread *t = &p->threads[p->thread_id];
5308 struct instruction *ip = t->ip;
5310 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5312 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5316 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5318 struct thread *t = &p->threads[p->thread_id];
5319 struct instruction *ip = t->ip;
5321 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5323 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5327 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5329 struct thread *t = &p->threads[p->thread_id];
5330 struct instruction *ip = t->ip;
5332 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5338 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5340 struct thread *t = &p->threads[p->thread_id];
5341 struct instruction *ip = t->ip;
5343 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5345 JMP_CMP_MH(t, ip, ==);
5349 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5351 struct thread *t = &p->threads[p->thread_id];
5352 struct instruction *ip = t->ip;
5354 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5356 JMP_CMP_HM(t, ip, ==);
5360 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5362 struct thread *t = &p->threads[p->thread_id];
5363 struct instruction *ip = t->ip;
5365 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5367 JMP_CMP_HH_FAST(t, ip, ==);
5371 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5373 struct thread *t = &p->threads[p->thread_id];
5374 struct instruction *ip = t->ip;
5376 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5378 JMP_CMP_I(t, ip, ==);
5382 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5384 struct thread *t = &p->threads[p->thread_id];
5385 struct instruction *ip = t->ip;
5387 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5393 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5395 struct thread *t = &p->threads[p->thread_id];
5396 struct instruction *ip = t->ip;
5398 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5400 JMP_CMP_MH(t, ip, !=);
5404 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5406 struct thread *t = &p->threads[p->thread_id];
5407 struct instruction *ip = t->ip;
5409 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5411 JMP_CMP_HM(t, ip, !=);
5415 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5417 struct thread *t = &p->threads[p->thread_id];
5418 struct instruction *ip = t->ip;
5420 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5422 JMP_CMP_HH_FAST(t, ip, !=);
5426 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5428 struct thread *t = &p->threads[p->thread_id];
5429 struct instruction *ip = t->ip;
5431 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5433 JMP_CMP_I(t, ip, !=);
5437 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5439 struct thread *t = &p->threads[p->thread_id];
5440 struct instruction *ip = t->ip;
5442 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5448 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5450 struct thread *t = &p->threads[p->thread_id];
5451 struct instruction *ip = t->ip;
5453 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5455 JMP_CMP_MH(t, ip, <);
5459 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5461 struct thread *t = &p->threads[p->thread_id];
5462 struct instruction *ip = t->ip;
5464 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5466 JMP_CMP_HM(t, ip, <);
5470 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5472 struct thread *t = &p->threads[p->thread_id];
5473 struct instruction *ip = t->ip;
5475 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5477 JMP_CMP_HH(t, ip, <);
5481 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5483 struct thread *t = &p->threads[p->thread_id];
5484 struct instruction *ip = t->ip;
5486 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5488 JMP_CMP_MI(t, ip, <);
5492 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5494 struct thread *t = &p->threads[p->thread_id];
5495 struct instruction *ip = t->ip;
5497 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5499 JMP_CMP_HI(t, ip, <);
5503 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5505 struct thread *t = &p->threads[p->thread_id];
5506 struct instruction *ip = t->ip;
5508 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5514 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5516 struct thread *t = &p->threads[p->thread_id];
5517 struct instruction *ip = t->ip;
5519 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5521 JMP_CMP_MH(t, ip, >);
5525 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5527 struct thread *t = &p->threads[p->thread_id];
5528 struct instruction *ip = t->ip;
5530 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5532 JMP_CMP_HM(t, ip, >);
5536 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5538 struct thread *t = &p->threads[p->thread_id];
5539 struct instruction *ip = t->ip;
5541 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5543 JMP_CMP_HH(t, ip, >);
5547 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5549 struct thread *t = &p->threads[p->thread_id];
5550 struct instruction *ip = t->ip;
5552 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5554 JMP_CMP_MI(t, ip, >);
5558 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5560 struct thread *t = &p->threads[p->thread_id];
5561 struct instruction *ip = t->ip;
5563 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5565 JMP_CMP_HI(t, ip, >);
5572 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5573 struct action *action,
5574 char **tokens __rte_unused,
5576 struct instruction *instr,
5577 struct instruction_data *data __rte_unused)
5579 CHECK(action, EINVAL);
5580 CHECK(n_tokens == 1, EINVAL);
5582 instr->type = INSTR_RETURN;
5587 instr_return_exec(struct rte_swx_pipeline *p)
5589 struct thread *t = &p->threads[p->thread_id];
5591 TRACE("[Thread %2u] return\n", p->thread_id);
5597 instr_translate(struct rte_swx_pipeline *p,
5598 struct action *action,
5600 struct instruction *instr,
5601 struct instruction_data *data)
5603 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5604 int n_tokens = 0, tpos = 0;
5606 /* Parse the instruction string into tokens. */
5610 token = strtok_r(string, " \t\v", &string);
5614 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5615 CHECK_NAME(token, EINVAL);
5617 tokens[n_tokens] = token;
5621 CHECK(n_tokens, EINVAL);
5623 /* Handle the optional instruction label. */
5624 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5625 strcpy(data->label, tokens[0]);
5628 CHECK(n_tokens - tpos, EINVAL);
5631 /* Identify the instruction type. */
5632 if (!strcmp(tokens[tpos], "rx"))
5633 return instr_rx_translate(p,
5640 if (!strcmp(tokens[tpos], "tx"))
5641 return instr_tx_translate(p,
5648 if (!strcmp(tokens[tpos], "drop"))
5649 return instr_drop_translate(p,
5656 if (!strcmp(tokens[tpos], "extract"))
5657 return instr_hdr_extract_translate(p,
5664 if (!strcmp(tokens[tpos], "lookahead"))
5665 return instr_hdr_lookahead_translate(p,
5672 if (!strcmp(tokens[tpos], "emit"))
5673 return instr_hdr_emit_translate(p,
5680 if (!strcmp(tokens[tpos], "validate"))
5681 return instr_hdr_validate_translate(p,
5688 if (!strcmp(tokens[tpos], "invalidate"))
5689 return instr_hdr_invalidate_translate(p,
5696 if (!strcmp(tokens[tpos], "mov"))
5697 return instr_mov_translate(p,
5704 if (!strcmp(tokens[tpos], "add"))
5705 return instr_alu_add_translate(p,
5712 if (!strcmp(tokens[tpos], "sub"))
5713 return instr_alu_sub_translate(p,
5720 if (!strcmp(tokens[tpos], "ckadd"))
5721 return instr_alu_ckadd_translate(p,
5728 if (!strcmp(tokens[tpos], "cksub"))
5729 return instr_alu_cksub_translate(p,
5736 if (!strcmp(tokens[tpos], "and"))
5737 return instr_alu_and_translate(p,
5744 if (!strcmp(tokens[tpos], "or"))
5745 return instr_alu_or_translate(p,
5752 if (!strcmp(tokens[tpos], "xor"))
5753 return instr_alu_xor_translate(p,
5760 if (!strcmp(tokens[tpos], "shl"))
5761 return instr_alu_shl_translate(p,
5768 if (!strcmp(tokens[tpos], "shr"))
5769 return instr_alu_shr_translate(p,
5776 if (!strcmp(tokens[tpos], "regprefetch"))
5777 return instr_regprefetch_translate(p,
5784 if (!strcmp(tokens[tpos], "regrd"))
5785 return instr_regrd_translate(p,
5792 if (!strcmp(tokens[tpos], "regwr"))
5793 return instr_regwr_translate(p,
5800 if (!strcmp(tokens[tpos], "regadd"))
5801 return instr_regadd_translate(p,
5808 if (!strcmp(tokens[tpos], "metprefetch"))
5809 return instr_metprefetch_translate(p,
5816 if (!strcmp(tokens[tpos], "meter"))
5817 return instr_meter_translate(p,
5824 if (!strcmp(tokens[tpos], "table"))
5825 return instr_table_translate(p,
5832 if (!strcmp(tokens[tpos], "learn"))
5833 return instr_learn_translate(p,
5840 if (!strcmp(tokens[tpos], "forget"))
5841 return instr_forget_translate(p,
5848 if (!strcmp(tokens[tpos], "extern"))
5849 return instr_extern_translate(p,
5856 if (!strcmp(tokens[tpos], "jmp"))
5857 return instr_jmp_translate(p,
5864 if (!strcmp(tokens[tpos], "jmpv"))
5865 return instr_jmp_valid_translate(p,
5872 if (!strcmp(tokens[tpos], "jmpnv"))
5873 return instr_jmp_invalid_translate(p,
5880 if (!strcmp(tokens[tpos], "jmph"))
5881 return instr_jmp_hit_translate(p,
5888 if (!strcmp(tokens[tpos], "jmpnh"))
5889 return instr_jmp_miss_translate(p,
5896 if (!strcmp(tokens[tpos], "jmpa"))
5897 return instr_jmp_action_hit_translate(p,
5904 if (!strcmp(tokens[tpos], "jmpna"))
5905 return instr_jmp_action_miss_translate(p,
5912 if (!strcmp(tokens[tpos], "jmpeq"))
5913 return instr_jmp_eq_translate(p,
5920 if (!strcmp(tokens[tpos], "jmpneq"))
5921 return instr_jmp_neq_translate(p,
5928 if (!strcmp(tokens[tpos], "jmplt"))
5929 return instr_jmp_lt_translate(p,
5936 if (!strcmp(tokens[tpos], "jmpgt"))
5937 return instr_jmp_gt_translate(p,
5944 if (!strcmp(tokens[tpos], "return"))
5945 return instr_return_translate(p,
5955 static struct instruction_data *
5956 label_find(struct instruction_data *data, uint32_t n, const char *label)
5960 for (i = 0; i < n; i++)
5961 if (!strcmp(label, data[i].label))
5968 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5970 uint32_t count = 0, i;
5975 for (i = 0; i < n; i++)
5976 if (!strcmp(label, data[i].jmp_label))
5983 instr_label_check(struct instruction_data *instruction_data,
5984 uint32_t n_instructions)
5988 /* Check that all instruction labels are unique. */
5989 for (i = 0; i < n_instructions; i++) {
5990 struct instruction_data *data = &instruction_data[i];
5991 char *label = data->label;
5997 for (j = i + 1; j < n_instructions; j++)
5998 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6001 /* Check that no jump instruction (either conditional or not) can jump to itself (loop). */
6002 for (i = 0; i < n_instructions; i++) {
6003 struct instruction_data *data = &instruction_data[i];
6004 char *label = data->label;
6005 char *jmp_label = data->jmp_label;
6007 /* Continue if this instruction does not have a label or it is not a jump. */
6008 if (!label[0] || !jmp_label[0])
6011 CHECK(strcmp(label, jmp_label), EINVAL);
6014 /* Get users for each instruction label. */
6015 for (i = 0; i < n_instructions; i++) {
6016 struct instruction_data *data = &instruction_data[i];
6017 char *label = data->label;
6019 data->n_users = label_is_used(instruction_data,
6028 instr_jmp_resolve(struct instruction *instructions,
6029 struct instruction_data *instruction_data,
6030 uint32_t n_instructions)
6034 for (i = 0; i < n_instructions; i++) {
6035 struct instruction *instr = &instructions[i];
6036 struct instruction_data *data = &instruction_data[i];
6037 struct instruction_data *found;
6039 if (!instruction_is_jmp(instr))
6042 found = label_find(instruction_data,
6045 CHECK(found, EINVAL);
6047 instr->jmp.ip = &instructions[found - instruction_data];
6054 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6056 struct instruction *instr,
6057 struct instruction_data *data __rte_unused,
6058 uint32_t n_instructions)
6061 enum instruction_type type;
6064 /* Check that the first instruction is rx. */
6065 CHECK(instr[0].type == INSTR_RX, EINVAL);
6067 /* Check that there is at least one tx instruction. */
6068 for (i = 0; i < n_instructions; i++) {
6069 type = instr[i].type;
6071 if (instruction_is_tx(type))
6074 CHECK(i < n_instructions, EINVAL);
6076 /* Check that the last instruction is either tx or unconditional
6079 type = instr[n_instructions - 1].type;
6080 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6084 enum instruction_type type;
6087 /* Check that there is at least one return or tx instruction. */
6088 for (i = 0; i < n_instructions; i++) {
6089 type = instr[i].type;
6091 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6094 CHECK(i < n_instructions, EINVAL);
6101 instr_compact(struct instruction *instructions,
6102 struct instruction_data *instruction_data,
6103 uint32_t n_instructions)
6105 uint32_t i, pos = 0;
6107 /* Eliminate the invalid instructions that have been optimized out. */
6108 for (i = 0; i < n_instructions; i++) {
6109 struct instruction *instr = &instructions[i];
6110 struct instruction_data *data = &instruction_data[i];
6116 memcpy(&instructions[pos], instr, sizeof(*instr));
6117 memcpy(&instruction_data[pos], data, sizeof(*data));
6127 instr_pattern_extract_many_search(struct instruction *instr,
6128 struct instruction_data *data,
6130 uint32_t *n_pattern_instr)
6134 for (i = 0; i < n_instr; i++) {
6135 if (data[i].invalid)
6138 if (instr[i].type != INSTR_HDR_EXTRACT)
6141 if (i == RTE_DIM(instr->io.hdr.header_id))
6144 if (i && data[i].n_users)
6151 *n_pattern_instr = i;
6156 instr_pattern_extract_many_replace(struct instruction *instr,
6157 struct instruction_data *data,
6162 for (i = 1; i < n_instr; i++) {
6164 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6165 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6166 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6168 data[i].invalid = 1;
6173 instr_pattern_extract_many_optimize(struct instruction *instructions,
6174 struct instruction_data *instruction_data,
6175 uint32_t n_instructions)
6179 for (i = 0; i < n_instructions; ) {
6180 struct instruction *instr = &instructions[i];
6181 struct instruction_data *data = &instruction_data[i];
6182 uint32_t n_instr = 0;
6186 detected = instr_pattern_extract_many_search(instr,
6191 instr_pattern_extract_many_replace(instr,
6198 /* No pattern starting at the current instruction. */
6202 /* Eliminate the invalid instructions that have been optimized out. */
6203 n_instructions = instr_compact(instructions,
6207 return n_instructions;
6211 instr_pattern_emit_many_tx_search(struct instruction *instr,
6212 struct instruction_data *data,
6214 uint32_t *n_pattern_instr)
6218 for (i = 0; i < n_instr; i++) {
6219 if (data[i].invalid)
6222 if (instr[i].type != INSTR_HDR_EMIT)
6225 if (i == RTE_DIM(instr->io.hdr.header_id))
6228 if (i && data[i].n_users)
6235 if (instr[i].type != INSTR_TX)
6238 if (data[i].n_users)
6243 *n_pattern_instr = i;
6248 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6249 struct instruction_data *data,
6254 /* Any emit instruction in addition to the first one. */
6255 for (i = 1; i < n_instr - 1; i++) {
6257 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6258 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6259 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6261 data[i].invalid = 1;
6264 /* The TX instruction is the last one in the pattern. */
6266 instr[0].io.io.offset = instr[i].io.io.offset;
6267 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6268 data[i].invalid = 1;
6272 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6273 struct instruction_data *instruction_data,
6274 uint32_t n_instructions)
6278 for (i = 0; i < n_instructions; ) {
6279 struct instruction *instr = &instructions[i];
6280 struct instruction_data *data = &instruction_data[i];
6281 uint32_t n_instr = 0;
6284 /* Emit many + TX. */
6285 detected = instr_pattern_emit_many_tx_search(instr,
6290 instr_pattern_emit_many_tx_replace(instr,
6297 /* No pattern starting at the current instruction. */
6301 /* Eliminate the invalid instructions that have been optimized out. */
6302 n_instructions = instr_compact(instructions,
6306 return n_instructions;
6310 action_arg_src_mov_count(struct action *a,
6312 struct instruction *instructions,
6313 struct instruction_data *instruction_data,
6314 uint32_t n_instructions);
6317 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6319 struct instruction *instr,
6320 struct instruction_data *data,
6322 struct instruction *instructions,
6323 struct instruction_data *instruction_data,
6324 uint32_t n_instructions,
6325 uint32_t *n_pattern_instr)
6328 uint32_t src_field_id, i, j;
6330 /* Prerequisites. */
6334 /* First instruction: MOV_HM. */
6335 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6338 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6339 if (!h || h->st->var_size)
6342 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6343 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6346 if (src_field_id == a->st->n_fields)
6349 if (instr[0].mov.dst.offset ||
6350 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6351 instr[0].mov.src.struct_id ||
6352 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6353 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6356 if ((n_instr < h->st->n_fields + 1) ||
6357 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6360 /* Subsequent instructions: MOV_HM. */
6361 for (i = 1; i < h->st->n_fields; i++)
6362 if (data[i].invalid ||
6364 (instr[i].type != INSTR_MOV_HM) ||
6365 (instr[i].mov.dst.struct_id != h->struct_id) ||
6366 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6367 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6368 instr[i].mov.src.struct_id ||
6369 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6370 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6371 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6374 /* Last instruction: HDR_VALIDATE. */
6375 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6376 (instr[i].valid.header_id != h->id))
6379 /* Check that none of the action args that are used as source for this
6380 * DMA transfer are not used as source in any other mov instruction.
6382 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6385 n_users = action_arg_src_mov_count(a,
6394 *n_pattern_instr = 1 + i;
6399 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6401 struct instruction *instr,
6402 struct instruction_data *data,
6406 uint32_t src_field_id, src_offset, i;
6408 /* Read from the instructions before they are modified. */
6409 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6413 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6414 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6417 if (src_field_id == a->st->n_fields)
6420 src_offset = instr[0].mov.src.offset;
6422 /* Modify the instructions. */
6423 instr[0].type = INSTR_DMA_HT;
6424 instr[0].dma.dst.header_id[0] = h->id;
6425 instr[0].dma.dst.struct_id[0] = h->struct_id;
6426 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6427 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6429 for (i = 1; i < n_instr; i++)
6430 data[i].invalid = 1;
6432 /* Update the endianness of the action arguments to header endianness. */
6433 for (i = 0; i < h->st->n_fields; i++)
6434 a->args_endianness[src_field_id + i] = 1;
6438 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6440 struct instruction *instructions,
6441 struct instruction_data *instruction_data,
6442 uint32_t n_instructions)
6447 return n_instructions;
6449 for (i = 0; i < n_instructions; ) {
6450 struct instruction *instr = &instructions[i];
6451 struct instruction_data *data = &instruction_data[i];
6452 uint32_t n_instr = 0;
6455 /* Mov all + validate. */
6456 detected = instr_pattern_mov_all_validate_search(p,
6466 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6471 /* No pattern starting at the current instruction. */
6475 /* Eliminate the invalid instructions that have been optimized out. */
6476 n_instructions = instr_compact(instructions,
6480 return n_instructions;
6484 instr_pattern_dma_many_search(struct instruction *instr,
6485 struct instruction_data *data,
6487 uint32_t *n_pattern_instr)
6491 for (i = 0; i < n_instr; i++) {
6492 if (data[i].invalid)
6495 if (instr[i].type != INSTR_DMA_HT)
6498 if (i == RTE_DIM(instr->dma.dst.header_id))
6501 if (i && data[i].n_users)
6508 *n_pattern_instr = i;
6513 instr_pattern_dma_many_replace(struct instruction *instr,
6514 struct instruction_data *data,
6519 for (i = 1; i < n_instr; i++) {
6521 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6522 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6523 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6524 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6526 data[i].invalid = 1;
6531 instr_pattern_dma_many_optimize(struct instruction *instructions,
6532 struct instruction_data *instruction_data,
6533 uint32_t n_instructions)
6537 for (i = 0; i < n_instructions; ) {
6538 struct instruction *instr = &instructions[i];
6539 struct instruction_data *data = &instruction_data[i];
6540 uint32_t n_instr = 0;
6544 detected = instr_pattern_dma_many_search(instr,
6549 instr_pattern_dma_many_replace(instr, data, n_instr);
6554 /* No pattern starting at the current instruction. */
6558 /* Eliminate the invalid instructions that have been optimized out. */
6559 n_instructions = instr_compact(instructions,
6563 return n_instructions;
6567 instr_optimize(struct rte_swx_pipeline *p,
6569 struct instruction *instructions,
6570 struct instruction_data *instruction_data,
6571 uint32_t n_instructions)
6574 n_instructions = instr_pattern_extract_many_optimize(instructions,
6578 /* Emit many + TX. */
6579 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6583 /* Mov all + validate. */
6584 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6591 n_instructions = instr_pattern_dma_many_optimize(instructions,
6595 return n_instructions;
6599 instruction_config(struct rte_swx_pipeline *p,
6601 const char **instructions,
6602 uint32_t n_instructions)
6604 struct instruction *instr = NULL;
6605 struct instruction_data *data = NULL;
6609 CHECK(n_instructions, EINVAL);
6610 CHECK(instructions, EINVAL);
6611 for (i = 0; i < n_instructions; i++)
6612 CHECK_INSTRUCTION(instructions[i], EINVAL);
6614 /* Memory allocation. */
6615 instr = calloc(n_instructions, sizeof(struct instruction));
6621 data = calloc(n_instructions, sizeof(struct instruction_data));
6627 for (i = 0; i < n_instructions; i++) {
6628 char *string = strdup(instructions[i]);
6634 err = instr_translate(p, a, string, &instr[i], &data[i]);
6643 err = instr_label_check(data, n_instructions);
6647 err = instr_verify(p, a, instr, data, n_instructions);
6651 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6653 err = instr_jmp_resolve(instr, data, n_instructions);
6658 a->instructions = instr;
6659 a->instruction_data = data;
6660 a->n_instructions = n_instructions;
6662 p->instructions = instr;
6663 p->instruction_data = data;
6664 p->n_instructions = n_instructions;
6675 static instr_exec_t instruction_table[] = {
6676 [INSTR_RX] = instr_rx_exec,
6677 [INSTR_TX] = instr_tx_exec,
6678 [INSTR_TX_I] = instr_tx_i_exec,
6679 [INSTR_DROP] = instr_drop_exec,
6681 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6682 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6683 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6684 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6685 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6686 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6687 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6688 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6689 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6690 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6692 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6693 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6694 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6695 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6696 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6697 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6698 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6699 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6700 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6702 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6703 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6705 [INSTR_MOV] = instr_mov_exec,
6706 [INSTR_MOV_MH] = instr_mov_mh_exec,
6707 [INSTR_MOV_HM] = instr_mov_hm_exec,
6708 [INSTR_MOV_HH] = instr_mov_hh_exec,
6709 [INSTR_MOV_I] = instr_mov_i_exec,
6711 [INSTR_DMA_HT] = instr_dma_ht_exec,
6712 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6713 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6714 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6715 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6716 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6717 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6718 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6720 [INSTR_ALU_ADD] = instr_alu_add_exec,
6721 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6722 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6723 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6724 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6725 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6727 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6728 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6729 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6730 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6731 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6732 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6734 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6735 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6736 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6737 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6739 [INSTR_ALU_AND] = instr_alu_and_exec,
6740 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6741 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6742 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6743 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6745 [INSTR_ALU_OR] = instr_alu_or_exec,
6746 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6747 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6748 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6749 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6751 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6752 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6753 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6754 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6755 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6757 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6758 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6759 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6760 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6761 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6762 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6764 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6765 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6766 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6767 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6768 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6769 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6771 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6772 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6773 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6775 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6776 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6777 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6778 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6779 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6780 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6782 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6783 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6784 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6785 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6786 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6787 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6788 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6789 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6790 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6792 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6793 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6794 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6795 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6796 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6797 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6798 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6799 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6800 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6802 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6803 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6804 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6806 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6807 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6808 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6809 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6810 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6811 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6812 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6813 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6814 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6815 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6816 [INSTR_METER_IMM] = instr_meter_imm_exec,
6817 [INSTR_METER_IMI] = instr_meter_imi_exec,
6819 [INSTR_TABLE] = instr_table_exec,
6820 [INSTR_TABLE_AF] = instr_table_af_exec,
6821 [INSTR_SELECTOR] = instr_selector_exec,
6822 [INSTR_LEARNER] = instr_learner_exec,
6823 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6824 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6825 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6826 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6827 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6829 [INSTR_JMP] = instr_jmp_exec,
6830 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6831 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6832 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6833 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6834 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6835 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6837 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6838 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6839 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6840 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6841 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6843 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6844 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6845 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6846 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6847 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6849 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6850 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6851 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6852 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6853 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6854 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6856 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6857 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6858 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6859 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6860 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6861 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6863 [INSTR_RETURN] = instr_return_exec,
6867 instruction_table_build(struct rte_swx_pipeline *p)
6869 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6870 sizeof(struct instr_exec_t *));
6871 if (!p->instruction_table)
6874 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6880 instruction_table_build_free(struct rte_swx_pipeline *p)
6882 if (!p->instruction_table)
6885 free(p->instruction_table);
6886 p->instruction_table = NULL;
6890 instruction_table_free(struct rte_swx_pipeline *p)
6892 instruction_table_build_free(p);
6896 instr_exec(struct rte_swx_pipeline *p)
6898 struct thread *t = &p->threads[p->thread_id];
6899 struct instruction *ip = t->ip;
6900 instr_exec_t instr = p->instruction_table[ip->type];
6908 static struct action *
6909 action_find(struct rte_swx_pipeline *p, const char *name)
6911 struct action *elem;
6916 TAILQ_FOREACH(elem, &p->actions, node)
6917 if (strcmp(elem->name, name) == 0)
6923 static struct action *
6924 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6926 struct action *action = NULL;
6928 TAILQ_FOREACH(action, &p->actions, node)
6929 if (action->id == id)
6935 static struct field *
6936 action_field_find(struct action *a, const char *name)
6938 return a->st ? struct_type_field_find(a->st, name) : NULL;
6941 static struct field *
6942 action_field_parse(struct action *action, const char *name)
6944 if (name[0] != 't' || name[1] != '.')
6947 return action_field_find(action, &name[2]);
6951 action_has_nbo_args(struct action *a)
6955 /* Return if the action does not have any args. */
6957 return 0; /* FALSE */
6959 for (i = 0; i < a->st->n_fields; i++)
6960 if (a->args_endianness[i])
6961 return 1; /* TRUE */
6963 return 0; /* FALSE */
6967 action_does_learning(struct action *a)
6971 for (i = 0; i < a->n_instructions; i++)
6972 switch (a->instructions[i].type) {
6973 case INSTR_LEARNER_LEARN:
6974 return 1; /* TRUE */
6976 case INSTR_LEARNER_FORGET:
6977 return 1; /* TRUE */
6983 return 0; /* FALSE */
6987 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6989 const char *args_struct_type_name,
6990 const char **instructions,
6991 uint32_t n_instructions)
6993 struct struct_type *args_struct_type = NULL;
6999 CHECK_NAME(name, EINVAL);
7000 CHECK(!action_find(p, name), EEXIST);
7002 if (args_struct_type_name) {
7003 CHECK_NAME(args_struct_type_name, EINVAL);
7004 args_struct_type = struct_type_find(p, args_struct_type_name);
7005 CHECK(args_struct_type, EINVAL);
7006 CHECK(!args_struct_type->var_size, EINVAL);
7009 /* Node allocation. */
7010 a = calloc(1, sizeof(struct action));
7012 if (args_struct_type) {
7013 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7014 if (!a->args_endianness) {
7020 /* Node initialization. */
7021 strcpy(a->name, name);
7022 a->st = args_struct_type;
7023 a->id = p->n_actions;
7025 /* Instruction translation. */
7026 err = instruction_config(p, a, instructions, n_instructions);
7028 free(a->args_endianness);
7033 /* Node add to tailq. */
7034 TAILQ_INSERT_TAIL(&p->actions, a, node);
7041 action_build(struct rte_swx_pipeline *p)
7043 struct action *action;
7045 /* p->action_instructions. */
7046 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7047 CHECK(p->action_instructions, ENOMEM);
7049 TAILQ_FOREACH(action, &p->actions, node)
7050 p->action_instructions[action->id] = action->instructions;
7052 /* p->action_funcs. */
7053 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7054 CHECK(p->action_funcs, ENOMEM);
7060 action_build_free(struct rte_swx_pipeline *p)
7062 free(p->action_funcs);
7063 p->action_funcs = NULL;
7065 free(p->action_instructions);
7066 p->action_instructions = NULL;
7070 action_free(struct rte_swx_pipeline *p)
7072 action_build_free(p);
7075 struct action *action;
7077 action = TAILQ_FIRST(&p->actions);
7081 TAILQ_REMOVE(&p->actions, action, node);
7082 free(action->instruction_data);
7083 free(action->instructions);
7089 action_arg_src_mov_count(struct action *a,
7091 struct instruction *instructions,
7092 struct instruction_data *instruction_data,
7093 uint32_t n_instructions)
7095 uint32_t offset, n_users = 0, i;
7098 (arg_id >= a->st->n_fields) ||
7100 !instruction_data ||
7104 offset = a->st->fields[arg_id].offset / 8;
7106 for (i = 0; i < n_instructions; i++) {
7107 struct instruction *instr = &instructions[i];
7108 struct instruction_data *data = &instruction_data[i];
7110 if (data->invalid ||
7111 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7112 instr->mov.src.struct_id ||
7113 (instr->mov.src.offset != offset))
7125 static struct table_type *
7126 table_type_find(struct rte_swx_pipeline *p, const char *name)
7128 struct table_type *elem;
7130 TAILQ_FOREACH(elem, &p->table_types, node)
7131 if (strcmp(elem->name, name) == 0)
7137 static struct table_type *
7138 table_type_resolve(struct rte_swx_pipeline *p,
7139 const char *recommended_type_name,
7140 enum rte_swx_table_match_type match_type)
7142 struct table_type *elem;
7144 /* Only consider the recommended type if the match type is correct. */
7145 if (recommended_type_name)
7146 TAILQ_FOREACH(elem, &p->table_types, node)
7147 if (!strcmp(elem->name, recommended_type_name) &&
7148 (elem->match_type == match_type))
7151 /* Ignore the recommended type and get the first element with this match
7154 TAILQ_FOREACH(elem, &p->table_types, node)
7155 if (elem->match_type == match_type)
7161 static struct table *
7162 table_find(struct rte_swx_pipeline *p, const char *name)
7166 TAILQ_FOREACH(elem, &p->tables, node)
7167 if (strcmp(elem->name, name) == 0)
7173 static struct table *
7174 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7176 struct table *table = NULL;
7178 TAILQ_FOREACH(table, &p->tables, node)
7179 if (table->id == id)
7186 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7188 enum rte_swx_table_match_type match_type,
7189 struct rte_swx_table_ops *ops)
7191 struct table_type *elem;
7195 CHECK_NAME(name, EINVAL);
7196 CHECK(!table_type_find(p, name), EEXIST);
7199 CHECK(ops->create, EINVAL);
7200 CHECK(ops->lkp, EINVAL);
7201 CHECK(ops->free, EINVAL);
7203 /* Node allocation. */
7204 elem = calloc(1, sizeof(struct table_type));
7205 CHECK(elem, ENOMEM);
7207 /* Node initialization. */
7208 strcpy(elem->name, name);
7209 elem->match_type = match_type;
7210 memcpy(&elem->ops, ops, sizeof(*ops));
7212 /* Node add to tailq. */
7213 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7219 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7221 enum rte_swx_table_match_type *match_type)
7223 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7225 for (i = 0; i < n_fields; i++) {
7226 struct rte_swx_match_field_params *f = &fields[i];
7228 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7231 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7235 if ((n_fields_lpm > 1) ||
7236 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7239 *match_type = (n_fields_em == n_fields) ?
7240 RTE_SWX_TABLE_MATCH_EXACT :
7241 RTE_SWX_TABLE_MATCH_WILDCARD;
7247 table_match_fields_check(struct rte_swx_pipeline *p,
7248 struct rte_swx_pipeline_table_params *params,
7249 struct header **header)
7251 struct header *h0 = NULL;
7252 struct field *hf, *mf;
7253 uint32_t *offset = NULL, i;
7256 /* Return if no match fields. */
7257 if (!params->n_fields) {
7258 if (params->fields) {
7269 /* Memory allocation. */
7270 offset = calloc(params->n_fields, sizeof(uint32_t));
7276 /* Check that all the match fields belong to either the same header or
7279 hf = header_field_parse(p, params->fields[0].name, &h0);
7280 mf = metadata_field_parse(p, params->fields[0].name);
7281 if ((!hf && !mf) || (hf && hf->var_size)) {
7286 offset[0] = h0 ? hf->offset : mf->offset;
7288 for (i = 1; i < params->n_fields; i++)
7292 hf = header_field_parse(p, params->fields[i].name, &h);
7293 if (!hf || (h->id != h0->id) || hf->var_size) {
7298 offset[i] = hf->offset;
7300 mf = metadata_field_parse(p, params->fields[i].name);
7306 offset[i] = mf->offset;
7309 /* Check that there are no duplicated match fields. */
7310 for (i = 0; i < params->n_fields; i++) {
7313 for (j = 0; j < i; j++)
7314 if (offset[j] == offset[i]) {
7330 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7332 struct rte_swx_pipeline_table_params *params,
7333 const char *recommended_table_type_name,
7337 struct table_type *type;
7338 struct table *t = NULL;
7339 struct action *default_action;
7340 struct header *header = NULL;
7341 uint32_t action_data_size_max = 0, i;
7346 CHECK_NAME(name, EINVAL);
7347 CHECK(!table_find(p, name), EEXIST);
7348 CHECK(!selector_find(p, name), EEXIST);
7349 CHECK(!learner_find(p, name), EEXIST);
7351 CHECK(params, EINVAL);
7354 status = table_match_fields_check(p, params, &header);
7358 /* Action checks. */
7359 CHECK(params->n_actions, EINVAL);
7360 CHECK(params->action_names, EINVAL);
7361 for (i = 0; i < params->n_actions; i++) {
7362 const char *action_name = params->action_names[i];
7364 uint32_t action_data_size;
7365 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7367 CHECK_NAME(action_name, EINVAL);
7369 a = action_find(p, action_name);
7371 CHECK(!action_does_learning(a), EINVAL);
7373 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7374 if (action_data_size > action_data_size_max)
7375 action_data_size_max = action_data_size;
7377 if (params->action_is_for_table_entries)
7378 action_is_for_table_entries = params->action_is_for_table_entries[i];
7379 if (params->action_is_for_default_entry)
7380 action_is_for_default_entry = params->action_is_for_default_entry[i];
7381 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7384 CHECK_NAME(params->default_action_name, EINVAL);
7385 for (i = 0; i < p->n_actions; i++)
7386 if (!strcmp(params->action_names[i],
7387 params->default_action_name))
7389 CHECK(i < params->n_actions, EINVAL);
7390 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7393 default_action = action_find(p, params->default_action_name);
7394 CHECK((default_action->st && params->default_action_data) ||
7395 !params->default_action_data, EINVAL);
7397 /* Table type checks. */
7398 if (recommended_table_type_name)
7399 CHECK_NAME(recommended_table_type_name, EINVAL);
7401 if (params->n_fields) {
7402 enum rte_swx_table_match_type match_type;
7404 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7408 type = table_type_resolve(p, recommended_table_type_name, match_type);
7409 CHECK(type, EINVAL);
7414 /* Memory allocation. */
7415 t = calloc(1, sizeof(struct table));
7419 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7423 t->actions = calloc(params->n_actions, sizeof(struct action *));
7427 if (action_data_size_max) {
7428 t->default_action_data = calloc(1, action_data_size_max);
7429 if (!t->default_action_data)
7433 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7434 if (!t->action_is_for_table_entries)
7437 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7438 if (!t->action_is_for_default_entry)
7441 /* Node initialization. */
7442 strcpy(t->name, name);
7443 if (args && args[0])
7444 strcpy(t->args, args);
7447 for (i = 0; i < params->n_fields; i++) {
7448 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7449 struct match_field *f = &t->fields[i];
7451 f->match_type = field->match_type;
7453 header_field_parse(p, field->name, NULL) :
7454 metadata_field_parse(p, field->name);
7456 t->n_fields = params->n_fields;
7459 for (i = 0; i < params->n_actions; i++) {
7460 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7462 if (params->action_is_for_table_entries)
7463 action_is_for_table_entries = params->action_is_for_table_entries[i];
7464 if (params->action_is_for_default_entry)
7465 action_is_for_default_entry = params->action_is_for_default_entry[i];
7467 t->actions[i] = action_find(p, params->action_names[i]);
7468 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7469 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7471 t->default_action = default_action;
7472 if (default_action->st)
7473 memcpy(t->default_action_data,
7474 params->default_action_data,
7475 default_action->st->n_bits / 8);
7476 t->n_actions = params->n_actions;
7477 t->default_action_is_const = params->default_action_is_const;
7478 t->action_data_size_max = action_data_size_max;
7481 t->id = p->n_tables;
7483 /* Node add to tailq. */
7484 TAILQ_INSERT_TAIL(&p->tables, t, node);
7493 free(t->action_is_for_default_entry);
7494 free(t->action_is_for_table_entries);
7495 free(t->default_action_data);
7503 static struct rte_swx_table_params *
7504 table_params_get(struct table *table)
7506 struct rte_swx_table_params *params;
7507 struct field *first, *last;
7509 uint32_t key_size, key_offset, action_data_size, i;
7511 /* Memory allocation. */
7512 params = calloc(1, sizeof(struct rte_swx_table_params));
7516 /* Find first (smallest offset) and last (biggest offset) match fields. */
7517 first = table->fields[0].field;
7518 last = table->fields[0].field;
7520 for (i = 0; i < table->n_fields; i++) {
7521 struct field *f = table->fields[i].field;
7523 if (f->offset < first->offset)
7526 if (f->offset > last->offset)
7530 /* Key offset and size. */
7531 key_offset = first->offset / 8;
7532 key_size = (last->offset + last->n_bits - first->offset) / 8;
7534 /* Memory allocation. */
7535 key_mask = calloc(1, key_size);
7542 for (i = 0; i < table->n_fields; i++) {
7543 struct field *f = table->fields[i].field;
7544 uint32_t start = (f->offset - first->offset) / 8;
7545 size_t size = f->n_bits / 8;
7547 memset(&key_mask[start], 0xFF, size);
7550 /* Action data size. */
7551 action_data_size = 0;
7552 for (i = 0; i < table->n_actions; i++) {
7553 struct action *action = table->actions[i];
7554 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7556 if (ads > action_data_size)
7557 action_data_size = ads;
7561 params->match_type = table->type->match_type;
7562 params->key_size = key_size;
7563 params->key_offset = key_offset;
7564 params->key_mask0 = key_mask;
7565 params->action_data_size = action_data_size;
7566 params->n_keys_max = table->size;
7572 table_params_free(struct rte_swx_table_params *params)
7577 free(params->key_mask0);
7582 table_stub_lkp(void *table __rte_unused,
7583 void *mailbox __rte_unused,
7584 uint8_t **key __rte_unused,
7585 uint64_t *action_id __rte_unused,
7586 uint8_t **action_data __rte_unused,
7590 return 1; /* DONE. */
7594 table_build(struct rte_swx_pipeline *p)
7598 /* Per pipeline: table statistics. */
7599 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7600 CHECK(p->table_stats, ENOMEM);
7602 for (i = 0; i < p->n_tables; i++) {
7603 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7604 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7607 /* Per thread: table runt-time. */
7608 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7609 struct thread *t = &p->threads[i];
7610 struct table *table;
7612 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7613 CHECK(t->tables, ENOMEM);
7615 TAILQ_FOREACH(table, &p->tables, node) {
7616 struct table_runtime *r = &t->tables[table->id];
7621 size = table->type->ops.mailbox_size_get();
7624 r->func = table->type->ops.lkp;
7628 r->mailbox = calloc(1, size);
7629 CHECK(r->mailbox, ENOMEM);
7633 r->key = table->header ?
7634 &t->structs[table->header->struct_id] :
7635 &t->structs[p->metadata_struct_id];
7637 r->func = table_stub_lkp;
7646 table_build_free(struct rte_swx_pipeline *p)
7650 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7651 struct thread *t = &p->threads[i];
7657 for (j = 0; j < p->n_tables; j++) {
7658 struct table_runtime *r = &t->tables[j];
7667 if (p->table_stats) {
7668 for (i = 0; i < p->n_tables; i++)
7669 free(p->table_stats[i].n_pkts_action);
7671 free(p->table_stats);
7676 table_free(struct rte_swx_pipeline *p)
7678 table_build_free(p);
7684 elem = TAILQ_FIRST(&p->tables);
7688 TAILQ_REMOVE(&p->tables, elem, node);
7690 free(elem->actions);
7691 free(elem->default_action_data);
7697 struct table_type *elem;
7699 elem = TAILQ_FIRST(&p->table_types);
7703 TAILQ_REMOVE(&p->table_types, elem, node);
7711 static struct selector *
7712 selector_find(struct rte_swx_pipeline *p, const char *name)
7716 TAILQ_FOREACH(s, &p->selectors, node)
7717 if (strcmp(s->name, name) == 0)
7723 static struct selector *
7724 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7726 struct selector *s = NULL;
7728 TAILQ_FOREACH(s, &p->selectors, node)
7736 selector_fields_check(struct rte_swx_pipeline *p,
7737 struct rte_swx_pipeline_selector_params *params,
7738 struct header **header)
7740 struct header *h0 = NULL;
7741 struct field *hf, *mf;
7744 /* Return if no selector fields. */
7745 if (!params->n_selector_fields || !params->selector_field_names)
7748 /* Check that all the selector fields either belong to the same header
7749 * or are all meta-data fields.
7751 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7752 mf = metadata_field_parse(p, params->selector_field_names[0]);
7756 for (i = 1; i < params->n_selector_fields; i++)
7760 hf = header_field_parse(p, params->selector_field_names[i], &h);
7761 if (!hf || (h->id != h0->id))
7764 mf = metadata_field_parse(p, params->selector_field_names[i]);
7769 /* Check that there are no duplicated match fields. */
7770 for (i = 0; i < params->n_selector_fields; i++) {
7771 const char *field_name = params->selector_field_names[i];
7774 for (j = i + 1; j < params->n_selector_fields; j++)
7775 if (!strcmp(params->selector_field_names[j], field_name))
7787 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7789 struct rte_swx_pipeline_selector_params *params)
7792 struct header *selector_header = NULL;
7793 struct field *group_id_field, *member_id_field;
7799 CHECK_NAME(name, EINVAL);
7800 CHECK(!table_find(p, name), EEXIST);
7801 CHECK(!selector_find(p, name), EEXIST);
7802 CHECK(!learner_find(p, name), EEXIST);
7804 CHECK(params, EINVAL);
7806 CHECK_NAME(params->group_id_field_name, EINVAL);
7807 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7808 CHECK(group_id_field, EINVAL);
7810 for (i = 0; i < params->n_selector_fields; i++) {
7811 const char *field_name = params->selector_field_names[i];
7813 CHECK_NAME(field_name, EINVAL);
7815 status = selector_fields_check(p, params, &selector_header);
7819 CHECK_NAME(params->member_id_field_name, EINVAL);
7820 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7821 CHECK(member_id_field, EINVAL);
7823 CHECK(params->n_groups_max, EINVAL);
7825 CHECK(params->n_members_per_group_max, EINVAL);
7827 /* Memory allocation. */
7828 s = calloc(1, sizeof(struct selector));
7834 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7835 if (!s->selector_fields) {
7840 /* Node initialization. */
7841 strcpy(s->name, name);
7843 s->group_id_field = group_id_field;
7845 for (i = 0; i < params->n_selector_fields; i++) {
7846 const char *field_name = params->selector_field_names[i];
7848 s->selector_fields[i] = selector_header ?
7849 header_field_parse(p, field_name, NULL) :
7850 metadata_field_parse(p, field_name);
7853 s->n_selector_fields = params->n_selector_fields;
7855 s->selector_header = selector_header;
7857 s->member_id_field = member_id_field;
7859 s->n_groups_max = params->n_groups_max;
7861 s->n_members_per_group_max = params->n_members_per_group_max;
7863 s->id = p->n_selectors;
7865 /* Node add to tailq. */
7866 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7875 free(s->selector_fields);
7883 selector_params_free(struct rte_swx_table_selector_params *params)
7888 free(params->selector_mask);
7893 static struct rte_swx_table_selector_params *
7894 selector_table_params_get(struct selector *s)
7896 struct rte_swx_table_selector_params *params = NULL;
7897 struct field *first, *last;
7900 /* Memory allocation. */
7901 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7906 params->group_id_offset = s->group_id_field->offset / 8;
7908 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7909 first = s->selector_fields[0];
7910 last = s->selector_fields[0];
7912 for (i = 0; i < s->n_selector_fields; i++) {
7913 struct field *f = s->selector_fields[i];
7915 if (f->offset < first->offset)
7918 if (f->offset > last->offset)
7922 /* Selector offset and size. */
7923 params->selector_offset = first->offset / 8;
7924 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7926 /* Memory allocation. */
7927 params->selector_mask = calloc(1, params->selector_size);
7928 if (!params->selector_mask)
7931 /* Selector mask. */
7932 for (i = 0; i < s->n_selector_fields; i++) {
7933 struct field *f = s->selector_fields[i];
7934 uint32_t start = (f->offset - first->offset) / 8;
7935 size_t size = f->n_bits / 8;
7937 memset(¶ms->selector_mask[start], 0xFF, size);
7941 params->member_id_offset = s->member_id_field->offset / 8;
7943 /* Maximum number of groups. */
7944 params->n_groups_max = s->n_groups_max;
7946 /* Maximum number of members per group. */
7947 params->n_members_per_group_max = s->n_members_per_group_max;
7952 selector_params_free(params);
7957 selector_build_free(struct rte_swx_pipeline *p)
7961 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7962 struct thread *t = &p->threads[i];
7968 for (j = 0; j < p->n_selectors; j++) {
7969 struct selector_runtime *r = &t->selectors[j];
7975 t->selectors = NULL;
7978 free(p->selector_stats);
7979 p->selector_stats = NULL;
7983 selector_build(struct rte_swx_pipeline *p)
7988 /* Per pipeline: selector statistics. */
7989 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7990 if (!p->selector_stats) {
7995 /* Per thread: selector run-time. */
7996 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7997 struct thread *t = &p->threads[i];
8000 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
8001 if (!t->selectors) {
8006 TAILQ_FOREACH(s, &p->selectors, node) {
8007 struct selector_runtime *r = &t->selectors[s->id];
8011 size = rte_swx_table_selector_mailbox_size_get();
8013 r->mailbox = calloc(1, size);
8020 /* r->group_id_buffer. */
8021 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8023 /* r->selector_buffer. */
8024 r->selector_buffer = s->selector_header ?
8025 &t->structs[s->selector_header->struct_id] :
8026 &t->structs[p->metadata_struct_id];
8028 /* r->member_id_buffer. */
8029 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8036 selector_build_free(p);
8041 selector_free(struct rte_swx_pipeline *p)
8043 selector_build_free(p);
8045 /* Selector tables. */
8047 struct selector *elem;
8049 elem = TAILQ_FIRST(&p->selectors);
8053 TAILQ_REMOVE(&p->selectors, elem, node);
8054 free(elem->selector_fields);
8062 static struct learner *
8063 learner_find(struct rte_swx_pipeline *p, const char *name)
8067 TAILQ_FOREACH(l, &p->learners, node)
8068 if (!strcmp(l->name, name))
8074 static struct learner *
8075 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8077 struct learner *l = NULL;
8079 TAILQ_FOREACH(l, &p->learners, node)
8087 learner_match_fields_check(struct rte_swx_pipeline *p,
8088 struct rte_swx_pipeline_learner_params *params,
8089 struct header **header)
8091 struct header *h0 = NULL;
8092 struct field *hf, *mf;
8095 /* Return if no match fields. */
8096 if (!params->n_fields || !params->field_names)
8099 /* Check that all the match fields either belong to the same header
8100 * or are all meta-data fields.
8102 hf = header_field_parse(p, params->field_names[0], &h0);
8103 mf = metadata_field_parse(p, params->field_names[0]);
8107 for (i = 1; i < params->n_fields; i++)
8111 hf = header_field_parse(p, params->field_names[i], &h);
8112 if (!hf || (h->id != h0->id))
8115 mf = metadata_field_parse(p, params->field_names[i]);
8120 /* Check that there are no duplicated match fields. */
8121 for (i = 0; i < params->n_fields; i++) {
8122 const char *field_name = params->field_names[i];
8125 for (j = i + 1; j < params->n_fields; j++)
8126 if (!strcmp(params->field_names[j], field_name))
8138 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8140 struct struct_type *mst = p->metadata_st, *ast = a->st;
8141 struct field *mf, *af;
8151 /* Check that mf_name is the name of a valid meta-data field. */
8152 CHECK_NAME(mf_name, EINVAL);
8153 mf = metadata_field_parse(p, mf_name);
8156 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8157 * all the action arguments.
8159 mf_pos = mf - mst->fields;
8160 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8162 /* Check that the size of each of the identified meta-data fields matches exactly the size
8163 * of the corresponding action argument.
8165 for (i = 0; i < ast->n_fields; i++) {
8166 mf = &mst->fields[mf_pos + i];
8167 af = &ast->fields[i];
8169 CHECK(mf->n_bits == af->n_bits, EINVAL);
8176 learner_action_learning_check(struct rte_swx_pipeline *p,
8177 struct action *action,
8178 const char **action_names,
8183 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8184 * the action passed as argument to the "learn" instruction) is also enabled for the
8185 * current learner table.
8187 for (i = 0; i < action->n_instructions; i++) {
8188 struct instruction *instr = &action->instructions[i];
8189 uint32_t found = 0, j;
8191 if (instr->type != INSTR_LEARNER_LEARN)
8194 for (j = 0; j < n_actions; j++) {
8197 a = action_find(p, action_names[j]);
8201 if (a->id == instr->learn.action_id)
8213 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8215 struct rte_swx_pipeline_learner_params *params,
8219 struct learner *l = NULL;
8220 struct action *default_action;
8221 struct header *header = NULL;
8222 uint32_t action_data_size_max = 0, i;
8227 CHECK_NAME(name, EINVAL);
8228 CHECK(!table_find(p, name), EEXIST);
8229 CHECK(!selector_find(p, name), EEXIST);
8230 CHECK(!learner_find(p, name), EEXIST);
8232 CHECK(params, EINVAL);
8235 status = learner_match_fields_check(p, params, &header);
8239 /* Action checks. */
8240 CHECK(params->n_actions, EINVAL);
8241 CHECK(params->action_names, EINVAL);
8242 for (i = 0; i < params->n_actions; i++) {
8243 const char *action_name = params->action_names[i];
8245 uint32_t action_data_size;
8246 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8248 CHECK_NAME(action_name, EINVAL);
8250 a = action_find(p, action_name);
8253 status = learner_action_learning_check(p,
8255 params->action_names,
8260 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8261 if (action_data_size > action_data_size_max)
8262 action_data_size_max = action_data_size;
8264 if (params->action_is_for_table_entries)
8265 action_is_for_table_entries = params->action_is_for_table_entries[i];
8266 if (params->action_is_for_default_entry)
8267 action_is_for_default_entry = params->action_is_for_default_entry[i];
8268 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8271 CHECK_NAME(params->default_action_name, EINVAL);
8272 for (i = 0; i < p->n_actions; i++)
8273 if (!strcmp(params->action_names[i],
8274 params->default_action_name))
8276 CHECK(i < params->n_actions, EINVAL);
8277 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8280 default_action = action_find(p, params->default_action_name);
8281 CHECK((default_action->st && params->default_action_data) ||
8282 !params->default_action_data, EINVAL);
8284 /* Any other checks. */
8285 CHECK(size, EINVAL);
8286 CHECK(timeout, EINVAL);
8288 /* Memory allocation. */
8289 l = calloc(1, sizeof(struct learner));
8293 l->fields = calloc(params->n_fields, sizeof(struct field *));
8297 l->actions = calloc(params->n_actions, sizeof(struct action *));
8301 if (action_data_size_max) {
8302 l->default_action_data = calloc(1, action_data_size_max);
8303 if (!l->default_action_data)
8307 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8308 if (!l->action_is_for_table_entries)
8311 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8312 if (!l->action_is_for_default_entry)
8315 /* Node initialization. */
8316 strcpy(l->name, name);
8318 for (i = 0; i < params->n_fields; i++) {
8319 const char *field_name = params->field_names[i];
8321 l->fields[i] = header ?
8322 header_field_parse(p, field_name, NULL) :
8323 metadata_field_parse(p, field_name);
8326 l->n_fields = params->n_fields;
8330 for (i = 0; i < params->n_actions; i++) {
8331 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8333 if (params->action_is_for_table_entries)
8334 action_is_for_table_entries = params->action_is_for_table_entries[i];
8335 if (params->action_is_for_default_entry)
8336 action_is_for_default_entry = params->action_is_for_default_entry[i];
8338 l->actions[i] = action_find(p, params->action_names[i]);
8339 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8340 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8343 l->default_action = default_action;
8345 if (default_action->st)
8346 memcpy(l->default_action_data,
8347 params->default_action_data,
8348 default_action->st->n_bits / 8);
8350 l->n_actions = params->n_actions;
8352 l->default_action_is_const = params->default_action_is_const;
8354 l->action_data_size_max = action_data_size_max;
8358 l->timeout = timeout;
8360 l->id = p->n_learners;
8362 /* Node add to tailq. */
8363 TAILQ_INSERT_TAIL(&p->learners, l, node);
8372 free(l->action_is_for_default_entry);
8373 free(l->action_is_for_table_entries);
8374 free(l->default_action_data);
8383 learner_params_free(struct rte_swx_table_learner_params *params)
8388 free(params->key_mask0);
8393 static struct rte_swx_table_learner_params *
8394 learner_params_get(struct learner *l)
8396 struct rte_swx_table_learner_params *params = NULL;
8397 struct field *first, *last;
8400 /* Memory allocation. */
8401 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8405 /* Find first (smallest offset) and last (biggest offset) match fields. */
8406 first = l->fields[0];
8407 last = l->fields[0];
8409 for (i = 0; i < l->n_fields; i++) {
8410 struct field *f = l->fields[i];
8412 if (f->offset < first->offset)
8415 if (f->offset > last->offset)
8419 /* Key offset and size. */
8420 params->key_offset = first->offset / 8;
8421 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8423 /* Memory allocation. */
8424 params->key_mask0 = calloc(1, params->key_size);
8425 if (!params->key_mask0)
8429 for (i = 0; i < l->n_fields; i++) {
8430 struct field *f = l->fields[i];
8431 uint32_t start = (f->offset - first->offset) / 8;
8432 size_t size = f->n_bits / 8;
8434 memset(¶ms->key_mask0[start], 0xFF, size);
8437 /* Action data size. */
8438 params->action_data_size = l->action_data_size_max;
8440 /* Maximum number of keys. */
8441 params->n_keys_max = l->size;
8444 params->key_timeout = l->timeout;
8449 learner_params_free(params);
8454 learner_build_free(struct rte_swx_pipeline *p)
8458 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8459 struct thread *t = &p->threads[i];
8465 for (j = 0; j < p->n_learners; j++) {
8466 struct learner_runtime *r = &t->learners[j];
8475 if (p->learner_stats) {
8476 for (i = 0; i < p->n_learners; i++)
8477 free(p->learner_stats[i].n_pkts_action);
8479 free(p->learner_stats);
8484 learner_build(struct rte_swx_pipeline *p)
8489 /* Per pipeline: learner statistics. */
8490 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8491 CHECK(p->learner_stats, ENOMEM);
8493 for (i = 0; i < p->n_learners; i++) {
8494 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8495 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8498 /* Per thread: learner run-time. */
8499 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8500 struct thread *t = &p->threads[i];
8503 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8509 TAILQ_FOREACH(l, &p->learners, node) {
8510 struct learner_runtime *r = &t->learners[l->id];
8514 size = rte_swx_table_learner_mailbox_size_get();
8516 r->mailbox = calloc(1, size);
8524 r->key = l->header ?
8525 &t->structs[l->header->struct_id] :
8526 &t->structs[p->metadata_struct_id];
8533 learner_build_free(p);
8538 learner_free(struct rte_swx_pipeline *p)
8540 learner_build_free(p);
8542 /* Learner tables. */
8546 l = TAILQ_FIRST(&p->learners);
8550 TAILQ_REMOVE(&p->learners, l, node);
8553 free(l->default_action_data);
8562 table_state_build(struct rte_swx_pipeline *p)
8564 struct table *table;
8568 p->table_state = calloc(p->n_tables + p->n_selectors + p->n_learners,
8569 sizeof(struct rte_swx_table_state));
8570 CHECK(p->table_state, ENOMEM);
8572 TAILQ_FOREACH(table, &p->tables, node) {
8573 struct rte_swx_table_state *ts = &p->table_state[table->id];
8576 struct rte_swx_table_params *params;
8579 params = table_params_get(table);
8580 CHECK(params, ENOMEM);
8582 ts->obj = table->type->ops.create(params,
8587 table_params_free(params);
8588 CHECK(ts->obj, ENODEV);
8591 /* ts->default_action_data. */
8592 if (table->action_data_size_max) {
8593 ts->default_action_data =
8594 malloc(table->action_data_size_max);
8595 CHECK(ts->default_action_data, ENOMEM);
8597 memcpy(ts->default_action_data,
8598 table->default_action_data,
8599 table->action_data_size_max);
8602 /* ts->default_action_id. */
8603 ts->default_action_id = table->default_action->id;
8606 TAILQ_FOREACH(s, &p->selectors, node) {
8607 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8608 struct rte_swx_table_selector_params *params;
8611 params = selector_table_params_get(s);
8612 CHECK(params, ENOMEM);
8614 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8616 selector_params_free(params);
8617 CHECK(ts->obj, ENODEV);
8620 TAILQ_FOREACH(l, &p->learners, node) {
8621 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8622 p->n_selectors + l->id];
8623 struct rte_swx_table_learner_params *params;
8626 params = learner_params_get(l);
8627 CHECK(params, ENOMEM);
8629 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8630 learner_params_free(params);
8631 CHECK(ts->obj, ENODEV);
8633 /* ts->default_action_data. */
8634 if (l->action_data_size_max) {
8635 ts->default_action_data = malloc(l->action_data_size_max);
8636 CHECK(ts->default_action_data, ENOMEM);
8638 memcpy(ts->default_action_data,
8639 l->default_action_data,
8640 l->action_data_size_max);
8643 /* ts->default_action_id. */
8644 ts->default_action_id = l->default_action->id;
8651 table_state_build_free(struct rte_swx_pipeline *p)
8655 if (!p->table_state)
8658 for (i = 0; i < p->n_tables; i++) {
8659 struct rte_swx_table_state *ts = &p->table_state[i];
8660 struct table *table = table_find_by_id(p, i);
8663 if (table->type && ts->obj)
8664 table->type->ops.free(ts->obj);
8666 /* ts->default_action_data. */
8667 free(ts->default_action_data);
8670 for (i = 0; i < p->n_selectors; i++) {
8671 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8675 rte_swx_table_selector_free(ts->obj);
8678 for (i = 0; i < p->n_learners; i++) {
8679 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8683 rte_swx_table_learner_free(ts->obj);
8685 /* ts->default_action_data. */
8686 free(ts->default_action_data);
8689 free(p->table_state);
8690 p->table_state = NULL;
8694 table_state_free(struct rte_swx_pipeline *p)
8696 table_state_build_free(p);
8702 static struct regarray *
8703 regarray_find(struct rte_swx_pipeline *p, const char *name)
8705 struct regarray *elem;
8707 TAILQ_FOREACH(elem, &p->regarrays, node)
8708 if (!strcmp(elem->name, name))
8714 static struct regarray *
8715 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8717 struct regarray *elem = NULL;
8719 TAILQ_FOREACH(elem, &p->regarrays, node)
8727 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8736 CHECK_NAME(name, EINVAL);
8737 CHECK(!regarray_find(p, name), EEXIST);
8739 CHECK(size, EINVAL);
8740 size = rte_align32pow2(size);
8742 /* Memory allocation. */
8743 r = calloc(1, sizeof(struct regarray));
8746 /* Node initialization. */
8747 strcpy(r->name, name);
8748 r->init_val = init_val;
8750 r->id = p->n_regarrays;
8752 /* Node add to tailq. */
8753 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8760 regarray_build(struct rte_swx_pipeline *p)
8762 struct regarray *regarray;
8764 if (!p->n_regarrays)
8767 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8768 CHECK(p->regarray_runtime, ENOMEM);
8770 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8771 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8774 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8775 RTE_CACHE_LINE_SIZE,
8777 CHECK(r->regarray, ENOMEM);
8779 if (regarray->init_val)
8780 for (i = 0; i < regarray->size; i++)
8781 r->regarray[i] = regarray->init_val;
8783 r->size_mask = regarray->size - 1;
8790 regarray_build_free(struct rte_swx_pipeline *p)
8794 if (!p->regarray_runtime)
8797 for (i = 0; i < p->n_regarrays; i++) {
8798 struct regarray *regarray = regarray_find_by_id(p, i);
8799 struct regarray_runtime *r = &p->regarray_runtime[i];
8801 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8804 free(p->regarray_runtime);
8805 p->regarray_runtime = NULL;
8809 regarray_free(struct rte_swx_pipeline *p)
8811 regarray_build_free(p);
8814 struct regarray *elem;
8816 elem = TAILQ_FIRST(&p->regarrays);
8820 TAILQ_REMOVE(&p->regarrays, elem, node);
8828 static struct meter_profile *
8829 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8831 struct meter_profile *elem;
8833 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8834 if (!strcmp(elem->name, name))
8840 static struct metarray *
8841 metarray_find(struct rte_swx_pipeline *p, const char *name)
8843 struct metarray *elem;
8845 TAILQ_FOREACH(elem, &p->metarrays, node)
8846 if (!strcmp(elem->name, name))
8852 static struct metarray *
8853 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8855 struct metarray *elem = NULL;
8857 TAILQ_FOREACH(elem, &p->metarrays, node)
8865 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8873 CHECK_NAME(name, EINVAL);
8874 CHECK(!metarray_find(p, name), EEXIST);
8876 CHECK(size, EINVAL);
8877 size = rte_align32pow2(size);
8879 /* Memory allocation. */
8880 m = calloc(1, sizeof(struct metarray));
8883 /* Node initialization. */
8884 strcpy(m->name, name);
8886 m->id = p->n_metarrays;
8888 /* Node add to tailq. */
8889 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8895 struct meter_profile meter_profile_default = {
8904 .cir_bytes_per_period = 1,
8906 .pir_bytes_per_period = 1,
8913 meter_init(struct meter *m)
8915 memset(m, 0, sizeof(struct meter));
8916 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8917 m->profile = &meter_profile_default;
8918 m->color_mask = RTE_COLOR_GREEN;
8920 meter_profile_default.n_users++;
8924 metarray_build(struct rte_swx_pipeline *p)
8928 if (!p->n_metarrays)
8931 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8932 CHECK(p->metarray_runtime, ENOMEM);
8934 TAILQ_FOREACH(m, &p->metarrays, node) {
8935 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8938 r->metarray = env_malloc(m->size * sizeof(struct meter),
8939 RTE_CACHE_LINE_SIZE,
8941 CHECK(r->metarray, ENOMEM);
8943 for (i = 0; i < m->size; i++)
8944 meter_init(&r->metarray[i]);
8946 r->size_mask = m->size - 1;
8953 metarray_build_free(struct rte_swx_pipeline *p)
8957 if (!p->metarray_runtime)
8960 for (i = 0; i < p->n_metarrays; i++) {
8961 struct metarray *m = metarray_find_by_id(p, i);
8962 struct metarray_runtime *r = &p->metarray_runtime[i];
8964 env_free(r->metarray, m->size * sizeof(struct meter));
8967 free(p->metarray_runtime);
8968 p->metarray_runtime = NULL;
8972 metarray_free(struct rte_swx_pipeline *p)
8974 metarray_build_free(p);
8978 struct metarray *elem;
8980 elem = TAILQ_FIRST(&p->metarrays);
8984 TAILQ_REMOVE(&p->metarrays, elem, node);
8988 /* Meter profiles. */
8990 struct meter_profile *elem;
8992 elem = TAILQ_FIRST(&p->meter_profiles);
8996 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9005 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9014 free(p->instruction_data);
9015 free(p->instructions);
9019 table_state_free(p);
9024 instruction_table_free(p);
9027 extern_func_free(p);
9040 port_in_types_register(struct rte_swx_pipeline *p)
9044 status = rte_swx_pipeline_port_in_type_register(p,
9046 &rte_swx_port_ethdev_reader_ops);
9050 status = rte_swx_pipeline_port_in_type_register(p,
9052 &rte_swx_port_ring_reader_ops);
9056 #ifdef RTE_PORT_PCAP
9057 status = rte_swx_pipeline_port_in_type_register(p,
9059 &rte_swx_port_source_ops);
9064 status = rte_swx_pipeline_port_in_type_register(p,
9066 &rte_swx_port_fd_reader_ops);
9074 port_out_types_register(struct rte_swx_pipeline *p)
9078 status = rte_swx_pipeline_port_out_type_register(p,
9080 &rte_swx_port_ethdev_writer_ops);
9084 status = rte_swx_pipeline_port_out_type_register(p,
9086 &rte_swx_port_ring_writer_ops);
9090 status = rte_swx_pipeline_port_out_type_register(p,
9092 &rte_swx_port_sink_ops);
9096 status = rte_swx_pipeline_port_out_type_register(p,
9098 &rte_swx_port_fd_writer_ops);
9106 table_types_register(struct rte_swx_pipeline *p)
9110 status = rte_swx_pipeline_table_type_register(p,
9112 RTE_SWX_TABLE_MATCH_EXACT,
9113 &rte_swx_table_exact_match_ops);
9117 status = rte_swx_pipeline_table_type_register(p,
9119 RTE_SWX_TABLE_MATCH_WILDCARD,
9120 &rte_swx_table_wildcard_match_ops);
9128 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9130 struct rte_swx_pipeline *pipeline = NULL;
9133 /* Check input parameters. */
9136 /* Memory allocation. */
9137 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9143 /* Initialization. */
9144 TAILQ_INIT(&pipeline->struct_types);
9145 TAILQ_INIT(&pipeline->port_in_types);
9146 TAILQ_INIT(&pipeline->ports_in);
9147 TAILQ_INIT(&pipeline->port_out_types);
9148 TAILQ_INIT(&pipeline->ports_out);
9149 TAILQ_INIT(&pipeline->extern_types);
9150 TAILQ_INIT(&pipeline->extern_objs);
9151 TAILQ_INIT(&pipeline->extern_funcs);
9152 TAILQ_INIT(&pipeline->headers);
9153 TAILQ_INIT(&pipeline->actions);
9154 TAILQ_INIT(&pipeline->table_types);
9155 TAILQ_INIT(&pipeline->tables);
9156 TAILQ_INIT(&pipeline->selectors);
9157 TAILQ_INIT(&pipeline->learners);
9158 TAILQ_INIT(&pipeline->regarrays);
9159 TAILQ_INIT(&pipeline->meter_profiles);
9160 TAILQ_INIT(&pipeline->metarrays);
9162 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9163 pipeline->numa_node = numa_node;
9165 status = port_in_types_register(pipeline);
9169 status = port_out_types_register(pipeline);
9173 status = table_types_register(pipeline);
9181 rte_swx_pipeline_free(pipeline);
9186 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9187 const char **instructions,
9188 uint32_t n_instructions)
9193 err = instruction_config(p, NULL, instructions, n_instructions);
9197 /* Thread instruction pointer reset. */
9198 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9199 struct thread *t = &p->threads[i];
9201 thread_ip_reset(p, t);
9208 pipeline_compile(struct rte_swx_pipeline *p);
9211 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9213 struct rte_swx_port_sink_params drop_port_params = {
9219 CHECK(p->build_done == 0, EEXIST);
9221 status = port_in_build(p);
9226 status = rte_swx_pipeline_port_out_config(p,
9233 status = port_out_build(p);
9237 status = struct_build(p);
9241 status = extern_obj_build(p);
9245 status = extern_func_build(p);
9249 status = header_build(p);
9253 status = metadata_build(p);
9257 status = instruction_table_build(p);
9261 status = action_build(p);
9265 status = table_build(p);
9269 status = selector_build(p);
9273 status = learner_build(p);
9277 status = table_state_build(p);
9281 status = regarray_build(p);
9285 status = metarray_build(p);
9291 pipeline_compile(p);
9296 metarray_build_free(p);
9297 regarray_build_free(p);
9298 table_state_build_free(p);
9299 learner_build_free(p);
9300 selector_build_free(p);
9301 table_build_free(p);
9302 action_build_free(p);
9303 instruction_table_build_free(p);
9304 metadata_build_free(p);
9305 header_build_free(p);
9306 extern_func_build_free(p);
9307 extern_obj_build_free(p);
9308 port_out_build_free(p);
9309 port_in_build_free(p);
9310 struct_build_free(p);
9316 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9320 for (i = 0; i < n_instructions; i++)
9325 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9329 for (i = 0; i < p->n_ports_out; i++) {
9330 struct port_out_runtime *port = &p->out[i];
9333 port->flush(port->obj);
9341 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9342 struct rte_swx_ctl_pipeline_info *pipeline)
9344 struct action *action;
9345 struct table *table;
9346 uint32_t n_actions = 0, n_tables = 0;
9348 if (!p || !pipeline)
9351 TAILQ_FOREACH(action, &p->actions, node)
9354 TAILQ_FOREACH(table, &p->tables, node)
9357 pipeline->n_ports_in = p->n_ports_in;
9358 pipeline->n_ports_out = p->n_ports_out;
9359 pipeline->n_actions = n_actions;
9360 pipeline->n_tables = n_tables;
9361 pipeline->n_selectors = p->n_selectors;
9362 pipeline->n_learners = p->n_learners;
9363 pipeline->n_regarrays = p->n_regarrays;
9364 pipeline->n_metarrays = p->n_metarrays;
9370 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9372 if (!p || !numa_node)
9375 *numa_node = p->numa_node;
9380 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9382 struct rte_swx_ctl_action_info *action)
9384 struct action *a = NULL;
9386 if (!p || (action_id >= p->n_actions) || !action)
9389 a = action_find_by_id(p, action_id);
9393 strcpy(action->name, a->name);
9394 action->n_args = a->st ? a->st->n_fields : 0;
9399 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9401 uint32_t action_arg_id,
9402 struct rte_swx_ctl_action_arg_info *action_arg)
9404 struct action *a = NULL;
9405 struct field *arg = NULL;
9407 if (!p || (action_id >= p->n_actions) || !action_arg)
9410 a = action_find_by_id(p, action_id);
9411 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9414 arg = &a->st->fields[action_arg_id];
9415 strcpy(action_arg->name, arg->name);
9416 action_arg->n_bits = arg->n_bits;
9417 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9423 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9425 struct rte_swx_ctl_table_info *table)
9427 struct table *t = NULL;
9432 t = table_find_by_id(p, table_id);
9436 strcpy(table->name, t->name);
9437 strcpy(table->args, t->args);
9438 table->n_match_fields = t->n_fields;
9439 table->n_actions = t->n_actions;
9440 table->default_action_is_const = t->default_action_is_const;
9441 table->size = t->size;
9446 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9448 uint32_t match_field_id,
9449 struct rte_swx_ctl_table_match_field_info *match_field)
9452 struct match_field *f;
9454 if (!p || (table_id >= p->n_tables) || !match_field)
9457 t = table_find_by_id(p, table_id);
9458 if (!t || (match_field_id >= t->n_fields))
9461 f = &t->fields[match_field_id];
9462 match_field->match_type = f->match_type;
9463 match_field->is_header = t->header ? 1 : 0;
9464 match_field->n_bits = f->field->n_bits;
9465 match_field->offset = f->field->offset;
9471 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9473 uint32_t table_action_id,
9474 struct rte_swx_ctl_table_action_info *table_action)
9478 if (!p || (table_id >= p->n_tables) || !table_action)
9481 t = table_find_by_id(p, table_id);
9482 if (!t || (table_action_id >= t->n_actions))
9485 table_action->action_id = t->actions[table_action_id]->id;
9487 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9488 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9494 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9496 struct rte_swx_table_ops *table_ops,
9501 if (!p || (table_id >= p->n_tables))
9504 t = table_find_by_id(p, table_id);
9510 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9520 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9521 uint32_t selector_id,
9522 struct rte_swx_ctl_selector_info *selector)
9524 struct selector *s = NULL;
9526 if (!p || !selector)
9529 s = selector_find_by_id(p, selector_id);
9533 strcpy(selector->name, s->name);
9535 selector->n_selector_fields = s->n_selector_fields;
9536 selector->n_groups_max = s->n_groups_max;
9537 selector->n_members_per_group_max = s->n_members_per_group_max;
9543 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9544 uint32_t selector_id,
9545 struct rte_swx_ctl_table_match_field_info *field)
9549 if (!p || (selector_id >= p->n_selectors) || !field)
9552 s = selector_find_by_id(p, selector_id);
9556 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9557 field->is_header = 0;
9558 field->n_bits = s->group_id_field->n_bits;
9559 field->offset = s->group_id_field->offset;
9565 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9566 uint32_t selector_id,
9567 uint32_t selector_field_id,
9568 struct rte_swx_ctl_table_match_field_info *field)
9573 if (!p || (selector_id >= p->n_selectors) || !field)
9576 s = selector_find_by_id(p, selector_id);
9577 if (!s || (selector_field_id >= s->n_selector_fields))
9580 f = s->selector_fields[selector_field_id];
9581 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9582 field->is_header = s->selector_header ? 1 : 0;
9583 field->n_bits = f->n_bits;
9584 field->offset = f->offset;
9590 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9591 uint32_t selector_id,
9592 struct rte_swx_ctl_table_match_field_info *field)
9596 if (!p || (selector_id >= p->n_selectors) || !field)
9599 s = selector_find_by_id(p, selector_id);
9603 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9604 field->is_header = 0;
9605 field->n_bits = s->member_id_field->n_bits;
9606 field->offset = s->member_id_field->offset;
9612 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9613 uint32_t learner_id,
9614 struct rte_swx_ctl_learner_info *learner)
9616 struct learner *l = NULL;
9621 l = learner_find_by_id(p, learner_id);
9625 strcpy(learner->name, l->name);
9627 learner->n_match_fields = l->n_fields;
9628 learner->n_actions = l->n_actions;
9629 learner->default_action_is_const = l->default_action_is_const;
9630 learner->size = l->size;
9636 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9637 uint32_t learner_id,
9638 uint32_t match_field_id,
9639 struct rte_swx_ctl_table_match_field_info *match_field)
9644 if (!p || (learner_id >= p->n_learners) || !match_field)
9647 l = learner_find_by_id(p, learner_id);
9648 if (!l || (match_field_id >= l->n_fields))
9651 f = l->fields[match_field_id];
9652 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9653 match_field->is_header = l->header ? 1 : 0;
9654 match_field->n_bits = f->n_bits;
9655 match_field->offset = f->offset;
9661 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9662 uint32_t learner_id,
9663 uint32_t learner_action_id,
9664 struct rte_swx_ctl_table_action_info *learner_action)
9668 if (!p || (learner_id >= p->n_learners) || !learner_action)
9671 l = learner_find_by_id(p, learner_id);
9672 if (!l || (learner_action_id >= l->n_actions))
9675 learner_action->action_id = l->actions[learner_action_id]->id;
9677 learner_action->action_is_for_table_entries =
9678 l->action_is_for_table_entries[learner_action_id];
9680 learner_action->action_is_for_default_entry =
9681 l->action_is_for_default_entry[learner_action_id];
9687 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9688 struct rte_swx_table_state **table_state)
9690 if (!p || !table_state || !p->build_done)
9693 *table_state = p->table_state;
9698 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9699 struct rte_swx_table_state *table_state)
9701 if (!p || !table_state || !p->build_done)
9704 p->table_state = table_state;
9709 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9711 struct rte_swx_port_in_stats *stats)
9713 struct port_in *port;
9718 port = port_in_find(p, port_id);
9722 port->type->ops.stats_read(port->obj, stats);
9727 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9729 struct rte_swx_port_out_stats *stats)
9731 struct port_out *port;
9736 port = port_out_find(p, port_id);
9740 port->type->ops.stats_read(port->obj, stats);
9745 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9746 const char *table_name,
9747 struct rte_swx_table_stats *stats)
9749 struct table *table;
9750 struct table_statistics *table_stats;
9752 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9755 table = table_find(p, table_name);
9759 table_stats = &p->table_stats[table->id];
9761 memcpy(stats->n_pkts_action,
9762 table_stats->n_pkts_action,
9763 p->n_actions * sizeof(uint64_t));
9765 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9766 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9772 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9773 const char *selector_name,
9774 struct rte_swx_pipeline_selector_stats *stats)
9778 if (!p || !selector_name || !selector_name[0] || !stats)
9781 s = selector_find(p, selector_name);
9785 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9791 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9792 const char *learner_name,
9793 struct rte_swx_learner_stats *stats)
9796 struct learner_statistics *learner_stats;
9798 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9801 l = learner_find(p, learner_name);
9805 learner_stats = &p->learner_stats[l->id];
9807 memcpy(stats->n_pkts_action,
9808 learner_stats->n_pkts_action,
9809 p->n_actions * sizeof(uint64_t));
9811 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9812 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9814 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9815 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9817 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9823 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9824 uint32_t regarray_id,
9825 struct rte_swx_ctl_regarray_info *regarray)
9829 if (!p || !regarray)
9832 r = regarray_find_by_id(p, regarray_id);
9836 strcpy(regarray->name, r->name);
9837 regarray->size = r->size;
9842 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9843 const char *regarray_name,
9844 uint32_t regarray_index,
9847 struct regarray *regarray;
9848 struct regarray_runtime *r;
9850 if (!p || !regarray_name || !value)
9853 regarray = regarray_find(p, regarray_name);
9854 if (!regarray || (regarray_index >= regarray->size))
9857 r = &p->regarray_runtime[regarray->id];
9858 *value = r->regarray[regarray_index];
9863 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9864 const char *regarray_name,
9865 uint32_t regarray_index,
9868 struct regarray *regarray;
9869 struct regarray_runtime *r;
9871 if (!p || !regarray_name)
9874 regarray = regarray_find(p, regarray_name);
9875 if (!regarray || (regarray_index >= regarray->size))
9878 r = &p->regarray_runtime[regarray->id];
9879 r->regarray[regarray_index] = value;
9884 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9885 uint32_t metarray_id,
9886 struct rte_swx_ctl_metarray_info *metarray)
9890 if (!p || !metarray)
9893 m = metarray_find_by_id(p, metarray_id);
9897 strcpy(metarray->name, m->name);
9898 metarray->size = m->size;
9903 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9905 struct rte_meter_trtcm_params *params)
9907 struct meter_profile *mp;
9911 CHECK_NAME(name, EINVAL);
9912 CHECK(params, EINVAL);
9913 CHECK(!meter_profile_find(p, name), EEXIST);
9915 /* Node allocation. */
9916 mp = calloc(1, sizeof(struct meter_profile));
9919 /* Node initialization. */
9920 strcpy(mp->name, name);
9921 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9922 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9928 /* Node add to tailq. */
9929 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9935 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9938 struct meter_profile *mp;
9941 CHECK_NAME(name, EINVAL);
9943 mp = meter_profile_find(p, name);
9945 CHECK(!mp->n_users, EBUSY);
9947 /* Remove node from tailq. */
9948 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9955 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9956 const char *metarray_name,
9957 uint32_t metarray_index)
9959 struct meter_profile *mp_old;
9960 struct metarray *metarray;
9961 struct metarray_runtime *metarray_runtime;
9965 CHECK_NAME(metarray_name, EINVAL);
9967 metarray = metarray_find(p, metarray_name);
9968 CHECK(metarray, EINVAL);
9969 CHECK(metarray_index < metarray->size, EINVAL);
9971 metarray_runtime = &p->metarray_runtime[metarray->id];
9972 m = &metarray_runtime->metarray[metarray_index];
9973 mp_old = m->profile;
9983 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9984 const char *metarray_name,
9985 uint32_t metarray_index,
9986 const char *profile_name)
9988 struct meter_profile *mp, *mp_old;
9989 struct metarray *metarray;
9990 struct metarray_runtime *metarray_runtime;
9994 CHECK_NAME(metarray_name, EINVAL);
9996 metarray = metarray_find(p, metarray_name);
9997 CHECK(metarray, EINVAL);
9998 CHECK(metarray_index < metarray->size, EINVAL);
10000 mp = meter_profile_find(p, profile_name);
10003 metarray_runtime = &p->metarray_runtime[metarray->id];
10004 m = &metarray_runtime->metarray[metarray_index];
10005 mp_old = m->profile;
10007 memset(m, 0, sizeof(struct meter));
10008 rte_meter_trtcm_config(&m->m, &mp->profile);
10010 m->color_mask = RTE_COLORS;
10019 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10020 const char *metarray_name,
10021 uint32_t metarray_index,
10022 struct rte_swx_ctl_meter_stats *stats)
10024 struct metarray *metarray;
10025 struct metarray_runtime *metarray_runtime;
10029 CHECK_NAME(metarray_name, EINVAL);
10031 metarray = metarray_find(p, metarray_name);
10032 CHECK(metarray, EINVAL);
10033 CHECK(metarray_index < metarray->size, EINVAL);
10035 CHECK(stats, EINVAL);
10037 metarray_runtime = &p->metarray_runtime[metarray->id];
10038 m = &metarray_runtime->metarray[metarray_index];
10040 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10041 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
10047 * Pipeline compilation.
10049 static const char *
10050 instr_type_to_name(struct instruction *instr)
10052 switch (instr->type) {
10053 case INSTR_RX: return "INSTR_RX";
10055 case INSTR_TX: return "INSTR_TX";
10056 case INSTR_TX_I: return "INSTR_TX_I";
10057 case INSTR_DROP: return "INSTR_DROP";
10059 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10060 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10061 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10062 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10063 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10064 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10065 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10066 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10068 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10070 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10072 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10073 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10074 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10075 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10076 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10077 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10078 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10079 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10080 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10082 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10083 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10085 case INSTR_MOV: return "INSTR_MOV";
10086 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10087 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10088 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10089 case INSTR_MOV_I: return "INSTR_MOV_I";
10091 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10092 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10093 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10094 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10095 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10096 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10097 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10098 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10100 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10101 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10102 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10103 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10104 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10105 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10107 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10108 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10109 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10110 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10111 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10112 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10114 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10115 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10116 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10117 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10119 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10120 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10121 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10122 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10123 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10125 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10126 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10127 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10128 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10129 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10131 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10132 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10133 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10134 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10135 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10137 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10138 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10139 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10140 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10141 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10142 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10144 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10145 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10146 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10147 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10148 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10149 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10151 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10152 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10153 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10155 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10156 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10157 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10158 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10159 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10160 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10162 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10163 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10164 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10165 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10166 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10167 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10168 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10169 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10170 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10172 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10173 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10174 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10175 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10176 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10177 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10178 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10179 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10180 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10182 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10183 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10184 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10186 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10187 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10188 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10189 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10190 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10191 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10192 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10193 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10194 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10195 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10196 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10197 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10199 case INSTR_TABLE: return "INSTR_TABLE";
10200 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10201 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10202 case INSTR_LEARNER: return "INSTR_LEARNER";
10203 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10205 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10206 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10208 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10209 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10211 case INSTR_JMP: return "INSTR_JMP";
10212 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10213 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10214 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10215 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10216 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10217 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10218 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10219 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10220 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10221 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10222 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10223 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10224 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10225 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10226 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10227 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10228 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10229 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10230 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10231 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10232 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10233 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10234 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10235 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10236 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10237 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10238 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10239 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10241 case INSTR_RETURN: return "INSTR_RETURN";
10243 default: return "INSTR_UNKNOWN";
10248 (*instruction_export_t)(struct instruction *, FILE *);
10251 instr_io_export(struct instruction *instr, FILE *f)
10253 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10255 /* n_io, n_io_imm, n_hdrs. */
10256 if (instr->type == INSTR_RX ||
10257 instr->type == INSTR_TX ||
10258 instr->type == INSTR_HDR_EXTRACT_M ||
10259 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10262 if (instr->type == INSTR_TX_I)
10265 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10266 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10268 if (instr->type == INSTR_HDR_EXTRACT_M ||
10269 instr->type == INSTR_HDR_LOOKAHEAD ||
10270 instr->type == INSTR_HDR_EMIT)
10273 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10274 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10279 "\t\t.type = %s,\n",
10280 instr_type_to_name(instr));
10283 if (n_io || n_io_imm || n_hdrs)
10291 "\t\t\t\t.offset = %u,\n"
10292 "\t\t\t\t.n_bits = %u,\n"
10294 instr->io.io.offset,
10295 instr->io.io.n_bits);
10300 "\t\t\t\t.val = %u,\n"
10304 /* instr.io.hdr. */
10309 /* instr.io.hdr.header_id. */
10311 "\t\t\t.header_id = {");
10313 for (i = 0; i < n_hdrs; i++)
10316 instr->io.hdr.header_id[i]);
10321 /* instr.io.hdr.struct_id. */
10323 "\t\t\t.struct_id = {");
10325 for (i = 0; i < n_hdrs; i++)
10328 instr->io.hdr.struct_id[i]);
10333 /* instr.io.hdr.n_bytes. */
10335 "\t\t\t.n_bytes = {");
10337 for (i = 0; i < n_hdrs; i++)
10340 instr->io.hdr.n_bytes[i]);
10345 /* instr.io.hdr - closing curly brace. */
10350 /* instr.io - closing curly brace. */
10351 if (n_io || n_io_imm || n_hdrs)
10355 /* instr - closing curly brace. */
10361 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10365 "\t\t.type = %s,\n"
10367 "\t\t\t.header_id = %u,\n"
10370 instr_type_to_name(instr),
10371 instr->valid.header_id);
10375 instr_mov_export(struct instruction *instr, FILE *f)
10377 if (instr->type != INSTR_MOV_I)
10380 "\t\t.type = %s,\n"
10383 "\t\t\t\t.struct_id = %u,\n"
10384 "\t\t\t\t.n_bits = %u,\n"
10385 "\t\t\t\t.offset = %u,\n"
10388 "\t\t\t\t.struct_id = %u,\n"
10389 "\t\t\t\t.n_bits = %u,\n"
10390 "\t\t\t\t.offset = %u,\n"
10394 instr_type_to_name(instr),
10395 instr->mov.dst.struct_id,
10396 instr->mov.dst.n_bits,
10397 instr->mov.dst.offset,
10398 instr->mov.src.struct_id,
10399 instr->mov.src.n_bits,
10400 instr->mov.src.offset);
10404 "\t\t.type = %s,\n"
10407 "\t\t\t\t.struct_id = %u,\n"
10408 "\t\t\t\t.n_bits = %u,\n"
10409 "\t\t\t\t.offset = %u,\n"
10411 "\t\t\t.src_val = %" PRIu64 ",\n"
10414 instr_type_to_name(instr),
10415 instr->mov.dst.struct_id,
10416 instr->mov.dst.n_bits,
10417 instr->mov.dst.offset,
10418 instr->mov.src_val);
10422 instr_dma_ht_export(struct instruction *instr, FILE *f)
10424 uint32_t n_dma = 0, i;
10427 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10432 "\t\t.type = %s,\n",
10433 instr_type_to_name(instr));
10439 /* instr.dma.dst. */
10441 "\t\t\t.dst = {\n");
10443 /* instr.dma.dst.header_id. */
10445 "\t\t\t\t.header_id = {");
10447 for (i = 0; i < n_dma; i++)
10450 instr->dma.dst.header_id[i]);
10455 /* instr.dma.dst.struct_id. */
10457 "\t\t\t\t.struct_id = {");
10459 for (i = 0; i < n_dma; i++)
10462 instr->dma.dst.struct_id[i]);
10467 /* instr.dma.dst - closing curly brace. */
10471 /* instr.dma.src. */
10473 "\t\t\t.src = {\n");
10475 /* instr.dma.src.offset. */
10477 "\t\t\t\t.offset = {");
10479 for (i = 0; i < n_dma; i++)
10482 instr->dma.src.offset[i]);
10487 /* instr.dma.src - closing curly brace. */
10491 /* instr.dma.n_bytes. */
10493 "\t\t\t.n_bytes = {");
10495 for (i = 0; i < n_dma; i++)
10498 instr->dma.n_bytes[i]);
10503 /* instr.dma - closing curly brace. */
10507 /* instr - closing curly brace. */
10513 instr_alu_export(struct instruction *instr, FILE *f)
10517 if (instr->type == INSTR_ALU_ADD_MI ||
10518 instr->type == INSTR_ALU_ADD_HI ||
10519 instr->type == INSTR_ALU_SUB_MI ||
10520 instr->type == INSTR_ALU_SUB_HI ||
10521 instr->type == INSTR_ALU_SHL_MI ||
10522 instr->type == INSTR_ALU_SHL_HI ||
10523 instr->type == INSTR_ALU_SHR_MI ||
10524 instr->type == INSTR_ALU_SHR_HI ||
10525 instr->type == INSTR_ALU_AND_I ||
10526 instr->type == INSTR_ALU_OR_I ||
10527 instr->type == INSTR_ALU_XOR_I)
10533 "\t\t.type = %s,\n"
10536 "\t\t\t\t.struct_id = %u,\n"
10537 "\t\t\t\t.n_bits = %u,\n"
10538 "\t\t\t\t.offset = %u,\n"
10541 "\t\t\t\t.struct_id = %u,\n"
10542 "\t\t\t\t.n_bits = %u,\n"
10543 "\t\t\t\t.offset = %u,\n"
10547 instr_type_to_name(instr),
10548 instr->alu.dst.struct_id,
10549 instr->alu.dst.n_bits,
10550 instr->alu.dst.offset,
10551 instr->alu.src.struct_id,
10552 instr->alu.src.n_bits,
10553 instr->alu.src.offset);
10557 "\t\t.type = %s,\n"
10560 "\t\t\t\t.struct_id = %u,\n"
10561 "\t\t\t\t.n_bits = %u,\n"
10562 "\t\t\t\t.offset = %u,\n"
10564 "\t\t\t.src_val = %" PRIu64 ",\n"
10567 instr_type_to_name(instr),
10568 instr->alu.dst.struct_id,
10569 instr->alu.dst.n_bits,
10570 instr->alu.dst.offset,
10571 instr->alu.src_val);
10575 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10577 int prefetch = 0, idx_imm = 0, src_imm = 0;
10579 if (instr->type == INSTR_REGPREFETCH_RH ||
10580 instr->type == INSTR_REGPREFETCH_RM ||
10581 instr->type == INSTR_REGPREFETCH_RI)
10584 /* index is the 3rd operand for the regrd instruction and the 2nd
10585 * operand for the regwr and regadd instructions.
10587 if (instr->type == INSTR_REGPREFETCH_RI ||
10588 instr->type == INSTR_REGRD_HRI ||
10589 instr->type == INSTR_REGRD_MRI ||
10590 instr->type == INSTR_REGWR_RIH ||
10591 instr->type == INSTR_REGWR_RIM ||
10592 instr->type == INSTR_REGWR_RII ||
10593 instr->type == INSTR_REGADD_RIH ||
10594 instr->type == INSTR_REGADD_RIM ||
10595 instr->type == INSTR_REGADD_RII)
10598 /* src is the 3rd operand for the regwr and regadd instructions. */
10599 if (instr->type == INSTR_REGWR_RHI ||
10600 instr->type == INSTR_REGWR_RMI ||
10601 instr->type == INSTR_REGWR_RII ||
10602 instr->type == INSTR_REGADD_RHI ||
10603 instr->type == INSTR_REGADD_RMI ||
10604 instr->type == INSTR_REGADD_RII)
10607 /* instr.regarray.regarray_id. */
10610 "\t\t.type = %s,\n"
10611 "\t\t.regarray = {\n"
10612 "\t\t\t.regarray_id = %u,\n",
10613 instr_type_to_name(instr),
10614 instr->regarray.regarray_id);
10616 /* instr.regarray.idx / instr.regarray.idx_val. */
10619 "\t\t\t\t.idx = {\n"
10620 "\t\t\t\t\t.struct_id = %u,\n"
10621 "\t\t\t\t\t.n_bits = %u,\n"
10622 "\t\t\t\t\t.offset = %u,\n"
10624 instr->regarray.idx.struct_id,
10625 instr->regarray.idx.n_bits,
10626 instr->regarray.idx.offset);
10629 "\t\t\t\t.idx_val = %u,\n",
10630 instr->regarray.idx_val);
10632 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10636 "\t\t\t\t.dstsrc = {\n"
10637 "\t\t\t\t\t.struct_id = %u,\n"
10638 "\t\t\t\t\t.n_bits = %u,\n"
10639 "\t\t\t\t\t.offset = %u,\n"
10641 instr->regarray.dstsrc.struct_id,
10642 instr->regarray.dstsrc.n_bits,
10643 instr->regarray.dstsrc.offset);
10646 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10647 instr->regarray.dstsrc_val);
10650 /* instr.regarray and instr - closing curly braces. */
10657 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10659 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10661 if (instr->type == INSTR_METPREFETCH_H ||
10662 instr->type == INSTR_METPREFETCH_M ||
10663 instr->type == INSTR_METPREFETCH_I)
10667 if (instr->type == INSTR_METPREFETCH_I ||
10668 instr->type == INSTR_METER_IHM ||
10669 instr->type == INSTR_METER_IHI ||
10670 instr->type == INSTR_METER_IMM ||
10671 instr->type == INSTR_METER_IMI)
10674 /* color_in_imm. */
10675 if (instr->type == INSTR_METER_HHI ||
10676 instr->type == INSTR_METER_HMI ||
10677 instr->type == INSTR_METER_MHI ||
10678 instr->type == INSTR_METER_MMI ||
10679 instr->type == INSTR_METER_IHI ||
10680 instr->type == INSTR_METER_IMI)
10683 /* instr.meter.metarray_id. */
10686 "\t\t.type = %s,\n"
10688 "\t\t\t.metarray_id = %u,\n",
10689 instr_type_to_name(instr),
10690 instr->meter.metarray_id);
10692 /* instr.meter.idx / instr.meter.idx_val. */
10696 "\t\t\t\t.struct_id = %u,\n"
10697 "\t\t\t\t.n_bits = %u,\n"
10698 "\t\t\t\t.offset = %u,\n"
10700 instr->meter.idx.struct_id,
10701 instr->meter.idx.n_bits,
10702 instr->meter.idx.offset);
10705 "\t\t\t.idx_val = %u,\n",
10706 instr->meter.idx_val);
10709 /* instr.meter.length. */
10711 "\t\t\t.length = {\n"
10712 "\t\t\t\t.struct_id = %u,\n"
10713 "\t\t\t\t.n_bits = %u,\n"
10714 "\t\t\t\t.offset = %u,\n"
10716 instr->meter.length.struct_id,
10717 instr->meter.length.n_bits,
10718 instr->meter.length.offset);
10720 /* instr.meter.color_in / instr.meter.color_in_val. */
10723 "\t\t\t.color_in = {\n"
10724 "\t\t\t\t.struct_id = %u,\n"
10725 "\t\t\t\t.n_bits = %u,\n"
10726 "\t\t\t\t.offset = %u,\n"
10728 instr->meter.color_in.struct_id,
10729 instr->meter.color_in.n_bits,
10730 instr->meter.color_in.offset);
10733 "\t\t\t.color_in_val = %u,\n",
10734 (uint32_t)instr->meter.color_in_val);
10736 /* instr.meter.color_out. */
10738 "\t\t\t.color_out = {\n"
10739 "\t\t\t\t.struct_id = %u,\n"
10740 "\t\t\t\t.n_bits = %u,\n"
10741 "\t\t\t\t.offset = %u,\n"
10743 instr->meter.color_out.struct_id,
10744 instr->meter.color_out.n_bits,
10745 instr->meter.color_out.offset);
10748 /* instr.meter and instr - closing curly braces. */
10755 instr_table_export(struct instruction *instr,
10760 "\t\t.type = %s,\n"
10762 "\t\t\t.table_id = %u,\n"
10765 instr_type_to_name(instr),
10766 instr->table.table_id);
10770 instr_learn_export(struct instruction *instr, FILE *f)
10774 "\t\t.type = %s,\n"
10776 "\t\t\t\t.action_id = %u,\n"
10779 instr_type_to_name(instr),
10780 instr->learn.action_id);
10784 instr_forget_export(struct instruction *instr, FILE *f)
10788 "\t\t.type = %s,\n"
10790 instr_type_to_name(instr));
10794 instr_extern_export(struct instruction *instr, FILE *f)
10796 if (instr->type == INSTR_EXTERN_OBJ)
10799 "\t\t.type = %s,\n"
10800 "\t\t.ext_obj = {\n"
10801 "\t\t\t.ext_obj_id = %u,\n"
10802 "\t\t\t.func_id = %u,\n"
10805 instr_type_to_name(instr),
10806 instr->ext_obj.ext_obj_id,
10807 instr->ext_obj.func_id);
10811 "\t\t.type = %s,\n"
10812 "\t\t.ext_func = {\n"
10813 "\t\t\t.ext_func_id = %u,\n"
10816 instr_type_to_name(instr),
10817 instr->ext_func.ext_func_id);
10821 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10825 "\t\t.type = %s,\n"
10827 "\t\t\t.ip = NULL,\n",
10828 instr_type_to_name(instr));
10830 switch (instr->type) {
10831 case INSTR_JMP_VALID:
10832 case INSTR_JMP_INVALID:
10834 "\t\t\t.header_id = %u,\n",
10835 instr->jmp.header_id);
10838 case INSTR_JMP_ACTION_HIT:
10839 case INSTR_JMP_ACTION_MISS:
10841 "\t\t\t.action_id = %u,\n",
10842 instr->jmp.action_id);
10846 case INSTR_JMP_EQ_MH:
10847 case INSTR_JMP_EQ_HM:
10848 case INSTR_JMP_EQ_HH:
10849 case INSTR_JMP_NEQ:
10850 case INSTR_JMP_NEQ_MH:
10851 case INSTR_JMP_NEQ_HM:
10852 case INSTR_JMP_NEQ_HH:
10854 case INSTR_JMP_LT_MH:
10855 case INSTR_JMP_LT_HM:
10856 case INSTR_JMP_LT_HH:
10858 case INSTR_JMP_GT_MH:
10859 case INSTR_JMP_GT_HM:
10860 case INSTR_JMP_GT_HH:
10863 "\t\t\t\t.struct_id = %u,\n"
10864 "\t\t\t\t.n_bits = %u,\n"
10865 "\t\t\t\t.offset = %u,\n"
10868 "\t\t\t\t.struct_id = %u,\n"
10869 "\t\t\t\t.n_bits = %u,\n"
10870 "\t\t\t\t.offset = %u,\n"
10872 instr->jmp.a.struct_id,
10873 instr->jmp.a.n_bits,
10874 instr->jmp.a.offset,
10875 instr->jmp.b.struct_id,
10876 instr->jmp.b.n_bits,
10877 instr->jmp.b.offset);
10880 case INSTR_JMP_EQ_I:
10881 case INSTR_JMP_NEQ_I:
10882 case INSTR_JMP_LT_MI:
10883 case INSTR_JMP_LT_HI:
10884 case INSTR_JMP_GT_MI:
10885 case INSTR_JMP_GT_HI:
10888 "\t\t\t\t.struct_id = %u,\n"
10889 "\t\t\t\t.n_bits = %u,\n"
10890 "\t\t\t\t.offset = %u,\n"
10892 "\t\t\t.b_val = %" PRIu64 ",\n",
10893 instr->jmp.a.struct_id,
10894 instr->jmp.a.n_bits,
10895 instr->jmp.a.offset,
10909 instr_return_export(struct instruction *instr,
10914 "\t\t.type = %s,\n",
10915 instr_type_to_name(instr));
10921 static instruction_export_t export_table[] = {
10922 [INSTR_RX] = instr_io_export,
10924 [INSTR_TX] = instr_io_export,
10925 [INSTR_TX_I] = instr_io_export,
10926 [INSTR_DROP] = instr_io_export,
10928 [INSTR_HDR_EXTRACT] = instr_io_export,
10929 [INSTR_HDR_EXTRACT2] = instr_io_export,
10930 [INSTR_HDR_EXTRACT3] = instr_io_export,
10931 [INSTR_HDR_EXTRACT4] = instr_io_export,
10932 [INSTR_HDR_EXTRACT5] = instr_io_export,
10933 [INSTR_HDR_EXTRACT6] = instr_io_export,
10934 [INSTR_HDR_EXTRACT7] = instr_io_export,
10935 [INSTR_HDR_EXTRACT8] = instr_io_export,
10937 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10939 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10941 [INSTR_HDR_EMIT] = instr_io_export,
10942 [INSTR_HDR_EMIT_TX] = instr_io_export,
10943 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10944 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10945 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10946 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10947 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10948 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10949 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10951 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10952 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10954 [INSTR_MOV] = instr_mov_export,
10955 [INSTR_MOV_MH] = instr_mov_export,
10956 [INSTR_MOV_HM] = instr_mov_export,
10957 [INSTR_MOV_HH] = instr_mov_export,
10958 [INSTR_MOV_I] = instr_mov_export,
10960 [INSTR_DMA_HT] = instr_dma_ht_export,
10961 [INSTR_DMA_HT2] = instr_dma_ht_export,
10962 [INSTR_DMA_HT3] = instr_dma_ht_export,
10963 [INSTR_DMA_HT4] = instr_dma_ht_export,
10964 [INSTR_DMA_HT5] = instr_dma_ht_export,
10965 [INSTR_DMA_HT6] = instr_dma_ht_export,
10966 [INSTR_DMA_HT7] = instr_dma_ht_export,
10967 [INSTR_DMA_HT8] = instr_dma_ht_export,
10969 [INSTR_ALU_ADD] = instr_alu_export,
10970 [INSTR_ALU_ADD_MH] = instr_alu_export,
10971 [INSTR_ALU_ADD_HM] = instr_alu_export,
10972 [INSTR_ALU_ADD_HH] = instr_alu_export,
10973 [INSTR_ALU_ADD_MI] = instr_alu_export,
10974 [INSTR_ALU_ADD_HI] = instr_alu_export,
10976 [INSTR_ALU_SUB] = instr_alu_export,
10977 [INSTR_ALU_SUB_MH] = instr_alu_export,
10978 [INSTR_ALU_SUB_HM] = instr_alu_export,
10979 [INSTR_ALU_SUB_HH] = instr_alu_export,
10980 [INSTR_ALU_SUB_MI] = instr_alu_export,
10981 [INSTR_ALU_SUB_HI] = instr_alu_export,
10983 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10984 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10985 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10986 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10988 [INSTR_ALU_AND] = instr_alu_export,
10989 [INSTR_ALU_AND_MH] = instr_alu_export,
10990 [INSTR_ALU_AND_HM] = instr_alu_export,
10991 [INSTR_ALU_AND_HH] = instr_alu_export,
10992 [INSTR_ALU_AND_I] = instr_alu_export,
10994 [INSTR_ALU_OR] = instr_alu_export,
10995 [INSTR_ALU_OR_MH] = instr_alu_export,
10996 [INSTR_ALU_OR_HM] = instr_alu_export,
10997 [INSTR_ALU_OR_HH] = instr_alu_export,
10998 [INSTR_ALU_OR_I] = instr_alu_export,
11000 [INSTR_ALU_XOR] = instr_alu_export,
11001 [INSTR_ALU_XOR_MH] = instr_alu_export,
11002 [INSTR_ALU_XOR_HM] = instr_alu_export,
11003 [INSTR_ALU_XOR_HH] = instr_alu_export,
11004 [INSTR_ALU_XOR_I] = instr_alu_export,
11006 [INSTR_ALU_SHL] = instr_alu_export,
11007 [INSTR_ALU_SHL_MH] = instr_alu_export,
11008 [INSTR_ALU_SHL_HM] = instr_alu_export,
11009 [INSTR_ALU_SHL_HH] = instr_alu_export,
11010 [INSTR_ALU_SHL_MI] = instr_alu_export,
11011 [INSTR_ALU_SHL_HI] = instr_alu_export,
11013 [INSTR_ALU_SHR] = instr_alu_export,
11014 [INSTR_ALU_SHR_MH] = instr_alu_export,
11015 [INSTR_ALU_SHR_HM] = instr_alu_export,
11016 [INSTR_ALU_SHR_HH] = instr_alu_export,
11017 [INSTR_ALU_SHR_MI] = instr_alu_export,
11018 [INSTR_ALU_SHR_HI] = instr_alu_export,
11020 [INSTR_REGPREFETCH_RH] = instr_reg_export,
11021 [INSTR_REGPREFETCH_RM] = instr_reg_export,
11022 [INSTR_REGPREFETCH_RI] = instr_reg_export,
11024 [INSTR_REGRD_HRH] = instr_reg_export,
11025 [INSTR_REGRD_HRM] = instr_reg_export,
11026 [INSTR_REGRD_MRH] = instr_reg_export,
11027 [INSTR_REGRD_MRM] = instr_reg_export,
11028 [INSTR_REGRD_HRI] = instr_reg_export,
11029 [INSTR_REGRD_MRI] = instr_reg_export,
11031 [INSTR_REGWR_RHH] = instr_reg_export,
11032 [INSTR_REGWR_RHM] = instr_reg_export,
11033 [INSTR_REGWR_RMH] = instr_reg_export,
11034 [INSTR_REGWR_RMM] = instr_reg_export,
11035 [INSTR_REGWR_RHI] = instr_reg_export,
11036 [INSTR_REGWR_RMI] = instr_reg_export,
11037 [INSTR_REGWR_RIH] = instr_reg_export,
11038 [INSTR_REGWR_RIM] = instr_reg_export,
11039 [INSTR_REGWR_RII] = instr_reg_export,
11041 [INSTR_REGADD_RHH] = instr_reg_export,
11042 [INSTR_REGADD_RHM] = instr_reg_export,
11043 [INSTR_REGADD_RMH] = instr_reg_export,
11044 [INSTR_REGADD_RMM] = instr_reg_export,
11045 [INSTR_REGADD_RHI] = instr_reg_export,
11046 [INSTR_REGADD_RMI] = instr_reg_export,
11047 [INSTR_REGADD_RIH] = instr_reg_export,
11048 [INSTR_REGADD_RIM] = instr_reg_export,
11049 [INSTR_REGADD_RII] = instr_reg_export,
11051 [INSTR_METPREFETCH_H] = instr_meter_export,
11052 [INSTR_METPREFETCH_M] = instr_meter_export,
11053 [INSTR_METPREFETCH_I] = instr_meter_export,
11055 [INSTR_METER_HHM] = instr_meter_export,
11056 [INSTR_METER_HHI] = instr_meter_export,
11057 [INSTR_METER_HMM] = instr_meter_export,
11058 [INSTR_METER_HMI] = instr_meter_export,
11059 [INSTR_METER_MHM] = instr_meter_export,
11060 [INSTR_METER_MHI] = instr_meter_export,
11061 [INSTR_METER_MMM] = instr_meter_export,
11062 [INSTR_METER_MMI] = instr_meter_export,
11063 [INSTR_METER_IHM] = instr_meter_export,
11064 [INSTR_METER_IHI] = instr_meter_export,
11065 [INSTR_METER_IMM] = instr_meter_export,
11066 [INSTR_METER_IMI] = instr_meter_export,
11068 [INSTR_TABLE] = instr_table_export,
11069 [INSTR_TABLE_AF] = instr_table_export,
11070 [INSTR_SELECTOR] = instr_table_export,
11071 [INSTR_LEARNER] = instr_table_export,
11072 [INSTR_LEARNER_AF] = instr_table_export,
11074 [INSTR_LEARNER_LEARN] = instr_learn_export,
11075 [INSTR_LEARNER_FORGET] = instr_forget_export,
11077 [INSTR_EXTERN_OBJ] = instr_extern_export,
11078 [INSTR_EXTERN_FUNC] = instr_extern_export,
11080 [INSTR_JMP] = instr_jmp_export,
11081 [INSTR_JMP_VALID] = instr_jmp_export,
11082 [INSTR_JMP_INVALID] = instr_jmp_export,
11083 [INSTR_JMP_HIT] = instr_jmp_export,
11084 [INSTR_JMP_MISS] = instr_jmp_export,
11085 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11086 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11088 [INSTR_JMP_EQ] = instr_jmp_export,
11089 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11090 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11091 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11092 [INSTR_JMP_EQ_I] = instr_jmp_export,
11094 [INSTR_JMP_NEQ] = instr_jmp_export,
11095 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11096 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11097 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11098 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11100 [INSTR_JMP_LT] = instr_jmp_export,
11101 [INSTR_JMP_LT_MH] = instr_jmp_export,
11102 [INSTR_JMP_LT_HM] = instr_jmp_export,
11103 [INSTR_JMP_LT_HH] = instr_jmp_export,
11104 [INSTR_JMP_LT_MI] = instr_jmp_export,
11105 [INSTR_JMP_LT_HI] = instr_jmp_export,
11107 [INSTR_JMP_GT] = instr_jmp_export,
11108 [INSTR_JMP_GT_MH] = instr_jmp_export,
11109 [INSTR_JMP_GT_HM] = instr_jmp_export,
11110 [INSTR_JMP_GT_HH] = instr_jmp_export,
11111 [INSTR_JMP_GT_MI] = instr_jmp_export,
11112 [INSTR_JMP_GT_HI] = instr_jmp_export,
11114 [INSTR_RETURN] = instr_return_export,
11118 action_data_codegen(struct action *a, FILE *f)
11123 "static const struct instruction action_%s_instructions[] = {\n",
11126 for (i = 0; i < a->n_instructions; i++) {
11127 struct instruction *instr = &a->instructions[i];
11128 instruction_export_t func = export_table[instr->type];
11133 fprintf(f, "};\n");
11136 static const char *
11137 instr_type_to_func(struct instruction *instr)
11139 switch (instr->type) {
11140 case INSTR_RX: return NULL;
11142 case INSTR_TX: return "__instr_tx_exec";
11143 case INSTR_TX_I: return "__instr_tx_i_exec";
11144 case INSTR_DROP: return "__instr_drop_exec";
11146 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11147 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11148 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11149 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11150 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11151 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11152 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11153 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11155 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11157 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11159 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11160 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11161 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11162 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11163 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11164 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11165 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11166 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11167 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11169 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11170 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11172 case INSTR_MOV: return "__instr_mov_exec";
11173 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11174 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11175 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11176 case INSTR_MOV_I: return "__instr_mov_i_exec";
11178 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11179 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11180 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11181 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11182 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11183 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11184 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11185 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11187 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11188 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11189 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11190 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11191 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11192 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11194 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11195 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11196 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11197 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11198 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11199 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11201 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11202 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11203 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11204 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11206 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11207 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11208 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11209 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11210 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11212 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11213 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11214 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11215 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11216 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11218 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11219 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11220 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11221 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11222 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11224 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11225 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11226 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11227 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11228 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11229 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11231 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11232 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11233 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11234 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11235 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11236 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11238 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11239 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11240 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11242 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11243 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11244 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11245 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11246 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11247 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11249 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11250 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11251 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11252 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11253 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11254 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11255 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11256 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11257 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11259 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11260 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11261 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11262 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11263 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11264 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11265 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11266 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11267 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11269 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11270 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11271 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11273 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11274 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11275 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11276 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11277 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11278 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11279 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11280 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11281 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11282 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11283 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11284 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11286 case INSTR_TABLE: return NULL;
11287 case INSTR_TABLE_AF: return NULL;
11288 case INSTR_SELECTOR: return NULL;
11289 case INSTR_LEARNER: return NULL;
11290 case INSTR_LEARNER_AF: return NULL;
11292 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11293 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11295 case INSTR_EXTERN_OBJ: return NULL;
11296 case INSTR_EXTERN_FUNC: return NULL;
11298 case INSTR_JMP: return NULL;
11299 case INSTR_JMP_VALID: return NULL;
11300 case INSTR_JMP_INVALID: return NULL;
11301 case INSTR_JMP_HIT: return NULL;
11302 case INSTR_JMP_MISS: return NULL;
11303 case INSTR_JMP_ACTION_HIT: return NULL;
11304 case INSTR_JMP_ACTION_MISS: return NULL;
11305 case INSTR_JMP_EQ: return NULL;
11306 case INSTR_JMP_EQ_MH: return NULL;
11307 case INSTR_JMP_EQ_HM: return NULL;
11308 case INSTR_JMP_EQ_HH: return NULL;
11309 case INSTR_JMP_EQ_I: return NULL;
11310 case INSTR_JMP_NEQ: return NULL;
11311 case INSTR_JMP_NEQ_MH: return NULL;
11312 case INSTR_JMP_NEQ_HM: return NULL;
11313 case INSTR_JMP_NEQ_HH: return NULL;
11314 case INSTR_JMP_NEQ_I: return NULL;
11315 case INSTR_JMP_LT: return NULL;
11316 case INSTR_JMP_LT_MH: return NULL;
11317 case INSTR_JMP_LT_HM: return NULL;
11318 case INSTR_JMP_LT_HH: return NULL;
11319 case INSTR_JMP_LT_MI: return NULL;
11320 case INSTR_JMP_LT_HI: return NULL;
11321 case INSTR_JMP_GT: return NULL;
11322 case INSTR_JMP_GT_MH: return NULL;
11323 case INSTR_JMP_GT_HM: return NULL;
11324 case INSTR_JMP_GT_HH: return NULL;
11325 case INSTR_JMP_GT_MI: return NULL;
11326 case INSTR_JMP_GT_HI: return NULL;
11328 case INSTR_RETURN: return NULL;
11330 default: return NULL;
11335 action_instr_does_tx_codegen(struct action *a,
11336 uint32_t instr_pos,
11337 struct instruction *instr,
11341 "%s(p, t, &action_%s_instructions[%u]);\n"
11342 "\tthread_ip_reset(p, t);\n"
11343 "\tinstr_rx_exec(p);\n"
11345 instr_type_to_func(instr),
11351 action_instr_extern_obj_codegen(struct action *a,
11352 uint32_t instr_pos,
11356 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11362 action_instr_extern_func_codegen(struct action *a,
11363 uint32_t instr_pos,
11367 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11373 action_instr_jmp_codegen(struct action *a,
11374 uint32_t instr_pos,
11375 struct instruction *instr,
11376 struct instruction_data *data,
11379 switch (instr->type) {
11386 case INSTR_JMP_VALID:
11388 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11395 case INSTR_JMP_INVALID:
11397 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11404 case INSTR_JMP_HIT:
11411 case INSTR_JMP_MISS:
11418 case INSTR_JMP_ACTION_HIT:
11420 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11427 case INSTR_JMP_ACTION_MISS:
11429 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11438 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11439 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11448 case INSTR_JMP_EQ_MH:
11450 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11451 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11460 case INSTR_JMP_EQ_HM:
11462 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11463 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11472 case INSTR_JMP_EQ_HH:
11474 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11475 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11484 case INSTR_JMP_EQ_I:
11486 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11487 "action_%s_instructions[%u].jmp.b_val)\n"
11496 case INSTR_JMP_NEQ:
11498 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11499 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11508 case INSTR_JMP_NEQ_MH:
11510 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11511 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11520 case INSTR_JMP_NEQ_HM:
11522 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11523 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11532 case INSTR_JMP_NEQ_HH:
11534 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11535 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11544 case INSTR_JMP_NEQ_I:
11546 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11547 "action_%s_instructions[%u].jmp.b_val)\n"
11558 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11559 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11568 case INSTR_JMP_LT_MH:
11570 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11571 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11580 case INSTR_JMP_LT_HM:
11582 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11583 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11592 case INSTR_JMP_LT_HH:
11594 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11595 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11604 case INSTR_JMP_LT_MI:
11606 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11607 "action_%s_instructions[%u].jmp.b_val)\n"
11616 case INSTR_JMP_LT_HI:
11618 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11619 "action_%s_instructions[%u].jmp.b_val)\n"
11630 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11631 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11640 case INSTR_JMP_GT_MH:
11642 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11643 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11652 case INSTR_JMP_GT_HM:
11654 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11655 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11664 case INSTR_JMP_GT_HH:
11666 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11667 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11676 case INSTR_JMP_GT_MI:
11678 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11679 "action_%s_instructions[%u].jmp.b_val)\n"
11688 case INSTR_JMP_GT_HI:
11690 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11691 "action_%s_instructions[%u].jmp.b_val)\n"
11706 action_instr_return_codegen(FILE *f)
11713 action_instr_codegen(struct action *a, FILE *f)
11719 "action_%s_run(struct rte_swx_pipeline *p)\n"
11721 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11725 for (i = 0; i < a->n_instructions; i++) {
11726 struct instruction *instr = &a->instructions[i];
11727 struct instruction_data *data = &a->instruction_data[i];
11729 /* Label, if present. */
11730 if (data->label[0])
11731 fprintf(f, "\n%s : ", data->label);
11733 fprintf(f, "\n\t");
11735 /* TX instruction type. */
11736 if (instruction_does_tx(instr)) {
11737 action_instr_does_tx_codegen(a, i, instr, f);
11741 /* Extern object/function instruction type. */
11742 if (instr->type == INSTR_EXTERN_OBJ) {
11743 action_instr_extern_obj_codegen(a, i, f);
11747 if (instr->type == INSTR_EXTERN_FUNC) {
11748 action_instr_extern_func_codegen(a, i, f);
11752 /* Jump instruction type. */
11753 if (instruction_is_jmp(instr)) {
11754 action_instr_jmp_codegen(a, i, instr, data, f);
11758 /* Return instruction type. */
11759 if (instr->type == INSTR_RETURN) {
11760 action_instr_return_codegen(f);
11764 /* Any other instruction type. */
11766 "%s(p, t, &action_%s_instructions[%u]);\n",
11767 instr_type_to_func(instr),
11772 fprintf(f, "}\n\n");
11775 struct instruction_group {
11776 TAILQ_ENTRY(instruction_group) node;
11780 uint32_t first_instr_id;
11782 uint32_t last_instr_id;
11787 TAILQ_HEAD(instruction_group_list, instruction_group);
11789 static struct instruction_group *
11790 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11792 struct instruction_group *g;
11794 TAILQ_FOREACH(g, igl, node)
11795 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11802 instruction_group_list_free(struct instruction_group_list *igl)
11808 struct instruction_group *g;
11810 g = TAILQ_FIRST(igl);
11814 TAILQ_REMOVE(igl, g, node);
11821 static struct instruction_group_list *
11822 instruction_group_list_create(struct rte_swx_pipeline *p)
11824 struct instruction_group_list *igl = NULL;
11825 struct instruction_group *g = NULL;
11826 uint32_t n_groups = 0, i;
11828 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11832 igl = calloc(1, sizeof(struct instruction_group_list));
11838 /* Allocate the first group. */
11839 g = calloc(1, sizeof(struct instruction_group));
11843 /* Iteration 1: Separate the instructions into groups based on the thread yield
11844 * instructions. Do not worry about the jump instructions at this point.
11846 for (i = 0; i < p->n_instructions; i++) {
11847 struct instruction *instr = &p->instructions[i];
11849 /* Check for thread yield instructions. */
11850 if (!instruction_does_thread_yield(instr))
11853 /* If the current group contains at least one instruction, then finalize it (with
11854 * the previous instruction), add it to the list and allocate a new group (that
11855 * starts with the current instruction).
11857 if (i - g->first_instr_id) {
11858 /* Finalize the group. */
11859 g->last_instr_id = i - 1;
11861 /* Add the group to the list. Advance the number of groups. */
11862 TAILQ_INSERT_TAIL(igl, g, node);
11865 /* Allocate a new group. */
11866 g = calloc(1, sizeof(struct instruction_group));
11870 /* Initialize the new group. */
11871 g->group_id = n_groups;
11872 g->first_instr_id = i;
11875 /* Finalize the current group (with the current instruction, therefore this group
11876 * contains just the current thread yield instruction), add it to the list and
11877 * allocate a new group (that starts with the next instruction).
11880 /* Finalize the group. */
11881 g->last_instr_id = i;
11883 /* Add the group to the list. Advance the number of groups. */
11884 TAILQ_INSERT_TAIL(igl, g, node);
11887 /* Allocate a new group. */
11888 g = calloc(1, sizeof(struct instruction_group));
11892 /* Initialize the new group. */
11893 g->group_id = n_groups;
11894 g->first_instr_id = i + 1;
11897 /* Handle the last group. */
11898 if (i - g->first_instr_id) {
11899 /* Finalize the group. */
11900 g->last_instr_id = i - 1;
11902 /* Add the group to the list. Advance the number of groups. */
11903 TAILQ_INSERT_TAIL(igl, g, node);
11910 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11911 * the destination of a jump instruction located in a different group ("far jump"), then the
11912 * current group has to be split, so that the instruction representing the far jump
11913 * destination is at the start of its group.
11916 int is_modified = 0;
11918 for (i = 0; i < p->n_instructions; i++) {
11919 struct instruction_data *data = &p->instruction_data[i];
11920 struct instruction_group *g;
11923 /* Continue when the current instruction is not a jump destination. */
11924 if (!data->n_users)
11927 g = instruction_group_list_group_find(igl, i);
11931 /* Find out all the jump instructions with this destination. */
11932 for (j = 0; j < p->n_instructions; j++) {
11933 struct instruction *jmp_instr = &p->instructions[j];
11934 struct instruction_data *jmp_data = &p->instruction_data[j];
11935 struct instruction_group *jmp_g, *new_g;
11937 /* Continue when not a jump instruction. Even when jump instruction,
11938 * continue when the jump destination is not this instruction.
11940 if (!instruction_is_jmp(jmp_instr) ||
11941 strcmp(jmp_data->jmp_label, data->label))
11944 jmp_g = instruction_group_list_group_find(igl, j);
11948 /* Continue when both the jump instruction and the jump destination
11949 * instruction are in the same group. Even when in different groups,
11950 * still continue if the jump destination instruction is already the
11951 * first instruction of its group.
11953 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11956 /* Split the group of the current jump destination instruction to
11957 * make this instruction the first instruction of a new group.
11959 new_g = calloc(1, sizeof(struct instruction_group));
11963 new_g->group_id = n_groups;
11964 new_g->first_instr_id = i;
11965 new_g->last_instr_id = g->last_instr_id;
11967 g->last_instr_id = i - 1;
11969 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11973 /* The decision to split this group (to make the current instruction
11974 * the first instruction of a new group) is already taken and fully
11975 * implemented, so no need to search for more reasons to do it.
11981 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11982 * previously considered local (i.e. the jump destination is in the same group as
11983 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11984 * different group than the jump instruction). Wost case scenario: each instruction
11985 * that is a jump destination ends up as the first instruction of its group.
11991 /* Re-assign the group IDs to be in incremental order. */
11993 TAILQ_FOREACH(g, igl, node) {
12002 instruction_group_list_free(igl);
12010 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
12011 uint32_t instr_pos,
12012 struct instruction *instr,
12016 "%s(p, t, &pipeline_instructions[%u]);\n"
12017 "\tthread_ip_reset(p, t);\n"
12018 "\tinstr_rx_exec(p);\n"
12020 instr_type_to_func(instr),
12025 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
12026 struct instruction_group_list *igl,
12027 uint32_t jmp_instr_id,
12028 struct instruction *jmp_instr,
12029 struct instruction_data *jmp_data,
12032 struct instruction_group *jmp_g, *g;
12033 struct instruction_data *data;
12036 switch (jmp_instr->type) {
12040 case INSTR_JMP_VALID:
12042 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12046 case INSTR_JMP_INVALID:
12048 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12052 case INSTR_JMP_HIT:
12057 case INSTR_JMP_MISS:
12062 case INSTR_JMP_ACTION_HIT:
12064 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12068 case INSTR_JMP_ACTION_MISS:
12070 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12076 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12077 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12082 case INSTR_JMP_EQ_MH:
12084 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12085 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12090 case INSTR_JMP_EQ_HM:
12092 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12093 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12098 case INSTR_JMP_EQ_HH:
12100 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12101 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12106 case INSTR_JMP_EQ_I:
12108 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12109 "pipeline_instructions[%u].jmp.b_val)",
12114 case INSTR_JMP_NEQ:
12116 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12117 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12122 case INSTR_JMP_NEQ_MH:
12124 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12125 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12130 case INSTR_JMP_NEQ_HM:
12132 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12133 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12138 case INSTR_JMP_NEQ_HH:
12140 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12141 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12146 case INSTR_JMP_NEQ_I:
12148 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12149 "pipeline_instructions[%u].jmp.b_val)",
12156 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12157 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12162 case INSTR_JMP_LT_MH:
12164 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12165 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12170 case INSTR_JMP_LT_HM:
12172 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12173 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12178 case INSTR_JMP_LT_HH:
12180 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12181 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12186 case INSTR_JMP_LT_MI:
12188 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12189 "pipeline_instructions[%u].jmp.b_val)",
12194 case INSTR_JMP_LT_HI:
12196 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12197 "pipeline_instructions[%u].jmp.b_val)",
12204 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12205 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12210 case INSTR_JMP_GT_MH:
12212 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12213 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12218 case INSTR_JMP_GT_HM:
12220 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12221 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12226 case INSTR_JMP_GT_HH:
12228 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12229 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12234 case INSTR_JMP_GT_MI:
12236 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12237 "pipeline_instructions[%u].jmp.b_val)",
12242 case INSTR_JMP_GT_HI:
12244 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12245 "pipeline_instructions[%u].jmp.b_val)",
12254 /* Find the instruction group of the jump instruction. */
12255 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12259 /* Find the instruction group of the jump destination instruction. */
12260 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12264 instr_id = data - p->instruction_data;
12266 g = instruction_group_list_group_find(igl, instr_id);
12270 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12271 * instruction group).
12273 if (g->group_id == jmp_g->group_id)
12275 "\n\t\tgoto %s;\n",
12276 jmp_data->jmp_label);
12280 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12289 instruction_group_list_codegen(struct instruction_group_list *igl,
12290 struct rte_swx_pipeline *p,
12293 struct instruction_group *g;
12295 int is_required = 0;
12297 /* Check if code generation is required. */
12298 TAILQ_FOREACH(g, igl, node)
12299 if (g->first_instr_id < g->last_instr_id)
12305 /* Generate the code for the pipeline instruction array. */
12307 "static const struct instruction pipeline_instructions[] = {\n");
12309 for (i = 0; i < p->n_instructions; i++) {
12310 struct instruction *instr = &p->instructions[i];
12311 instruction_export_t func = export_table[instr->type];
12316 fprintf(f, "};\n\n");
12318 /* Generate the code for the pipeline functions: one function for each instruction group
12319 * that contains more than one instruction.
12321 TAILQ_FOREACH(g, igl, node) {
12322 struct instruction *last_instr;
12325 /* Skip if group contains a single instruction. */
12326 if (g->last_instr_id == g->first_instr_id)
12329 /* Generate new pipeline function. */
12332 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12334 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12338 /* Generate the code for each pipeline instruction. */
12339 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12340 struct instruction *instr = &p->instructions[j];
12341 struct instruction_data *data = &p->instruction_data[j];
12343 /* Label, if present. */
12344 if (data->label[0])
12345 fprintf(f, "\n%s : ", data->label);
12347 fprintf(f, "\n\t");
12349 /* TX instruction type. */
12350 if (instruction_does_tx(instr)) {
12351 pipeline_instr_does_tx_codegen(p, j, instr, f);
12355 /* Jump instruction type. */
12356 if (instruction_is_jmp(instr)) {
12357 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12361 /* Any other instruction type. */
12363 "%s(p, t, &pipeline_instructions[%u]);\n",
12364 instr_type_to_func(instr),
12368 /* Finalize the generated pipeline function. For some instructions such as TX,
12369 * emit-many-and-TX and unconditional jump, the next instruction has been already
12370 * decided unconditionally and the instruction pointer of the current thread set
12371 * accordingly; for all the other instructions, the instruction pointer must be
12374 last_instr = &p->instructions[g->last_instr_id];
12376 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12378 "thread_ip_inc(p);\n");
12387 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12389 struct instruction_group *g;
12390 uint32_t n_custom_instr = 0;
12392 /* Groups with a single instruction: no function is generated for this group, the group
12393 * keeps its current instruction. Groups with more than two instructions: one function and
12394 * the associated custom instruction get generated for each such group.
12396 TAILQ_FOREACH(g, igl, node) {
12397 if (g->first_instr_id == g->last_instr_id)
12403 return n_custom_instr;
12407 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12412 /* Create the .c file. */
12413 f = fopen("/tmp/pipeline.c", "w");
12417 /* Include the .h file. */
12418 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12420 /* Add the code for each action. */
12421 TAILQ_FOREACH(a, &p->actions, node) {
12422 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12424 action_data_codegen(a, f);
12428 action_instr_codegen(a, f);
12433 /* Add the pipeline code. */
12434 instruction_group_list_codegen(igl, p, f);
12436 /* Close the .c file. */
12442 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12443 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12447 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12450 struct instruction_group *g;
12451 char *dir_in, *buffer = NULL;
12452 const char *dir_out;
12455 /* Get the environment variables. */
12456 dir_in = getenv("RTE_INSTALL_DIR");
12464 /* Memory allocation for the command buffer. */
12465 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12472 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12473 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12474 "-I %s/lib/pipeline "
12475 "-I %s/lib/eal/include "
12476 "-I %s/lib/eal/x86/include "
12477 "-I %s/lib/eal/include/generic "
12481 "-I %s/lib/pipeline "
12484 "-I %s/lib/eal/linux/include "
12485 ">%s/pipeline.log 2>&1 "
12487 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12488 ">>%s/pipeline.log 2>&1",
12507 /* Build the shared object library. */
12508 status = system(buffer);
12512 /* Open library. */
12514 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12515 "%s/libpipeline.so",
12518 p->lib = dlopen(buffer, RTLD_LAZY);
12524 /* Get the action function symbols. */
12525 TAILQ_FOREACH(a, &p->actions, node) {
12526 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12528 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12529 if (!p->action_funcs[a->id]) {
12535 /* Get the pipeline function symbols. */
12536 TAILQ_FOREACH(g, igl, node) {
12537 if (g->first_instr_id == g->last_instr_id)
12540 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12542 g->func = dlsym(p->lib, buffer);
12550 if (status && p->lib) {
12561 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12562 struct instruction_group_list *igl)
12564 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12566 /* Check that enough space is available within the pipeline instruction table to store all
12567 * the custom instructions.
12569 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12576 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12578 struct instruction_group *g;
12581 /* Pipeline table instructions. */
12582 for (i = 0; i < p->n_instructions; i++) {
12583 struct instruction *instr = &p->instructions[i];
12585 if (instr->type == INSTR_TABLE)
12586 instr->type = INSTR_TABLE_AF;
12588 if (instr->type == INSTR_LEARNER)
12589 instr->type = INSTR_LEARNER_AF;
12592 /* Pipeline custom instructions. */
12594 TAILQ_FOREACH(g, igl, node) {
12595 struct instruction *instr = &p->instructions[g->first_instr_id];
12598 if (g->first_instr_id == g->last_instr_id)
12601 /* Install a new custom instruction. */
12602 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12604 /* First instruction of the group: change its type to the new custom instruction. */
12605 instr->type = INSTR_CUSTOM_0 + i;
12607 /* All the subsequent instructions of the group: invalidate. */
12608 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12609 struct instruction_data *data = &p->instruction_data[j];
12617 /* Remove the invalidated instructions. */
12618 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12620 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12621 * instructions that are the only instruction within their group, so they were left
12624 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12628 pipeline_compile(struct rte_swx_pipeline *p)
12630 struct instruction_group_list *igl = NULL;
12633 igl = instruction_group_list_create(p);
12639 /* Code generation. */
12640 status = pipeline_codegen(p, igl);
12644 /* Build and load the shared object library. */
12645 status = pipeline_libload(p, igl);
12649 /* Adjust instructions. */
12650 status = pipeline_adjust_check(p, igl);
12654 pipeline_adjust(p, igl);
12657 instruction_group_list_free(igl);