1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <rte_swx_port_ethdev.h>
11 #include <rte_swx_port_fd.h>
12 #include <rte_swx_port_ring.h>
13 #include "rte_swx_port_source_sink.h"
15 #include "rte_swx_pipeline_internal.h"
17 #define CHECK(condition, err_code) \
23 #define CHECK_NAME(name, err_code) \
26 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
29 #define CHECK_INSTRUCTION(instr, err_code) \
32 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
33 RTE_SWX_INSTRUCTION_SIZE), \
39 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
41 #include <rte_malloc.h>
44 env_malloc(size_t size, size_t alignment, int numa_node)
46 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
50 env_free(void *start, size_t size __rte_unused)
60 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
64 if (numa_available() == -1)
67 start = numa_alloc_onnode(size, numa_node);
71 memset(start, 0, size);
76 env_free(void *start, size_t size)
78 if (numa_available() == -1)
81 numa_free(start, size);
89 static struct struct_type *
90 struct_type_find(struct rte_swx_pipeline *p, const char *name)
92 struct struct_type *elem;
94 TAILQ_FOREACH(elem, &p->struct_types, node)
95 if (strcmp(elem->name, name) == 0)
101 static struct field *
102 struct_type_field_find(struct struct_type *st, const char *name)
106 for (i = 0; i < st->n_fields; i++) {
107 struct field *f = &st->fields[i];
109 if (strcmp(f->name, name) == 0)
117 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
119 struct rte_swx_field_params *fields,
121 int last_field_has_variable_size)
123 struct struct_type *st;
127 CHECK_NAME(name, EINVAL);
128 CHECK(fields, EINVAL);
129 CHECK(n_fields, EINVAL);
131 for (i = 0; i < n_fields; i++) {
132 struct rte_swx_field_params *f = &fields[i];
133 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
136 CHECK_NAME(f->name, EINVAL);
137 CHECK(f->n_bits, EINVAL);
138 CHECK((f->n_bits <= 64) || var_size, EINVAL);
139 CHECK((f->n_bits & 7) == 0, EINVAL);
141 for (j = 0; j < i; j++) {
142 struct rte_swx_field_params *f_prev = &fields[j];
144 CHECK(strcmp(f->name, f_prev->name), EINVAL);
148 CHECK(!struct_type_find(p, name), EEXIST);
150 /* Node allocation. */
151 st = calloc(1, sizeof(struct struct_type));
154 st->fields = calloc(n_fields, sizeof(struct field));
160 /* Node initialization. */
161 strcpy(st->name, name);
162 for (i = 0; i < n_fields; i++) {
163 struct field *dst = &st->fields[i];
164 struct rte_swx_field_params *src = &fields[i];
165 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
167 strcpy(dst->name, src->name);
168 dst->n_bits = src->n_bits;
169 dst->offset = st->n_bits;
170 dst->var_size = var_size;
172 st->n_bits += src->n_bits;
173 st->n_bits_min += var_size ? 0 : src->n_bits;
175 st->n_fields = n_fields;
176 st->var_size = last_field_has_variable_size;
178 /* Node add to tailq. */
179 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
185 struct_build(struct rte_swx_pipeline *p)
189 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
190 struct thread *t = &p->threads[i];
192 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
193 CHECK(t->structs, ENOMEM);
200 struct_build_free(struct rte_swx_pipeline *p)
204 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
205 struct thread *t = &p->threads[i];
213 struct_free(struct rte_swx_pipeline *p)
215 struct_build_free(p);
219 struct struct_type *elem;
221 elem = TAILQ_FIRST(&p->struct_types);
225 TAILQ_REMOVE(&p->struct_types, elem, node);
234 static struct port_in_type *
235 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
237 struct port_in_type *elem;
242 TAILQ_FOREACH(elem, &p->port_in_types, node)
243 if (strcmp(elem->name, name) == 0)
250 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
252 struct rte_swx_port_in_ops *ops)
254 struct port_in_type *elem;
257 CHECK_NAME(name, EINVAL);
259 CHECK(ops->create, EINVAL);
260 CHECK(ops->free, EINVAL);
261 CHECK(ops->pkt_rx, EINVAL);
262 CHECK(ops->stats_read, EINVAL);
264 CHECK(!port_in_type_find(p, name), EEXIST);
266 /* Node allocation. */
267 elem = calloc(1, sizeof(struct port_in_type));
270 /* Node initialization. */
271 strcpy(elem->name, name);
272 memcpy(&elem->ops, ops, sizeof(*ops));
274 /* Node add to tailq. */
275 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
280 static struct port_in *
281 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
283 struct port_in *port;
285 TAILQ_FOREACH(port, &p->ports_in, node)
286 if (port->id == port_id)
293 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
295 const char *port_type_name,
298 struct port_in_type *type = NULL;
299 struct port_in *port = NULL;
304 CHECK(!port_in_find(p, port_id), EINVAL);
306 CHECK_NAME(port_type_name, EINVAL);
307 type = port_in_type_find(p, port_type_name);
310 obj = type->ops.create(args);
313 /* Node allocation. */
314 port = calloc(1, sizeof(struct port_in));
317 /* Node initialization. */
322 /* Node add to tailq. */
323 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
324 if (p->n_ports_in < port_id + 1)
325 p->n_ports_in = port_id + 1;
331 port_in_build(struct rte_swx_pipeline *p)
333 struct port_in *port;
336 CHECK(p->n_ports_in, EINVAL);
337 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
339 for (i = 0; i < p->n_ports_in; i++)
340 CHECK(port_in_find(p, i), EINVAL);
342 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
343 CHECK(p->in, ENOMEM);
345 TAILQ_FOREACH(port, &p->ports_in, node) {
346 struct port_in_runtime *in = &p->in[port->id];
348 in->pkt_rx = port->type->ops.pkt_rx;
356 port_in_build_free(struct rte_swx_pipeline *p)
363 port_in_free(struct rte_swx_pipeline *p)
365 port_in_build_free(p);
369 struct port_in *port;
371 port = TAILQ_FIRST(&p->ports_in);
375 TAILQ_REMOVE(&p->ports_in, port, node);
376 port->type->ops.free(port->obj);
380 /* Input port types. */
382 struct port_in_type *elem;
384 elem = TAILQ_FIRST(&p->port_in_types);
388 TAILQ_REMOVE(&p->port_in_types, elem, node);
396 static struct port_out_type *
397 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
399 struct port_out_type *elem;
404 TAILQ_FOREACH(elem, &p->port_out_types, node)
405 if (!strcmp(elem->name, name))
412 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
414 struct rte_swx_port_out_ops *ops)
416 struct port_out_type *elem;
419 CHECK_NAME(name, EINVAL);
421 CHECK(ops->create, EINVAL);
422 CHECK(ops->free, EINVAL);
423 CHECK(ops->pkt_tx, EINVAL);
424 CHECK(ops->stats_read, EINVAL);
426 CHECK(!port_out_type_find(p, name), EEXIST);
428 /* Node allocation. */
429 elem = calloc(1, sizeof(struct port_out_type));
432 /* Node initialization. */
433 strcpy(elem->name, name);
434 memcpy(&elem->ops, ops, sizeof(*ops));
436 /* Node add to tailq. */
437 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
442 static struct port_out *
443 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
445 struct port_out *port;
447 TAILQ_FOREACH(port, &p->ports_out, node)
448 if (port->id == port_id)
455 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
457 const char *port_type_name,
460 struct port_out_type *type = NULL;
461 struct port_out *port = NULL;
466 CHECK(!port_out_find(p, port_id), EINVAL);
468 CHECK_NAME(port_type_name, EINVAL);
469 type = port_out_type_find(p, port_type_name);
472 obj = type->ops.create(args);
475 /* Node allocation. */
476 port = calloc(1, sizeof(struct port_out));
479 /* Node initialization. */
484 /* Node add to tailq. */
485 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
486 if (p->n_ports_out < port_id + 1)
487 p->n_ports_out = port_id + 1;
493 port_out_build(struct rte_swx_pipeline *p)
495 struct port_out *port;
498 CHECK(p->n_ports_out, EINVAL);
500 for (i = 0; i < p->n_ports_out; i++)
501 CHECK(port_out_find(p, i), EINVAL);
503 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
504 CHECK(p->out, ENOMEM);
506 TAILQ_FOREACH(port, &p->ports_out, node) {
507 struct port_out_runtime *out = &p->out[port->id];
509 out->pkt_tx = port->type->ops.pkt_tx;
510 out->flush = port->type->ops.flush;
511 out->obj = port->obj;
518 port_out_build_free(struct rte_swx_pipeline *p)
525 port_out_free(struct rte_swx_pipeline *p)
527 port_out_build_free(p);
531 struct port_out *port;
533 port = TAILQ_FIRST(&p->ports_out);
537 TAILQ_REMOVE(&p->ports_out, port, node);
538 port->type->ops.free(port->obj);
542 /* Output port types. */
544 struct port_out_type *elem;
546 elem = TAILQ_FIRST(&p->port_out_types);
550 TAILQ_REMOVE(&p->port_out_types, elem, node);
558 static struct extern_type *
559 extern_type_find(struct rte_swx_pipeline *p, const char *name)
561 struct extern_type *elem;
563 TAILQ_FOREACH(elem, &p->extern_types, node)
564 if (strcmp(elem->name, name) == 0)
570 static struct extern_type_member_func *
571 extern_type_member_func_find(struct extern_type *type, const char *name)
573 struct extern_type_member_func *elem;
575 TAILQ_FOREACH(elem, &type->funcs, node)
576 if (strcmp(elem->name, name) == 0)
582 static struct extern_obj *
583 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
585 struct extern_obj *elem;
587 TAILQ_FOREACH(elem, &p->extern_objs, node)
588 if (strcmp(elem->name, name) == 0)
594 static struct extern_type_member_func *
595 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
597 struct extern_obj **obj)
599 struct extern_obj *object;
600 struct extern_type_member_func *func;
601 char *object_name, *func_name;
603 if (name[0] != 'e' || name[1] != '.')
606 object_name = strdup(&name[2]);
610 func_name = strchr(object_name, '.');
619 object = extern_obj_find(p, object_name);
625 func = extern_type_member_func_find(object->type, func_name);
638 static struct field *
639 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
641 struct extern_obj **object)
643 struct extern_obj *obj;
645 char *obj_name, *field_name;
647 if ((name[0] != 'e') || (name[1] != '.'))
650 obj_name = strdup(&name[2]);
654 field_name = strchr(obj_name, '.');
663 obj = extern_obj_find(p, obj_name);
669 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
683 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
685 const char *mailbox_struct_type_name,
686 rte_swx_extern_type_constructor_t constructor,
687 rte_swx_extern_type_destructor_t destructor)
689 struct extern_type *elem;
690 struct struct_type *mailbox_struct_type;
694 CHECK_NAME(name, EINVAL);
695 CHECK(!extern_type_find(p, name), EEXIST);
697 CHECK_NAME(mailbox_struct_type_name, EINVAL);
698 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
699 CHECK(mailbox_struct_type, EINVAL);
700 CHECK(!mailbox_struct_type->var_size, EINVAL);
702 CHECK(constructor, EINVAL);
703 CHECK(destructor, EINVAL);
705 /* Node allocation. */
706 elem = calloc(1, sizeof(struct extern_type));
709 /* Node initialization. */
710 strcpy(elem->name, name);
711 elem->mailbox_struct_type = mailbox_struct_type;
712 elem->constructor = constructor;
713 elem->destructor = destructor;
714 TAILQ_INIT(&elem->funcs);
716 /* Node add to tailq. */
717 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
723 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
724 const char *extern_type_name,
726 rte_swx_extern_type_member_func_t member_func)
728 struct extern_type *type;
729 struct extern_type_member_func *type_member;
733 CHECK_NAME(extern_type_name, EINVAL);
734 type = extern_type_find(p, extern_type_name);
736 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
738 CHECK_NAME(name, EINVAL);
739 CHECK(!extern_type_member_func_find(type, name), EEXIST);
741 CHECK(member_func, EINVAL);
743 /* Node allocation. */
744 type_member = calloc(1, sizeof(struct extern_type_member_func));
745 CHECK(type_member, ENOMEM);
747 /* Node initialization. */
748 strcpy(type_member->name, name);
749 type_member->func = member_func;
750 type_member->id = type->n_funcs;
752 /* Node add to tailq. */
753 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
760 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
761 const char *extern_type_name,
765 struct extern_type *type;
766 struct extern_obj *obj;
771 CHECK_NAME(extern_type_name, EINVAL);
772 type = extern_type_find(p, extern_type_name);
775 CHECK_NAME(name, EINVAL);
776 CHECK(!extern_obj_find(p, name), EEXIST);
778 /* Node allocation. */
779 obj = calloc(1, sizeof(struct extern_obj));
782 /* Object construction. */
783 obj_handle = type->constructor(args);
789 /* Node initialization. */
790 strcpy(obj->name, name);
792 obj->obj = obj_handle;
793 obj->struct_id = p->n_structs;
794 obj->id = p->n_extern_objs;
796 /* Node add to tailq. */
797 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
805 extern_obj_build(struct rte_swx_pipeline *p)
809 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
810 struct thread *t = &p->threads[i];
811 struct extern_obj *obj;
813 t->extern_objs = calloc(p->n_extern_objs,
814 sizeof(struct extern_obj_runtime));
815 CHECK(t->extern_objs, ENOMEM);
817 TAILQ_FOREACH(obj, &p->extern_objs, node) {
818 struct extern_obj_runtime *r =
819 &t->extern_objs[obj->id];
820 struct extern_type_member_func *func;
821 uint32_t mailbox_size =
822 obj->type->mailbox_struct_type->n_bits / 8;
826 r->mailbox = calloc(1, mailbox_size);
827 CHECK(r->mailbox, ENOMEM);
829 TAILQ_FOREACH(func, &obj->type->funcs, node)
830 r->funcs[func->id] = func->func;
832 t->structs[obj->struct_id] = r->mailbox;
840 extern_obj_build_free(struct rte_swx_pipeline *p)
844 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
845 struct thread *t = &p->threads[i];
851 for (j = 0; j < p->n_extern_objs; j++) {
852 struct extern_obj_runtime *r = &t->extern_objs[j];
857 free(t->extern_objs);
858 t->extern_objs = NULL;
863 extern_obj_free(struct rte_swx_pipeline *p)
865 extern_obj_build_free(p);
867 /* Extern objects. */
869 struct extern_obj *elem;
871 elem = TAILQ_FIRST(&p->extern_objs);
875 TAILQ_REMOVE(&p->extern_objs, elem, node);
877 elem->type->destructor(elem->obj);
883 struct extern_type *elem;
885 elem = TAILQ_FIRST(&p->extern_types);
889 TAILQ_REMOVE(&p->extern_types, elem, node);
892 struct extern_type_member_func *func;
894 func = TAILQ_FIRST(&elem->funcs);
898 TAILQ_REMOVE(&elem->funcs, func, node);
909 static struct extern_func *
910 extern_func_find(struct rte_swx_pipeline *p, const char *name)
912 struct extern_func *elem;
914 TAILQ_FOREACH(elem, &p->extern_funcs, node)
915 if (strcmp(elem->name, name) == 0)
921 static struct extern_func *
922 extern_func_parse(struct rte_swx_pipeline *p,
925 if (name[0] != 'f' || name[1] != '.')
928 return extern_func_find(p, &name[2]);
931 static struct field *
932 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
934 struct extern_func **function)
936 struct extern_func *func;
938 char *func_name, *field_name;
940 if ((name[0] != 'f') || (name[1] != '.'))
943 func_name = strdup(&name[2]);
947 field_name = strchr(func_name, '.');
956 func = extern_func_find(p, func_name);
962 f = struct_type_field_find(func->mailbox_struct_type, field_name);
976 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
978 const char *mailbox_struct_type_name,
979 rte_swx_extern_func_t func)
981 struct extern_func *f;
982 struct struct_type *mailbox_struct_type;
986 CHECK_NAME(name, EINVAL);
987 CHECK(!extern_func_find(p, name), EEXIST);
989 CHECK_NAME(mailbox_struct_type_name, EINVAL);
990 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
991 CHECK(mailbox_struct_type, EINVAL);
992 CHECK(!mailbox_struct_type->var_size, EINVAL);
996 /* Node allocation. */
997 f = calloc(1, sizeof(struct extern_func));
1000 /* Node initialization. */
1001 strcpy(f->name, name);
1002 f->mailbox_struct_type = mailbox_struct_type;
1004 f->struct_id = p->n_structs;
1005 f->id = p->n_extern_funcs;
1007 /* Node add to tailq. */
1008 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1009 p->n_extern_funcs++;
1016 extern_func_build(struct rte_swx_pipeline *p)
1020 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1021 struct thread *t = &p->threads[i];
1022 struct extern_func *func;
1024 /* Memory allocation. */
1025 t->extern_funcs = calloc(p->n_extern_funcs,
1026 sizeof(struct extern_func_runtime));
1027 CHECK(t->extern_funcs, ENOMEM);
1029 /* Extern function. */
1030 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1031 struct extern_func_runtime *r =
1032 &t->extern_funcs[func->id];
1033 uint32_t mailbox_size =
1034 func->mailbox_struct_type->n_bits / 8;
1036 r->func = func->func;
1038 r->mailbox = calloc(1, mailbox_size);
1039 CHECK(r->mailbox, ENOMEM);
1041 t->structs[func->struct_id] = r->mailbox;
1049 extern_func_build_free(struct rte_swx_pipeline *p)
1053 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1054 struct thread *t = &p->threads[i];
1057 if (!t->extern_funcs)
1060 for (j = 0; j < p->n_extern_funcs; j++) {
1061 struct extern_func_runtime *r = &t->extern_funcs[j];
1066 free(t->extern_funcs);
1067 t->extern_funcs = NULL;
1072 extern_func_free(struct rte_swx_pipeline *p)
1074 extern_func_build_free(p);
1077 struct extern_func *elem;
1079 elem = TAILQ_FIRST(&p->extern_funcs);
1083 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1091 static struct header *
1092 header_find(struct rte_swx_pipeline *p, const char *name)
1094 struct header *elem;
1096 TAILQ_FOREACH(elem, &p->headers, node)
1097 if (strcmp(elem->name, name) == 0)
1103 static struct header *
1104 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1106 struct header *elem;
1108 TAILQ_FOREACH(elem, &p->headers, node)
1109 if (elem->struct_id == struct_id)
1115 static struct header *
1116 header_parse(struct rte_swx_pipeline *p,
1119 if (name[0] != 'h' || name[1] != '.')
1122 return header_find(p, &name[2]);
1125 static struct field *
1126 header_field_parse(struct rte_swx_pipeline *p,
1128 struct header **header)
1132 char *header_name, *field_name;
1134 if ((name[0] != 'h') || (name[1] != '.'))
1137 header_name = strdup(&name[2]);
1141 field_name = strchr(header_name, '.');
1150 h = header_find(p, header_name);
1156 f = struct_type_field_find(h->st, field_name);
1170 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1172 const char *struct_type_name)
1174 struct struct_type *st;
1176 size_t n_headers_max;
1179 CHECK_NAME(name, EINVAL);
1180 CHECK_NAME(struct_type_name, EINVAL);
1182 CHECK(!header_find(p, name), EEXIST);
1184 st = struct_type_find(p, struct_type_name);
1187 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1188 CHECK(p->n_headers < n_headers_max, ENOSPC);
1190 /* Node allocation. */
1191 h = calloc(1, sizeof(struct header));
1194 /* Node initialization. */
1195 strcpy(h->name, name);
1197 h->struct_id = p->n_structs;
1198 h->id = p->n_headers;
1200 /* Node add to tailq. */
1201 TAILQ_INSERT_TAIL(&p->headers, h, node);
1209 header_build(struct rte_swx_pipeline *p)
1212 uint32_t n_bytes = 0, i;
1214 TAILQ_FOREACH(h, &p->headers, node) {
1215 n_bytes += h->st->n_bits / 8;
1218 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1219 struct thread *t = &p->threads[i];
1220 uint32_t offset = 0;
1222 t->headers = calloc(p->n_headers,
1223 sizeof(struct header_runtime));
1224 CHECK(t->headers, ENOMEM);
1226 t->headers_out = calloc(p->n_headers,
1227 sizeof(struct header_out_runtime));
1228 CHECK(t->headers_out, ENOMEM);
1230 t->header_storage = calloc(1, n_bytes);
1231 CHECK(t->header_storage, ENOMEM);
1233 t->header_out_storage = calloc(1, n_bytes);
1234 CHECK(t->header_out_storage, ENOMEM);
1236 TAILQ_FOREACH(h, &p->headers, node) {
1237 uint8_t *header_storage;
1238 uint32_t n_bytes = h->st->n_bits / 8;
1240 header_storage = &t->header_storage[offset];
1243 t->headers[h->id].ptr0 = header_storage;
1244 t->headers[h->id].n_bytes = n_bytes;
1246 t->structs[h->struct_id] = header_storage;
1254 header_build_free(struct rte_swx_pipeline *p)
1258 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1259 struct thread *t = &p->threads[i];
1261 free(t->headers_out);
1262 t->headers_out = NULL;
1267 free(t->header_out_storage);
1268 t->header_out_storage = NULL;
1270 free(t->header_storage);
1271 t->header_storage = NULL;
1276 header_free(struct rte_swx_pipeline *p)
1278 header_build_free(p);
1281 struct header *elem;
1283 elem = TAILQ_FIRST(&p->headers);
1287 TAILQ_REMOVE(&p->headers, elem, node);
1295 static struct field *
1296 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1298 if (!p->metadata_st)
1301 if (name[0] != 'm' || name[1] != '.')
1304 return struct_type_field_find(p->metadata_st, &name[2]);
1308 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1309 const char *struct_type_name)
1311 struct struct_type *st = NULL;
1315 CHECK_NAME(struct_type_name, EINVAL);
1316 st = struct_type_find(p, struct_type_name);
1318 CHECK(!st->var_size, EINVAL);
1319 CHECK(!p->metadata_st, EINVAL);
1321 p->metadata_st = st;
1322 p->metadata_struct_id = p->n_structs;
1330 metadata_build(struct rte_swx_pipeline *p)
1332 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1335 /* Thread-level initialization. */
1336 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1337 struct thread *t = &p->threads[i];
1340 metadata = calloc(1, n_bytes);
1341 CHECK(metadata, ENOMEM);
1343 t->metadata = metadata;
1344 t->structs[p->metadata_struct_id] = metadata;
1351 metadata_build_free(struct rte_swx_pipeline *p)
1355 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1356 struct thread *t = &p->threads[i];
1364 metadata_free(struct rte_swx_pipeline *p)
1366 metadata_build_free(p);
1373 instruction_is_tx(enum instruction_type type)
1387 instruction_does_tx(struct instruction *instr)
1389 switch (instr->type) {
1393 case INSTR_HDR_EMIT_TX:
1394 case INSTR_HDR_EMIT2_TX:
1395 case INSTR_HDR_EMIT3_TX:
1396 case INSTR_HDR_EMIT4_TX:
1397 case INSTR_HDR_EMIT5_TX:
1398 case INSTR_HDR_EMIT6_TX:
1399 case INSTR_HDR_EMIT7_TX:
1400 case INSTR_HDR_EMIT8_TX:
1408 instruction_is_jmp(struct instruction *instr)
1410 switch (instr->type) {
1412 case INSTR_JMP_VALID:
1413 case INSTR_JMP_INVALID:
1415 case INSTR_JMP_MISS:
1416 case INSTR_JMP_ACTION_HIT:
1417 case INSTR_JMP_ACTION_MISS:
1419 case INSTR_JMP_EQ_MH:
1420 case INSTR_JMP_EQ_HM:
1421 case INSTR_JMP_EQ_HH:
1422 case INSTR_JMP_EQ_I:
1424 case INSTR_JMP_NEQ_MH:
1425 case INSTR_JMP_NEQ_HM:
1426 case INSTR_JMP_NEQ_HH:
1427 case INSTR_JMP_NEQ_I:
1429 case INSTR_JMP_LT_MH:
1430 case INSTR_JMP_LT_HM:
1431 case INSTR_JMP_LT_HH:
1432 case INSTR_JMP_LT_MI:
1433 case INSTR_JMP_LT_HI:
1435 case INSTR_JMP_GT_MH:
1436 case INSTR_JMP_GT_HM:
1437 case INSTR_JMP_GT_HH:
1438 case INSTR_JMP_GT_MI:
1439 case INSTR_JMP_GT_HI:
1448 instruction_does_thread_yield(struct instruction *instr)
1450 switch (instr->type) {
1453 case INSTR_TABLE_AF:
1454 case INSTR_SELECTOR:
1456 case INSTR_LEARNER_AF:
1457 case INSTR_EXTERN_OBJ:
1458 case INSTR_EXTERN_FUNC:
1465 static struct field *
1466 action_field_parse(struct action *action, const char *name);
1468 static struct field *
1469 struct_field_parse(struct rte_swx_pipeline *p,
1470 struct action *action,
1472 uint32_t *struct_id)
1479 struct header *header;
1481 f = header_field_parse(p, name, &header);
1485 *struct_id = header->struct_id;
1491 f = metadata_field_parse(p, name);
1495 *struct_id = p->metadata_struct_id;
1504 f = action_field_parse(action, name);
1514 struct extern_obj *obj;
1516 f = extern_obj_mailbox_field_parse(p, name, &obj);
1520 *struct_id = obj->struct_id;
1526 struct extern_func *func;
1528 f = extern_func_mailbox_field_parse(p, name, &func);
1532 *struct_id = func->struct_id;
1545 instr_rx_translate(struct rte_swx_pipeline *p,
1546 struct action *action,
1549 struct instruction *instr,
1550 struct instruction_data *data __rte_unused)
1554 CHECK(!action, EINVAL);
1555 CHECK(n_tokens == 2, EINVAL);
1557 f = metadata_field_parse(p, tokens[1]);
1560 instr->type = INSTR_RX;
1561 instr->io.io.offset = f->offset / 8;
1562 instr->io.io.n_bits = f->n_bits;
1570 instr_tx_translate(struct rte_swx_pipeline *p,
1571 struct action *action __rte_unused,
1574 struct instruction *instr,
1575 struct instruction_data *data __rte_unused)
1577 char *port = tokens[1];
1581 CHECK(n_tokens == 2, EINVAL);
1583 f = metadata_field_parse(p, port);
1585 instr->type = INSTR_TX;
1586 instr->io.io.offset = f->offset / 8;
1587 instr->io.io.n_bits = f->n_bits;
1592 port_val = strtoul(port, &port, 0);
1593 CHECK(!port[0], EINVAL);
1595 instr->type = INSTR_TX_I;
1596 instr->io.io.val = port_val;
1601 instr_drop_translate(struct rte_swx_pipeline *p __rte_unused,
1602 struct action *action __rte_unused,
1603 char **tokens __rte_unused,
1605 struct instruction *instr,
1606 struct instruction_data *data __rte_unused)
1608 CHECK(n_tokens == 1, EINVAL);
1611 instr->type = INSTR_DROP;
1616 instr_tx_exec(struct rte_swx_pipeline *p)
1618 struct thread *t = &p->threads[p->thread_id];
1619 struct instruction *ip = t->ip;
1621 __instr_tx_exec(p, t, ip);
1624 thread_ip_reset(p, t);
1629 instr_tx_i_exec(struct rte_swx_pipeline *p)
1631 struct thread *t = &p->threads[p->thread_id];
1632 struct instruction *ip = t->ip;
1634 __instr_tx_i_exec(p, t, ip);
1637 thread_ip_reset(p, t);
1642 instr_drop_exec(struct rte_swx_pipeline *p)
1644 struct thread *t = &p->threads[p->thread_id];
1645 struct instruction *ip = t->ip;
1647 __instr_drop_exec(p, t, ip);
1650 thread_ip_reset(p, t);
1658 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1659 struct action *action,
1662 struct instruction *instr,
1663 struct instruction_data *data __rte_unused)
1667 CHECK(!action, EINVAL);
1668 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1670 h = header_parse(p, tokens[1]);
1673 if (n_tokens == 2) {
1674 CHECK(!h->st->var_size, EINVAL);
1676 instr->type = INSTR_HDR_EXTRACT;
1677 instr->io.hdr.header_id[0] = h->id;
1678 instr->io.hdr.struct_id[0] = h->struct_id;
1679 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1683 CHECK(h->st->var_size, EINVAL);
1685 mf = metadata_field_parse(p, tokens[2]);
1687 CHECK(!mf->var_size, EINVAL);
1689 instr->type = INSTR_HDR_EXTRACT_M;
1690 instr->io.io.offset = mf->offset / 8;
1691 instr->io.io.n_bits = mf->n_bits;
1692 instr->io.hdr.header_id[0] = h->id;
1693 instr->io.hdr.struct_id[0] = h->struct_id;
1694 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1701 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1702 struct action *action,
1705 struct instruction *instr,
1706 struct instruction_data *data __rte_unused)
1710 CHECK(!action, EINVAL);
1711 CHECK(n_tokens == 2, EINVAL);
1713 h = header_parse(p, tokens[1]);
1715 CHECK(!h->st->var_size, EINVAL);
1717 instr->type = INSTR_HDR_LOOKAHEAD;
1718 instr->io.hdr.header_id[0] = h->id;
1719 instr->io.hdr.struct_id[0] = h->struct_id;
1720 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1726 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1728 struct thread *t = &p->threads[p->thread_id];
1729 struct instruction *ip = t->ip;
1731 __instr_hdr_extract_exec(p, t, ip);
1738 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1740 struct thread *t = &p->threads[p->thread_id];
1741 struct instruction *ip = t->ip;
1743 __instr_hdr_extract2_exec(p, t, ip);
1750 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1752 struct thread *t = &p->threads[p->thread_id];
1753 struct instruction *ip = t->ip;
1755 __instr_hdr_extract3_exec(p, t, ip);
1762 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1764 struct thread *t = &p->threads[p->thread_id];
1765 struct instruction *ip = t->ip;
1767 __instr_hdr_extract4_exec(p, t, ip);
1774 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1776 struct thread *t = &p->threads[p->thread_id];
1777 struct instruction *ip = t->ip;
1779 __instr_hdr_extract5_exec(p, t, ip);
1786 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1788 struct thread *t = &p->threads[p->thread_id];
1789 struct instruction *ip = t->ip;
1791 __instr_hdr_extract6_exec(p, t, ip);
1798 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1800 struct thread *t = &p->threads[p->thread_id];
1801 struct instruction *ip = t->ip;
1803 __instr_hdr_extract7_exec(p, t, ip);
1810 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1812 struct thread *t = &p->threads[p->thread_id];
1813 struct instruction *ip = t->ip;
1815 __instr_hdr_extract8_exec(p, t, ip);
1822 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1824 struct thread *t = &p->threads[p->thread_id];
1825 struct instruction *ip = t->ip;
1827 __instr_hdr_extract_m_exec(p, t, ip);
1834 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1836 struct thread *t = &p->threads[p->thread_id];
1837 struct instruction *ip = t->ip;
1839 __instr_hdr_lookahead_exec(p, t, ip);
1849 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1850 struct action *action __rte_unused,
1853 struct instruction *instr,
1854 struct instruction_data *data __rte_unused)
1858 CHECK(n_tokens == 2, EINVAL);
1860 h = header_parse(p, tokens[1]);
1863 instr->type = INSTR_HDR_EMIT;
1864 instr->io.hdr.header_id[0] = h->id;
1865 instr->io.hdr.struct_id[0] = h->struct_id;
1866 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1871 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1873 struct thread *t = &p->threads[p->thread_id];
1874 struct instruction *ip = t->ip;
1876 __instr_hdr_emit_exec(p, t, ip);
1883 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1885 struct thread *t = &p->threads[p->thread_id];
1886 struct instruction *ip = t->ip;
1888 __instr_hdr_emit_tx_exec(p, t, ip);
1891 thread_ip_reset(p, t);
1896 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1898 struct thread *t = &p->threads[p->thread_id];
1899 struct instruction *ip = t->ip;
1901 __instr_hdr_emit2_tx_exec(p, t, ip);
1904 thread_ip_reset(p, t);
1909 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1911 struct thread *t = &p->threads[p->thread_id];
1912 struct instruction *ip = t->ip;
1914 __instr_hdr_emit3_tx_exec(p, t, ip);
1917 thread_ip_reset(p, t);
1922 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1924 struct thread *t = &p->threads[p->thread_id];
1925 struct instruction *ip = t->ip;
1927 __instr_hdr_emit4_tx_exec(p, t, ip);
1930 thread_ip_reset(p, t);
1935 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1937 struct thread *t = &p->threads[p->thread_id];
1938 struct instruction *ip = t->ip;
1940 __instr_hdr_emit5_tx_exec(p, t, ip);
1943 thread_ip_reset(p, t);
1948 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1950 struct thread *t = &p->threads[p->thread_id];
1951 struct instruction *ip = t->ip;
1953 __instr_hdr_emit6_tx_exec(p, t, ip);
1956 thread_ip_reset(p, t);
1961 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1963 struct thread *t = &p->threads[p->thread_id];
1964 struct instruction *ip = t->ip;
1966 __instr_hdr_emit7_tx_exec(p, t, ip);
1969 thread_ip_reset(p, t);
1974 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1976 struct thread *t = &p->threads[p->thread_id];
1977 struct instruction *ip = t->ip;
1979 __instr_hdr_emit8_tx_exec(p, t, ip);
1982 thread_ip_reset(p, t);
1990 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1991 struct action *action __rte_unused,
1994 struct instruction *instr,
1995 struct instruction_data *data __rte_unused)
1999 CHECK(n_tokens == 2, EINVAL);
2001 h = header_parse(p, tokens[1]);
2004 instr->type = INSTR_HDR_VALIDATE;
2005 instr->valid.header_id = h->id;
2010 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
2012 struct thread *t = &p->threads[p->thread_id];
2013 struct instruction *ip = t->ip;
2015 __instr_hdr_validate_exec(p, t, ip);
2025 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
2026 struct action *action __rte_unused,
2029 struct instruction *instr,
2030 struct instruction_data *data __rte_unused)
2034 CHECK(n_tokens == 2, EINVAL);
2036 h = header_parse(p, tokens[1]);
2039 instr->type = INSTR_HDR_INVALIDATE;
2040 instr->valid.header_id = h->id;
2045 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
2047 struct thread *t = &p->threads[p->thread_id];
2048 struct instruction *ip = t->ip;
2050 __instr_hdr_invalidate_exec(p, t, ip);
2059 static struct table *
2060 table_find(struct rte_swx_pipeline *p, const char *name);
2062 static struct selector *
2063 selector_find(struct rte_swx_pipeline *p, const char *name);
2065 static struct learner *
2066 learner_find(struct rte_swx_pipeline *p, const char *name);
2069 instr_table_translate(struct rte_swx_pipeline *p,
2070 struct action *action,
2073 struct instruction *instr,
2074 struct instruction_data *data __rte_unused)
2080 CHECK(!action, EINVAL);
2081 CHECK(n_tokens == 2, EINVAL);
2083 t = table_find(p, tokens[1]);
2085 instr->type = INSTR_TABLE;
2086 instr->table.table_id = t->id;
2090 s = selector_find(p, tokens[1]);
2092 instr->type = INSTR_SELECTOR;
2093 instr->table.table_id = s->id;
2097 l = learner_find(p, tokens[1]);
2099 instr->type = INSTR_LEARNER;
2100 instr->table.table_id = l->id;
2108 instr_table_exec(struct rte_swx_pipeline *p)
2110 struct thread *t = &p->threads[p->thread_id];
2111 struct instruction *ip = t->ip;
2112 uint32_t table_id = ip->table.table_id;
2113 struct rte_swx_table_state *ts = &t->table_state[table_id];
2114 struct table_runtime *table = &t->tables[table_id];
2115 struct table_statistics *stats = &p->table_stats[table_id];
2116 uint64_t action_id, n_pkts_hit, n_pkts_action;
2117 uint8_t *action_data;
2121 done = table->func(ts->obj,
2129 TRACE("[Thread %2u] table %u (not finalized)\n",
2137 action_id = hit ? action_id : ts->default_action_id;
2138 action_data = hit ? action_data : ts->default_action_data;
2139 n_pkts_hit = stats->n_pkts_hit[hit];
2140 n_pkts_action = stats->n_pkts_action[action_id];
2142 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2145 hit ? "hit" : "miss",
2146 (uint32_t)action_id);
2148 t->action_id = action_id;
2149 t->structs[0] = action_data;
2151 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2152 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2155 thread_ip_action_call(p, t, action_id);
2159 instr_table_af_exec(struct rte_swx_pipeline *p)
2161 struct thread *t = &p->threads[p->thread_id];
2162 struct instruction *ip = t->ip;
2163 uint32_t table_id = ip->table.table_id;
2164 struct rte_swx_table_state *ts = &t->table_state[table_id];
2165 struct table_runtime *table = &t->tables[table_id];
2166 struct table_statistics *stats = &p->table_stats[table_id];
2167 uint64_t action_id, n_pkts_hit, n_pkts_action;
2168 uint8_t *action_data;
2169 action_func_t action_func;
2173 done = table->func(ts->obj,
2181 TRACE("[Thread %2u] table %u (not finalized)\n",
2189 action_id = hit ? action_id : ts->default_action_id;
2190 action_data = hit ? action_data : ts->default_action_data;
2191 action_func = p->action_funcs[action_id];
2192 n_pkts_hit = stats->n_pkts_hit[hit];
2193 n_pkts_action = stats->n_pkts_action[action_id];
2195 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2198 hit ? "hit" : "miss",
2199 (uint32_t)action_id);
2201 t->action_id = action_id;
2202 t->structs[0] = action_data;
2204 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2205 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2215 instr_selector_exec(struct rte_swx_pipeline *p)
2217 struct thread *t = &p->threads[p->thread_id];
2218 struct instruction *ip = t->ip;
2219 uint32_t selector_id = ip->table.table_id;
2220 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2221 struct selector_runtime *selector = &t->selectors[selector_id];
2222 struct selector_statistics *stats = &p->selector_stats[selector_id];
2223 uint64_t n_pkts = stats->n_pkts;
2227 done = rte_swx_table_selector_select(ts->obj,
2229 selector->group_id_buffer,
2230 selector->selector_buffer,
2231 selector->member_id_buffer);
2234 TRACE("[Thread %2u] selector %u (not finalized)\n",
2243 TRACE("[Thread %2u] selector %u\n",
2247 stats->n_pkts = n_pkts + 1;
2254 instr_learner_exec(struct rte_swx_pipeline *p)
2256 struct thread *t = &p->threads[p->thread_id];
2257 struct instruction *ip = t->ip;
2258 uint32_t learner_id = ip->table.table_id;
2259 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2260 p->n_selectors + learner_id];
2261 struct learner_runtime *l = &t->learners[learner_id];
2262 struct learner_statistics *stats = &p->learner_stats[learner_id];
2263 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2264 uint8_t *action_data;
2268 time = rte_get_tsc_cycles();
2270 done = rte_swx_table_learner_lookup(ts->obj,
2279 TRACE("[Thread %2u] learner %u (not finalized)\n",
2287 action_id = hit ? action_id : ts->default_action_id;
2288 action_data = hit ? action_data : ts->default_action_data;
2289 n_pkts_hit = stats->n_pkts_hit[hit];
2290 n_pkts_action = stats->n_pkts_action[action_id];
2292 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2295 hit ? "hit" : "miss",
2296 (uint32_t)action_id);
2298 t->action_id = action_id;
2299 t->structs[0] = action_data;
2301 t->learner_id = learner_id;
2303 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2304 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2307 thread_ip_action_call(p, t, action_id);
2311 instr_learner_af_exec(struct rte_swx_pipeline *p)
2313 struct thread *t = &p->threads[p->thread_id];
2314 struct instruction *ip = t->ip;
2315 uint32_t learner_id = ip->table.table_id;
2316 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2317 p->n_selectors + learner_id];
2318 struct learner_runtime *l = &t->learners[learner_id];
2319 struct learner_statistics *stats = &p->learner_stats[learner_id];
2320 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2321 uint8_t *action_data;
2322 action_func_t action_func;
2326 time = rte_get_tsc_cycles();
2328 done = rte_swx_table_learner_lookup(ts->obj,
2337 TRACE("[Thread %2u] learner %u (not finalized)\n",
2345 action_id = hit ? action_id : ts->default_action_id;
2346 action_data = hit ? action_data : ts->default_action_data;
2347 action_func = p->action_funcs[action_id];
2348 n_pkts_hit = stats->n_pkts_hit[hit];
2349 n_pkts_action = stats->n_pkts_action[action_id];
2351 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2354 hit ? "hit" : "miss",
2355 (uint32_t)action_id);
2357 t->action_id = action_id;
2358 t->structs[0] = action_data;
2360 t->learner_id = learner_id;
2362 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2363 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2366 thread_ip_action_call(p, t, action_id);
2375 static struct action *
2376 action_find(struct rte_swx_pipeline *p, const char *name);
2379 action_has_nbo_args(struct action *a);
2382 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
2385 instr_learn_translate(struct rte_swx_pipeline *p,
2386 struct action *action,
2389 struct instruction *instr,
2390 struct instruction_data *data __rte_unused)
2393 const char *mf_name;
2394 uint32_t mf_offset = 0;
2396 CHECK(action, EINVAL);
2397 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
2399 a = action_find(p, tokens[1]);
2401 CHECK(!action_has_nbo_args(a), EINVAL);
2403 mf_name = (n_tokens > 2) ? tokens[2] : NULL;
2404 CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
2409 mf = metadata_field_parse(p, mf_name);
2412 mf_offset = mf->offset / 8;
2415 instr->type = INSTR_LEARNER_LEARN;
2416 instr->learn.action_id = a->id;
2417 instr->learn.mf_offset = mf_offset;
2423 instr_learn_exec(struct rte_swx_pipeline *p)
2425 struct thread *t = &p->threads[p->thread_id];
2426 struct instruction *ip = t->ip;
2428 __instr_learn_exec(p, t, ip);
2438 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2439 struct action *action,
2440 char **tokens __rte_unused,
2442 struct instruction *instr,
2443 struct instruction_data *data __rte_unused)
2445 CHECK(action, EINVAL);
2446 CHECK(n_tokens == 1, EINVAL);
2448 instr->type = INSTR_LEARNER_FORGET;
2454 instr_forget_exec(struct rte_swx_pipeline *p)
2456 struct thread *t = &p->threads[p->thread_id];
2457 struct instruction *ip = t->ip;
2459 __instr_forget_exec(p, t, ip);
2469 instr_extern_translate(struct rte_swx_pipeline *p,
2470 struct action *action __rte_unused,
2473 struct instruction *instr,
2474 struct instruction_data *data __rte_unused)
2476 char *token = tokens[1];
2478 CHECK(n_tokens == 2, EINVAL);
2480 if (token[0] == 'e') {
2481 struct extern_obj *obj;
2482 struct extern_type_member_func *func;
2484 func = extern_obj_member_func_parse(p, token, &obj);
2485 CHECK(func, EINVAL);
2487 instr->type = INSTR_EXTERN_OBJ;
2488 instr->ext_obj.ext_obj_id = obj->id;
2489 instr->ext_obj.func_id = func->id;
2494 if (token[0] == 'f') {
2495 struct extern_func *func;
2497 func = extern_func_parse(p, token);
2498 CHECK(func, EINVAL);
2500 instr->type = INSTR_EXTERN_FUNC;
2501 instr->ext_func.ext_func_id = func->id;
2510 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2512 struct thread *t = &p->threads[p->thread_id];
2513 struct instruction *ip = t->ip;
2516 /* Extern object member function execute. */
2517 done = __instr_extern_obj_exec(p, t, ip);
2520 thread_ip_inc_cond(t, done);
2521 thread_yield_cond(p, done ^ 1);
2525 instr_extern_func_exec(struct rte_swx_pipeline *p)
2527 struct thread *t = &p->threads[p->thread_id];
2528 struct instruction *ip = t->ip;
2531 /* Extern function execute. */
2532 done = __instr_extern_func_exec(p, t, ip);
2535 thread_ip_inc_cond(t, done);
2536 thread_yield_cond(p, done ^ 1);
2543 instr_mov_translate(struct rte_swx_pipeline *p,
2544 struct action *action,
2547 struct instruction *instr,
2548 struct instruction_data *data __rte_unused)
2550 char *dst = tokens[1], *src = tokens[2];
2551 struct field *fdst, *fsrc;
2553 uint32_t dst_struct_id = 0, src_struct_id = 0;
2555 CHECK(n_tokens == 3, EINVAL);
2557 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2558 CHECK(fdst, EINVAL);
2559 CHECK(!fdst->var_size, EINVAL);
2561 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2562 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2564 CHECK(!fsrc->var_size, EINVAL);
2566 instr->type = INSTR_MOV;
2567 if (dst[0] != 'h' && src[0] == 'h')
2568 instr->type = INSTR_MOV_MH;
2569 if (dst[0] == 'h' && src[0] != 'h')
2570 instr->type = INSTR_MOV_HM;
2571 if (dst[0] == 'h' && src[0] == 'h')
2572 instr->type = INSTR_MOV_HH;
2574 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2575 instr->mov.dst.n_bits = fdst->n_bits;
2576 instr->mov.dst.offset = fdst->offset / 8;
2577 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2578 instr->mov.src.n_bits = fsrc->n_bits;
2579 instr->mov.src.offset = fsrc->offset / 8;
2584 src_val = strtoull(src, &src, 0);
2585 CHECK(!src[0], EINVAL);
2588 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2590 instr->type = INSTR_MOV_I;
2591 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2592 instr->mov.dst.n_bits = fdst->n_bits;
2593 instr->mov.dst.offset = fdst->offset / 8;
2594 instr->mov.src_val = src_val;
2599 instr_mov_exec(struct rte_swx_pipeline *p)
2601 struct thread *t = &p->threads[p->thread_id];
2602 struct instruction *ip = t->ip;
2604 __instr_mov_exec(p, t, ip);
2611 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2613 struct thread *t = &p->threads[p->thread_id];
2614 struct instruction *ip = t->ip;
2616 __instr_mov_mh_exec(p, t, ip);
2623 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2625 struct thread *t = &p->threads[p->thread_id];
2626 struct instruction *ip = t->ip;
2628 __instr_mov_hm_exec(p, t, ip);
2635 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2637 struct thread *t = &p->threads[p->thread_id];
2638 struct instruction *ip = t->ip;
2640 __instr_mov_hh_exec(p, t, ip);
2647 instr_mov_i_exec(struct rte_swx_pipeline *p)
2649 struct thread *t = &p->threads[p->thread_id];
2650 struct instruction *ip = t->ip;
2652 __instr_mov_i_exec(p, t, ip);
2662 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2664 struct thread *t = &p->threads[p->thread_id];
2665 struct instruction *ip = t->ip;
2667 __instr_dma_ht_exec(p, t, ip);
2674 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2676 struct thread *t = &p->threads[p->thread_id];
2677 struct instruction *ip = t->ip;
2679 __instr_dma_ht2_exec(p, t, ip);
2686 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2688 struct thread *t = &p->threads[p->thread_id];
2689 struct instruction *ip = t->ip;
2691 __instr_dma_ht3_exec(p, t, ip);
2698 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2700 struct thread *t = &p->threads[p->thread_id];
2701 struct instruction *ip = t->ip;
2703 __instr_dma_ht4_exec(p, t, ip);
2710 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2712 struct thread *t = &p->threads[p->thread_id];
2713 struct instruction *ip = t->ip;
2715 __instr_dma_ht5_exec(p, t, ip);
2722 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2724 struct thread *t = &p->threads[p->thread_id];
2725 struct instruction *ip = t->ip;
2727 __instr_dma_ht6_exec(p, t, ip);
2734 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2736 struct thread *t = &p->threads[p->thread_id];
2737 struct instruction *ip = t->ip;
2739 __instr_dma_ht7_exec(p, t, ip);
2746 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2748 struct thread *t = &p->threads[p->thread_id];
2749 struct instruction *ip = t->ip;
2751 __instr_dma_ht8_exec(p, t, ip);
2761 instr_alu_add_translate(struct rte_swx_pipeline *p,
2762 struct action *action,
2765 struct instruction *instr,
2766 struct instruction_data *data __rte_unused)
2768 char *dst = tokens[1], *src = tokens[2];
2769 struct field *fdst, *fsrc;
2771 uint32_t dst_struct_id = 0, src_struct_id = 0;
2773 CHECK(n_tokens == 3, EINVAL);
2775 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2776 CHECK(fdst, EINVAL);
2777 CHECK(!fdst->var_size, EINVAL);
2779 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2780 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2782 CHECK(!fsrc->var_size, EINVAL);
2784 instr->type = INSTR_ALU_ADD;
2785 if (dst[0] == 'h' && src[0] != 'h')
2786 instr->type = INSTR_ALU_ADD_HM;
2787 if (dst[0] != 'h' && src[0] == 'h')
2788 instr->type = INSTR_ALU_ADD_MH;
2789 if (dst[0] == 'h' && src[0] == 'h')
2790 instr->type = INSTR_ALU_ADD_HH;
2792 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2793 instr->alu.dst.n_bits = fdst->n_bits;
2794 instr->alu.dst.offset = fdst->offset / 8;
2795 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2796 instr->alu.src.n_bits = fsrc->n_bits;
2797 instr->alu.src.offset = fsrc->offset / 8;
2801 /* ADD_MI, ADD_HI. */
2802 src_val = strtoull(src, &src, 0);
2803 CHECK(!src[0], EINVAL);
2805 instr->type = INSTR_ALU_ADD_MI;
2807 instr->type = INSTR_ALU_ADD_HI;
2809 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2810 instr->alu.dst.n_bits = fdst->n_bits;
2811 instr->alu.dst.offset = fdst->offset / 8;
2812 instr->alu.src_val = src_val;
2817 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2818 struct action *action,
2821 struct instruction *instr,
2822 struct instruction_data *data __rte_unused)
2824 char *dst = tokens[1], *src = tokens[2];
2825 struct field *fdst, *fsrc;
2827 uint32_t dst_struct_id = 0, src_struct_id = 0;
2829 CHECK(n_tokens == 3, EINVAL);
2831 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2832 CHECK(fdst, EINVAL);
2833 CHECK(!fdst->var_size, EINVAL);
2835 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2836 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2838 CHECK(!fsrc->var_size, EINVAL);
2840 instr->type = INSTR_ALU_SUB;
2841 if (dst[0] == 'h' && src[0] != 'h')
2842 instr->type = INSTR_ALU_SUB_HM;
2843 if (dst[0] != 'h' && src[0] == 'h')
2844 instr->type = INSTR_ALU_SUB_MH;
2845 if (dst[0] == 'h' && src[0] == 'h')
2846 instr->type = INSTR_ALU_SUB_HH;
2848 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2849 instr->alu.dst.n_bits = fdst->n_bits;
2850 instr->alu.dst.offset = fdst->offset / 8;
2851 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2852 instr->alu.src.n_bits = fsrc->n_bits;
2853 instr->alu.src.offset = fsrc->offset / 8;
2857 /* SUB_MI, SUB_HI. */
2858 src_val = strtoull(src, &src, 0);
2859 CHECK(!src[0], EINVAL);
2861 instr->type = INSTR_ALU_SUB_MI;
2863 instr->type = INSTR_ALU_SUB_HI;
2865 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2866 instr->alu.dst.n_bits = fdst->n_bits;
2867 instr->alu.dst.offset = fdst->offset / 8;
2868 instr->alu.src_val = src_val;
2873 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2874 struct action *action __rte_unused,
2877 struct instruction *instr,
2878 struct instruction_data *data __rte_unused)
2880 char *dst = tokens[1], *src = tokens[2];
2881 struct header *hdst, *hsrc;
2882 struct field *fdst, *fsrc;
2884 CHECK(n_tokens == 3, EINVAL);
2886 fdst = header_field_parse(p, dst, &hdst);
2887 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2888 CHECK(!fdst->var_size, EINVAL);
2891 fsrc = header_field_parse(p, src, &hsrc);
2893 CHECK(!fsrc->var_size, EINVAL);
2895 instr->type = INSTR_ALU_CKADD_FIELD;
2896 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2897 instr->alu.dst.n_bits = fdst->n_bits;
2898 instr->alu.dst.offset = fdst->offset / 8;
2899 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2900 instr->alu.src.n_bits = fsrc->n_bits;
2901 instr->alu.src.offset = fsrc->offset / 8;
2905 /* CKADD_STRUCT, CKADD_STRUCT20. */
2906 hsrc = header_parse(p, src);
2907 CHECK(hsrc, EINVAL);
2908 CHECK(!hsrc->st->var_size, EINVAL);
2910 instr->type = INSTR_ALU_CKADD_STRUCT;
2911 if ((hsrc->st->n_bits / 8) == 20)
2912 instr->type = INSTR_ALU_CKADD_STRUCT20;
2914 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2915 instr->alu.dst.n_bits = fdst->n_bits;
2916 instr->alu.dst.offset = fdst->offset / 8;
2917 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2918 instr->alu.src.n_bits = hsrc->st->n_bits;
2919 instr->alu.src.offset = 0; /* Unused. */
2924 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2925 struct action *action __rte_unused,
2928 struct instruction *instr,
2929 struct instruction_data *data __rte_unused)
2931 char *dst = tokens[1], *src = tokens[2];
2932 struct header *hdst, *hsrc;
2933 struct field *fdst, *fsrc;
2935 CHECK(n_tokens == 3, EINVAL);
2937 fdst = header_field_parse(p, dst, &hdst);
2938 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2939 CHECK(!fdst->var_size, EINVAL);
2941 fsrc = header_field_parse(p, src, &hsrc);
2942 CHECK(fsrc, EINVAL);
2943 CHECK(!fsrc->var_size, EINVAL);
2945 instr->type = INSTR_ALU_CKSUB_FIELD;
2946 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2947 instr->alu.dst.n_bits = fdst->n_bits;
2948 instr->alu.dst.offset = fdst->offset / 8;
2949 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2950 instr->alu.src.n_bits = fsrc->n_bits;
2951 instr->alu.src.offset = fsrc->offset / 8;
2956 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2957 struct action *action,
2960 struct instruction *instr,
2961 struct instruction_data *data __rte_unused)
2963 char *dst = tokens[1], *src = tokens[2];
2964 struct field *fdst, *fsrc;
2966 uint32_t dst_struct_id = 0, src_struct_id = 0;
2968 CHECK(n_tokens == 3, EINVAL);
2970 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2971 CHECK(fdst, EINVAL);
2972 CHECK(!fdst->var_size, EINVAL);
2974 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2975 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2977 CHECK(!fsrc->var_size, EINVAL);
2979 instr->type = INSTR_ALU_SHL;
2980 if (dst[0] == 'h' && src[0] != 'h')
2981 instr->type = INSTR_ALU_SHL_HM;
2982 if (dst[0] != 'h' && src[0] == 'h')
2983 instr->type = INSTR_ALU_SHL_MH;
2984 if (dst[0] == 'h' && src[0] == 'h')
2985 instr->type = INSTR_ALU_SHL_HH;
2987 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2988 instr->alu.dst.n_bits = fdst->n_bits;
2989 instr->alu.dst.offset = fdst->offset / 8;
2990 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2991 instr->alu.src.n_bits = fsrc->n_bits;
2992 instr->alu.src.offset = fsrc->offset / 8;
2996 /* SHL_MI, SHL_HI. */
2997 src_val = strtoull(src, &src, 0);
2998 CHECK(!src[0], EINVAL);
3000 instr->type = INSTR_ALU_SHL_MI;
3002 instr->type = INSTR_ALU_SHL_HI;
3004 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3005 instr->alu.dst.n_bits = fdst->n_bits;
3006 instr->alu.dst.offset = fdst->offset / 8;
3007 instr->alu.src_val = src_val;
3012 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3013 struct action *action,
3016 struct instruction *instr,
3017 struct instruction_data *data __rte_unused)
3019 char *dst = tokens[1], *src = tokens[2];
3020 struct field *fdst, *fsrc;
3022 uint32_t dst_struct_id = 0, src_struct_id = 0;
3024 CHECK(n_tokens == 3, EINVAL);
3026 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3027 CHECK(fdst, EINVAL);
3028 CHECK(!fdst->var_size, EINVAL);
3030 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3031 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3033 CHECK(!fsrc->var_size, EINVAL);
3035 instr->type = INSTR_ALU_SHR;
3036 if (dst[0] == 'h' && src[0] != 'h')
3037 instr->type = INSTR_ALU_SHR_HM;
3038 if (dst[0] != 'h' && src[0] == 'h')
3039 instr->type = INSTR_ALU_SHR_MH;
3040 if (dst[0] == 'h' && src[0] == 'h')
3041 instr->type = INSTR_ALU_SHR_HH;
3043 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3044 instr->alu.dst.n_bits = fdst->n_bits;
3045 instr->alu.dst.offset = fdst->offset / 8;
3046 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3047 instr->alu.src.n_bits = fsrc->n_bits;
3048 instr->alu.src.offset = fsrc->offset / 8;
3052 /* SHR_MI, SHR_HI. */
3053 src_val = strtoull(src, &src, 0);
3054 CHECK(!src[0], EINVAL);
3056 instr->type = INSTR_ALU_SHR_MI;
3058 instr->type = INSTR_ALU_SHR_HI;
3060 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3061 instr->alu.dst.n_bits = fdst->n_bits;
3062 instr->alu.dst.offset = fdst->offset / 8;
3063 instr->alu.src_val = src_val;
3068 instr_alu_and_translate(struct rte_swx_pipeline *p,
3069 struct action *action,
3072 struct instruction *instr,
3073 struct instruction_data *data __rte_unused)
3075 char *dst = tokens[1], *src = tokens[2];
3076 struct field *fdst, *fsrc;
3078 uint32_t dst_struct_id = 0, src_struct_id = 0;
3080 CHECK(n_tokens == 3, EINVAL);
3082 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3083 CHECK(fdst, EINVAL);
3084 CHECK(!fdst->var_size, EINVAL);
3086 /* AND, AND_MH, AND_HM, AND_HH. */
3087 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3089 CHECK(!fsrc->var_size, EINVAL);
3091 instr->type = INSTR_ALU_AND;
3092 if (dst[0] != 'h' && src[0] == 'h')
3093 instr->type = INSTR_ALU_AND_MH;
3094 if (dst[0] == 'h' && src[0] != 'h')
3095 instr->type = INSTR_ALU_AND_HM;
3096 if (dst[0] == 'h' && src[0] == 'h')
3097 instr->type = INSTR_ALU_AND_HH;
3099 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3100 instr->alu.dst.n_bits = fdst->n_bits;
3101 instr->alu.dst.offset = fdst->offset / 8;
3102 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3103 instr->alu.src.n_bits = fsrc->n_bits;
3104 instr->alu.src.offset = fsrc->offset / 8;
3109 src_val = strtoull(src, &src, 0);
3110 CHECK(!src[0], EINVAL);
3113 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3115 instr->type = INSTR_ALU_AND_I;
3116 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3117 instr->alu.dst.n_bits = fdst->n_bits;
3118 instr->alu.dst.offset = fdst->offset / 8;
3119 instr->alu.src_val = src_val;
3124 instr_alu_or_translate(struct rte_swx_pipeline *p,
3125 struct action *action,
3128 struct instruction *instr,
3129 struct instruction_data *data __rte_unused)
3131 char *dst = tokens[1], *src = tokens[2];
3132 struct field *fdst, *fsrc;
3134 uint32_t dst_struct_id = 0, src_struct_id = 0;
3136 CHECK(n_tokens == 3, EINVAL);
3138 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3139 CHECK(fdst, EINVAL);
3140 CHECK(!fdst->var_size, EINVAL);
3142 /* OR, OR_MH, OR_HM, OR_HH. */
3143 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3145 CHECK(!fsrc->var_size, EINVAL);
3147 instr->type = INSTR_ALU_OR;
3148 if (dst[0] != 'h' && src[0] == 'h')
3149 instr->type = INSTR_ALU_OR_MH;
3150 if (dst[0] == 'h' && src[0] != 'h')
3151 instr->type = INSTR_ALU_OR_HM;
3152 if (dst[0] == 'h' && src[0] == 'h')
3153 instr->type = INSTR_ALU_OR_HH;
3155 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3156 instr->alu.dst.n_bits = fdst->n_bits;
3157 instr->alu.dst.offset = fdst->offset / 8;
3158 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3159 instr->alu.src.n_bits = fsrc->n_bits;
3160 instr->alu.src.offset = fsrc->offset / 8;
3165 src_val = strtoull(src, &src, 0);
3166 CHECK(!src[0], EINVAL);
3169 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3171 instr->type = INSTR_ALU_OR_I;
3172 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3173 instr->alu.dst.n_bits = fdst->n_bits;
3174 instr->alu.dst.offset = fdst->offset / 8;
3175 instr->alu.src_val = src_val;
3180 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3181 struct action *action,
3184 struct instruction *instr,
3185 struct instruction_data *data __rte_unused)
3187 char *dst = tokens[1], *src = tokens[2];
3188 struct field *fdst, *fsrc;
3190 uint32_t dst_struct_id = 0, src_struct_id = 0;
3192 CHECK(n_tokens == 3, EINVAL);
3194 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3195 CHECK(fdst, EINVAL);
3196 CHECK(!fdst->var_size, EINVAL);
3198 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3199 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3201 CHECK(!fsrc->var_size, EINVAL);
3203 instr->type = INSTR_ALU_XOR;
3204 if (dst[0] != 'h' && src[0] == 'h')
3205 instr->type = INSTR_ALU_XOR_MH;
3206 if (dst[0] == 'h' && src[0] != 'h')
3207 instr->type = INSTR_ALU_XOR_HM;
3208 if (dst[0] == 'h' && src[0] == 'h')
3209 instr->type = INSTR_ALU_XOR_HH;
3211 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3212 instr->alu.dst.n_bits = fdst->n_bits;
3213 instr->alu.dst.offset = fdst->offset / 8;
3214 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3215 instr->alu.src.n_bits = fsrc->n_bits;
3216 instr->alu.src.offset = fsrc->offset / 8;
3221 src_val = strtoull(src, &src, 0);
3222 CHECK(!src[0], EINVAL);
3225 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3227 instr->type = INSTR_ALU_XOR_I;
3228 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3229 instr->alu.dst.n_bits = fdst->n_bits;
3230 instr->alu.dst.offset = fdst->offset / 8;
3231 instr->alu.src_val = src_val;
3236 instr_alu_add_exec(struct rte_swx_pipeline *p)
3238 struct thread *t = &p->threads[p->thread_id];
3239 struct instruction *ip = t->ip;
3242 __instr_alu_add_exec(p, t, ip);
3249 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3251 struct thread *t = &p->threads[p->thread_id];
3252 struct instruction *ip = t->ip;
3255 __instr_alu_add_mh_exec(p, t, ip);
3262 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3264 struct thread *t = &p->threads[p->thread_id];
3265 struct instruction *ip = t->ip;
3268 __instr_alu_add_hm_exec(p, t, ip);
3275 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3277 struct thread *t = &p->threads[p->thread_id];
3278 struct instruction *ip = t->ip;
3281 __instr_alu_add_hh_exec(p, t, ip);
3288 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3290 struct thread *t = &p->threads[p->thread_id];
3291 struct instruction *ip = t->ip;
3294 __instr_alu_add_mi_exec(p, t, ip);
3301 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3303 struct thread *t = &p->threads[p->thread_id];
3304 struct instruction *ip = t->ip;
3307 __instr_alu_add_hi_exec(p, t, ip);
3314 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3316 struct thread *t = &p->threads[p->thread_id];
3317 struct instruction *ip = t->ip;
3320 __instr_alu_sub_exec(p, t, ip);
3327 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3329 struct thread *t = &p->threads[p->thread_id];
3330 struct instruction *ip = t->ip;
3333 __instr_alu_sub_mh_exec(p, t, ip);
3340 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3342 struct thread *t = &p->threads[p->thread_id];
3343 struct instruction *ip = t->ip;
3346 __instr_alu_sub_hm_exec(p, t, ip);
3353 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3355 struct thread *t = &p->threads[p->thread_id];
3356 struct instruction *ip = t->ip;
3359 __instr_alu_sub_hh_exec(p, t, ip);
3366 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3368 struct thread *t = &p->threads[p->thread_id];
3369 struct instruction *ip = t->ip;
3372 __instr_alu_sub_mi_exec(p, t, ip);
3379 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3381 struct thread *t = &p->threads[p->thread_id];
3382 struct instruction *ip = t->ip;
3385 __instr_alu_sub_hi_exec(p, t, ip);
3392 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3394 struct thread *t = &p->threads[p->thread_id];
3395 struct instruction *ip = t->ip;
3398 __instr_alu_shl_exec(p, t, ip);
3405 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3407 struct thread *t = &p->threads[p->thread_id];
3408 struct instruction *ip = t->ip;
3411 __instr_alu_shl_mh_exec(p, t, ip);
3418 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3420 struct thread *t = &p->threads[p->thread_id];
3421 struct instruction *ip = t->ip;
3424 __instr_alu_shl_hm_exec(p, t, ip);
3431 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3433 struct thread *t = &p->threads[p->thread_id];
3434 struct instruction *ip = t->ip;
3437 __instr_alu_shl_hh_exec(p, t, ip);
3444 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3446 struct thread *t = &p->threads[p->thread_id];
3447 struct instruction *ip = t->ip;
3450 __instr_alu_shl_mi_exec(p, t, ip);
3457 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3459 struct thread *t = &p->threads[p->thread_id];
3460 struct instruction *ip = t->ip;
3463 __instr_alu_shl_hi_exec(p, t, ip);
3470 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3472 struct thread *t = &p->threads[p->thread_id];
3473 struct instruction *ip = t->ip;
3476 __instr_alu_shr_exec(p, t, ip);
3483 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3485 struct thread *t = &p->threads[p->thread_id];
3486 struct instruction *ip = t->ip;
3489 __instr_alu_shr_mh_exec(p, t, ip);
3496 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3498 struct thread *t = &p->threads[p->thread_id];
3499 struct instruction *ip = t->ip;
3502 __instr_alu_shr_hm_exec(p, t, ip);
3509 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3511 struct thread *t = &p->threads[p->thread_id];
3512 struct instruction *ip = t->ip;
3515 __instr_alu_shr_hh_exec(p, t, ip);
3522 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3524 struct thread *t = &p->threads[p->thread_id];
3525 struct instruction *ip = t->ip;
3528 __instr_alu_shr_mi_exec(p, t, ip);
3535 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3537 struct thread *t = &p->threads[p->thread_id];
3538 struct instruction *ip = t->ip;
3541 __instr_alu_shr_hi_exec(p, t, ip);
3548 instr_alu_and_exec(struct rte_swx_pipeline *p)
3550 struct thread *t = &p->threads[p->thread_id];
3551 struct instruction *ip = t->ip;
3554 __instr_alu_and_exec(p, t, ip);
3561 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3563 struct thread *t = &p->threads[p->thread_id];
3564 struct instruction *ip = t->ip;
3567 __instr_alu_and_mh_exec(p, t, ip);
3574 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3576 struct thread *t = &p->threads[p->thread_id];
3577 struct instruction *ip = t->ip;
3580 __instr_alu_and_hm_exec(p, t, ip);
3587 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3589 struct thread *t = &p->threads[p->thread_id];
3590 struct instruction *ip = t->ip;
3593 __instr_alu_and_hh_exec(p, t, ip);
3600 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3602 struct thread *t = &p->threads[p->thread_id];
3603 struct instruction *ip = t->ip;
3606 __instr_alu_and_i_exec(p, t, ip);
3613 instr_alu_or_exec(struct rte_swx_pipeline *p)
3615 struct thread *t = &p->threads[p->thread_id];
3616 struct instruction *ip = t->ip;
3619 __instr_alu_or_exec(p, t, ip);
3626 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3628 struct thread *t = &p->threads[p->thread_id];
3629 struct instruction *ip = t->ip;
3632 __instr_alu_or_mh_exec(p, t, ip);
3639 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3641 struct thread *t = &p->threads[p->thread_id];
3642 struct instruction *ip = t->ip;
3645 __instr_alu_or_hm_exec(p, t, ip);
3652 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3654 struct thread *t = &p->threads[p->thread_id];
3655 struct instruction *ip = t->ip;
3658 __instr_alu_or_hh_exec(p, t, ip);
3665 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3667 struct thread *t = &p->threads[p->thread_id];
3668 struct instruction *ip = t->ip;
3671 __instr_alu_or_i_exec(p, t, ip);
3678 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3680 struct thread *t = &p->threads[p->thread_id];
3681 struct instruction *ip = t->ip;
3684 __instr_alu_xor_exec(p, t, ip);
3691 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3693 struct thread *t = &p->threads[p->thread_id];
3694 struct instruction *ip = t->ip;
3697 __instr_alu_xor_mh_exec(p, t, ip);
3704 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3706 struct thread *t = &p->threads[p->thread_id];
3707 struct instruction *ip = t->ip;
3710 __instr_alu_xor_hm_exec(p, t, ip);
3717 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3719 struct thread *t = &p->threads[p->thread_id];
3720 struct instruction *ip = t->ip;
3723 __instr_alu_xor_hh_exec(p, t, ip);
3730 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3732 struct thread *t = &p->threads[p->thread_id];
3733 struct instruction *ip = t->ip;
3736 __instr_alu_xor_i_exec(p, t, ip);
3743 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3745 struct thread *t = &p->threads[p->thread_id];
3746 struct instruction *ip = t->ip;
3749 __instr_alu_ckadd_field_exec(p, t, ip);
3756 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3758 struct thread *t = &p->threads[p->thread_id];
3759 struct instruction *ip = t->ip;
3762 __instr_alu_cksub_field_exec(p, t, ip);
3769 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3771 struct thread *t = &p->threads[p->thread_id];
3772 struct instruction *ip = t->ip;
3775 __instr_alu_ckadd_struct20_exec(p, t, ip);
3782 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3784 struct thread *t = &p->threads[p->thread_id];
3785 struct instruction *ip = t->ip;
3788 __instr_alu_ckadd_struct_exec(p, t, ip);
3797 static struct regarray *
3798 regarray_find(struct rte_swx_pipeline *p, const char *name);
3801 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3802 struct action *action,
3805 struct instruction *instr,
3806 struct instruction_data *data __rte_unused)
3808 char *regarray = tokens[1], *idx = tokens[2];
3811 uint32_t idx_struct_id, idx_val;
3813 CHECK(n_tokens == 3, EINVAL);
3815 r = regarray_find(p, regarray);
3818 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3819 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3821 CHECK(!fidx->var_size, EINVAL);
3823 instr->type = INSTR_REGPREFETCH_RM;
3825 instr->type = INSTR_REGPREFETCH_RH;
3827 instr->regarray.regarray_id = r->id;
3828 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3829 instr->regarray.idx.n_bits = fidx->n_bits;
3830 instr->regarray.idx.offset = fidx->offset / 8;
3831 instr->regarray.dstsrc_val = 0; /* Unused. */
3835 /* REGPREFETCH_RI. */
3836 idx_val = strtoul(idx, &idx, 0);
3837 CHECK(!idx[0], EINVAL);
3839 instr->type = INSTR_REGPREFETCH_RI;
3840 instr->regarray.regarray_id = r->id;
3841 instr->regarray.idx_val = idx_val;
3842 instr->regarray.dstsrc_val = 0; /* Unused. */
3847 instr_regrd_translate(struct rte_swx_pipeline *p,
3848 struct action *action,
3851 struct instruction *instr,
3852 struct instruction_data *data __rte_unused)
3854 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3856 struct field *fdst, *fidx;
3857 uint32_t dst_struct_id, idx_struct_id, idx_val;
3859 CHECK(n_tokens == 4, EINVAL);
3861 r = regarray_find(p, regarray);
3864 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3865 CHECK(fdst, EINVAL);
3866 CHECK(!fdst->var_size, EINVAL);
3868 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3869 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3871 CHECK(!fidx->var_size, EINVAL);
3873 instr->type = INSTR_REGRD_MRM;
3874 if (dst[0] == 'h' && idx[0] != 'h')
3875 instr->type = INSTR_REGRD_HRM;
3876 if (dst[0] != 'h' && idx[0] == 'h')
3877 instr->type = INSTR_REGRD_MRH;
3878 if (dst[0] == 'h' && idx[0] == 'h')
3879 instr->type = INSTR_REGRD_HRH;
3881 instr->regarray.regarray_id = r->id;
3882 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3883 instr->regarray.idx.n_bits = fidx->n_bits;
3884 instr->regarray.idx.offset = fidx->offset / 8;
3885 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3886 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3887 instr->regarray.dstsrc.offset = fdst->offset / 8;
3891 /* REGRD_MRI, REGRD_HRI. */
3892 idx_val = strtoul(idx, &idx, 0);
3893 CHECK(!idx[0], EINVAL);
3895 instr->type = INSTR_REGRD_MRI;
3897 instr->type = INSTR_REGRD_HRI;
3899 instr->regarray.regarray_id = r->id;
3900 instr->regarray.idx_val = idx_val;
3901 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3902 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3903 instr->regarray.dstsrc.offset = fdst->offset / 8;
3908 instr_regwr_translate(struct rte_swx_pipeline *p,
3909 struct action *action,
3912 struct instruction *instr,
3913 struct instruction_data *data __rte_unused)
3915 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3917 struct field *fidx, *fsrc;
3919 uint32_t idx_struct_id, idx_val, src_struct_id;
3921 CHECK(n_tokens == 4, EINVAL);
3923 r = regarray_find(p, regarray);
3926 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3927 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3928 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3930 CHECK(!fidx->var_size, EINVAL);
3931 CHECK(!fsrc->var_size, EINVAL);
3933 instr->type = INSTR_REGWR_RMM;
3934 if (idx[0] == 'h' && src[0] != 'h')
3935 instr->type = INSTR_REGWR_RHM;
3936 if (idx[0] != 'h' && src[0] == 'h')
3937 instr->type = INSTR_REGWR_RMH;
3938 if (idx[0] == 'h' && src[0] == 'h')
3939 instr->type = INSTR_REGWR_RHH;
3941 instr->regarray.regarray_id = r->id;
3942 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3943 instr->regarray.idx.n_bits = fidx->n_bits;
3944 instr->regarray.idx.offset = fidx->offset / 8;
3945 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3946 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3947 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3951 /* REGWR_RHI, REGWR_RMI. */
3952 if (fidx && !fsrc) {
3953 CHECK(!fidx->var_size, EINVAL);
3955 src_val = strtoull(src, &src, 0);
3956 CHECK(!src[0], EINVAL);
3958 instr->type = INSTR_REGWR_RMI;
3960 instr->type = INSTR_REGWR_RHI;
3962 instr->regarray.regarray_id = r->id;
3963 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3964 instr->regarray.idx.n_bits = fidx->n_bits;
3965 instr->regarray.idx.offset = fidx->offset / 8;
3966 instr->regarray.dstsrc_val = src_val;
3970 /* REGWR_RIH, REGWR_RIM. */
3971 if (!fidx && fsrc) {
3972 idx_val = strtoul(idx, &idx, 0);
3973 CHECK(!idx[0], EINVAL);
3975 CHECK(!fsrc->var_size, EINVAL);
3977 instr->type = INSTR_REGWR_RIM;
3979 instr->type = INSTR_REGWR_RIH;
3981 instr->regarray.regarray_id = r->id;
3982 instr->regarray.idx_val = idx_val;
3983 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3984 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3985 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3990 src_val = strtoull(src, &src, 0);
3991 CHECK(!src[0], EINVAL);
3993 idx_val = strtoul(idx, &idx, 0);
3994 CHECK(!idx[0], EINVAL);
3996 instr->type = INSTR_REGWR_RII;
3997 instr->regarray.idx_val = idx_val;
3998 instr->regarray.dstsrc_val = src_val;
4004 instr_regadd_translate(struct rte_swx_pipeline *p,
4005 struct action *action,
4008 struct instruction *instr,
4009 struct instruction_data *data __rte_unused)
4011 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4013 struct field *fidx, *fsrc;
4015 uint32_t idx_struct_id, idx_val, src_struct_id;
4017 CHECK(n_tokens == 4, EINVAL);
4019 r = regarray_find(p, regarray);
4022 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
4023 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4024 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4026 CHECK(!fidx->var_size, EINVAL);
4027 CHECK(!fsrc->var_size, EINVAL);
4029 instr->type = INSTR_REGADD_RMM;
4030 if (idx[0] == 'h' && src[0] != 'h')
4031 instr->type = INSTR_REGADD_RHM;
4032 if (idx[0] != 'h' && src[0] == 'h')
4033 instr->type = INSTR_REGADD_RMH;
4034 if (idx[0] == 'h' && src[0] == 'h')
4035 instr->type = INSTR_REGADD_RHH;
4037 instr->regarray.regarray_id = r->id;
4038 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4039 instr->regarray.idx.n_bits = fidx->n_bits;
4040 instr->regarray.idx.offset = fidx->offset / 8;
4041 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4042 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4043 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4047 /* REGADD_RHI, REGADD_RMI. */
4048 if (fidx && !fsrc) {
4049 CHECK(!fidx->var_size, EINVAL);
4051 src_val = strtoull(src, &src, 0);
4052 CHECK(!src[0], EINVAL);
4054 instr->type = INSTR_REGADD_RMI;
4056 instr->type = INSTR_REGADD_RHI;
4058 instr->regarray.regarray_id = r->id;
4059 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4060 instr->regarray.idx.n_bits = fidx->n_bits;
4061 instr->regarray.idx.offset = fidx->offset / 8;
4062 instr->regarray.dstsrc_val = src_val;
4066 /* REGADD_RIH, REGADD_RIM. */
4067 if (!fidx && fsrc) {
4068 idx_val = strtoul(idx, &idx, 0);
4069 CHECK(!idx[0], EINVAL);
4071 CHECK(!fsrc->var_size, EINVAL);
4073 instr->type = INSTR_REGADD_RIM;
4075 instr->type = INSTR_REGADD_RIH;
4077 instr->regarray.regarray_id = r->id;
4078 instr->regarray.idx_val = idx_val;
4079 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4080 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4081 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4086 src_val = strtoull(src, &src, 0);
4087 CHECK(!src[0], EINVAL);
4089 idx_val = strtoul(idx, &idx, 0);
4090 CHECK(!idx[0], EINVAL);
4092 instr->type = INSTR_REGADD_RII;
4093 instr->regarray.idx_val = idx_val;
4094 instr->regarray.dstsrc_val = src_val;
4099 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4101 struct thread *t = &p->threads[p->thread_id];
4102 struct instruction *ip = t->ip;
4105 __instr_regprefetch_rh_exec(p, t, ip);
4112 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4114 struct thread *t = &p->threads[p->thread_id];
4115 struct instruction *ip = t->ip;
4118 __instr_regprefetch_rm_exec(p, t, ip);
4125 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4127 struct thread *t = &p->threads[p->thread_id];
4128 struct instruction *ip = t->ip;
4131 __instr_regprefetch_ri_exec(p, t, ip);
4138 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4140 struct thread *t = &p->threads[p->thread_id];
4141 struct instruction *ip = t->ip;
4144 __instr_regrd_hrh_exec(p, t, ip);
4151 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4153 struct thread *t = &p->threads[p->thread_id];
4154 struct instruction *ip = t->ip;
4157 __instr_regrd_hrm_exec(p, t, ip);
4164 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4166 struct thread *t = &p->threads[p->thread_id];
4167 struct instruction *ip = t->ip;
4170 __instr_regrd_mrh_exec(p, t, ip);
4177 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4179 struct thread *t = &p->threads[p->thread_id];
4180 struct instruction *ip = t->ip;
4183 __instr_regrd_mrm_exec(p, t, ip);
4190 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4192 struct thread *t = &p->threads[p->thread_id];
4193 struct instruction *ip = t->ip;
4196 __instr_regrd_hri_exec(p, t, ip);
4203 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4205 struct thread *t = &p->threads[p->thread_id];
4206 struct instruction *ip = t->ip;
4209 __instr_regrd_mri_exec(p, t, ip);
4216 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4218 struct thread *t = &p->threads[p->thread_id];
4219 struct instruction *ip = t->ip;
4222 __instr_regwr_rhh_exec(p, t, ip);
4229 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4231 struct thread *t = &p->threads[p->thread_id];
4232 struct instruction *ip = t->ip;
4235 __instr_regwr_rhm_exec(p, t, ip);
4242 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4244 struct thread *t = &p->threads[p->thread_id];
4245 struct instruction *ip = t->ip;
4248 __instr_regwr_rmh_exec(p, t, ip);
4255 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4257 struct thread *t = &p->threads[p->thread_id];
4258 struct instruction *ip = t->ip;
4261 __instr_regwr_rmm_exec(p, t, ip);
4268 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4270 struct thread *t = &p->threads[p->thread_id];
4271 struct instruction *ip = t->ip;
4274 __instr_regwr_rhi_exec(p, t, ip);
4281 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4283 struct thread *t = &p->threads[p->thread_id];
4284 struct instruction *ip = t->ip;
4287 __instr_regwr_rmi_exec(p, t, ip);
4294 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4296 struct thread *t = &p->threads[p->thread_id];
4297 struct instruction *ip = t->ip;
4300 __instr_regwr_rih_exec(p, t, ip);
4307 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4309 struct thread *t = &p->threads[p->thread_id];
4310 struct instruction *ip = t->ip;
4313 __instr_regwr_rim_exec(p, t, ip);
4320 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4322 struct thread *t = &p->threads[p->thread_id];
4323 struct instruction *ip = t->ip;
4326 __instr_regwr_rii_exec(p, t, ip);
4333 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4335 struct thread *t = &p->threads[p->thread_id];
4336 struct instruction *ip = t->ip;
4339 __instr_regadd_rhh_exec(p, t, ip);
4346 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4348 struct thread *t = &p->threads[p->thread_id];
4349 struct instruction *ip = t->ip;
4352 __instr_regadd_rhm_exec(p, t, ip);
4359 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4361 struct thread *t = &p->threads[p->thread_id];
4362 struct instruction *ip = t->ip;
4365 __instr_regadd_rmh_exec(p, t, ip);
4372 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4374 struct thread *t = &p->threads[p->thread_id];
4375 struct instruction *ip = t->ip;
4378 __instr_regadd_rmm_exec(p, t, ip);
4385 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4387 struct thread *t = &p->threads[p->thread_id];
4388 struct instruction *ip = t->ip;
4391 __instr_regadd_rhi_exec(p, t, ip);
4398 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4400 struct thread *t = &p->threads[p->thread_id];
4401 struct instruction *ip = t->ip;
4404 __instr_regadd_rmi_exec(p, t, ip);
4411 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4413 struct thread *t = &p->threads[p->thread_id];
4414 struct instruction *ip = t->ip;
4417 __instr_regadd_rih_exec(p, t, ip);
4424 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4426 struct thread *t = &p->threads[p->thread_id];
4427 struct instruction *ip = t->ip;
4430 __instr_regadd_rim_exec(p, t, ip);
4437 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4439 struct thread *t = &p->threads[p->thread_id];
4440 struct instruction *ip = t->ip;
4443 __instr_regadd_rii_exec(p, t, ip);
4452 static struct metarray *
4453 metarray_find(struct rte_swx_pipeline *p, const char *name);
4456 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4457 struct action *action,
4460 struct instruction *instr,
4461 struct instruction_data *data __rte_unused)
4463 char *metarray = tokens[1], *idx = tokens[2];
4466 uint32_t idx_struct_id, idx_val;
4468 CHECK(n_tokens == 3, EINVAL);
4470 m = metarray_find(p, metarray);
4473 /* METPREFETCH_H, METPREFETCH_M. */
4474 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4476 CHECK(!fidx->var_size, EINVAL);
4478 instr->type = INSTR_METPREFETCH_M;
4480 instr->type = INSTR_METPREFETCH_H;
4482 instr->meter.metarray_id = m->id;
4483 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4484 instr->meter.idx.n_bits = fidx->n_bits;
4485 instr->meter.idx.offset = fidx->offset / 8;
4489 /* METPREFETCH_I. */
4490 idx_val = strtoul(idx, &idx, 0);
4491 CHECK(!idx[0], EINVAL);
4493 instr->type = INSTR_METPREFETCH_I;
4494 instr->meter.metarray_id = m->id;
4495 instr->meter.idx_val = idx_val;
4500 instr_meter_translate(struct rte_swx_pipeline *p,
4501 struct action *action,
4504 struct instruction *instr,
4505 struct instruction_data *data __rte_unused)
4507 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4508 char *color_in = tokens[4], *color_out = tokens[5];
4510 struct field *fidx, *flength, *fcin, *fcout;
4511 uint32_t idx_struct_id, length_struct_id;
4512 uint32_t color_in_struct_id, color_out_struct_id;
4514 CHECK(n_tokens == 6, EINVAL);
4516 m = metarray_find(p, metarray);
4519 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4521 flength = struct_field_parse(p, action, length, &length_struct_id);
4522 CHECK(flength, EINVAL);
4523 CHECK(!flength->var_size, EINVAL);
4525 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4527 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4528 CHECK(fcout, EINVAL);
4529 CHECK(!fcout->var_size, EINVAL);
4531 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4533 CHECK(!fidx->var_size, EINVAL);
4534 CHECK(!fcin->var_size, EINVAL);
4536 instr->type = INSTR_METER_MMM;
4537 if (idx[0] == 'h' && length[0] == 'h')
4538 instr->type = INSTR_METER_HHM;
4539 if (idx[0] == 'h' && length[0] != 'h')
4540 instr->type = INSTR_METER_HMM;
4541 if (idx[0] != 'h' && length[0] == 'h')
4542 instr->type = INSTR_METER_MHM;
4544 instr->meter.metarray_id = m->id;
4546 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4547 instr->meter.idx.n_bits = fidx->n_bits;
4548 instr->meter.idx.offset = fidx->offset / 8;
4550 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4551 instr->meter.length.n_bits = flength->n_bits;
4552 instr->meter.length.offset = flength->offset / 8;
4554 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4555 instr->meter.color_in.n_bits = fcin->n_bits;
4556 instr->meter.color_in.offset = fcin->offset / 8;
4558 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4559 instr->meter.color_out.n_bits = fcout->n_bits;
4560 instr->meter.color_out.offset = fcout->offset / 8;
4563 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4564 if (fidx && !fcin) {
4565 uint32_t color_in_val;
4567 CHECK(!fidx->var_size, EINVAL);
4569 color_in_val = strtoul(color_in, &color_in, 0);
4570 CHECK(!color_in[0], EINVAL);
4572 instr->type = INSTR_METER_MMI;
4573 if (idx[0] == 'h' && length[0] == 'h')
4574 instr->type = INSTR_METER_HHI;
4575 if (idx[0] == 'h' && length[0] != 'h')
4576 instr->type = INSTR_METER_HMI;
4577 if (idx[0] != 'h' && length[0] == 'h')
4578 instr->type = INSTR_METER_MHI;
4580 instr->meter.metarray_id = m->id;
4582 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4583 instr->meter.idx.n_bits = fidx->n_bits;
4584 instr->meter.idx.offset = fidx->offset / 8;
4586 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4587 instr->meter.length.n_bits = flength->n_bits;
4588 instr->meter.length.offset = flength->offset / 8;
4590 instr->meter.color_in_val = color_in_val;
4592 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4593 instr->meter.color_out.n_bits = fcout->n_bits;
4594 instr->meter.color_out.offset = fcout->offset / 8;
4597 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4598 if (!fidx && fcin) {
4601 idx_val = strtoul(idx, &idx, 0);
4602 CHECK(!idx[0], EINVAL);
4604 CHECK(!fcin->var_size, EINVAL);
4606 instr->type = INSTR_METER_IMM;
4607 if (length[0] == 'h')
4608 instr->type = INSTR_METER_IHM;
4610 instr->meter.metarray_id = m->id;
4612 instr->meter.idx_val = idx_val;
4614 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4615 instr->meter.length.n_bits = flength->n_bits;
4616 instr->meter.length.offset = flength->offset / 8;
4618 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4619 instr->meter.color_in.n_bits = fcin->n_bits;
4620 instr->meter.color_in.offset = fcin->offset / 8;
4622 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4623 instr->meter.color_out.n_bits = fcout->n_bits;
4624 instr->meter.color_out.offset = fcout->offset / 8;
4627 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4628 if (!fidx && !fcin) {
4629 uint32_t idx_val, color_in_val;
4631 idx_val = strtoul(idx, &idx, 0);
4632 CHECK(!idx[0], EINVAL);
4634 color_in_val = strtoul(color_in, &color_in, 0);
4635 CHECK(!color_in[0], EINVAL);
4637 instr->type = INSTR_METER_IMI;
4638 if (length[0] == 'h')
4639 instr->type = INSTR_METER_IHI;
4641 instr->meter.metarray_id = m->id;
4643 instr->meter.idx_val = idx_val;
4645 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4646 instr->meter.length.n_bits = flength->n_bits;
4647 instr->meter.length.offset = flength->offset / 8;
4649 instr->meter.color_in_val = color_in_val;
4651 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4652 instr->meter.color_out.n_bits = fcout->n_bits;
4653 instr->meter.color_out.offset = fcout->offset / 8;
4660 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4662 struct thread *t = &p->threads[p->thread_id];
4663 struct instruction *ip = t->ip;
4666 __instr_metprefetch_h_exec(p, t, ip);
4673 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4675 struct thread *t = &p->threads[p->thread_id];
4676 struct instruction *ip = t->ip;
4679 __instr_metprefetch_m_exec(p, t, ip);
4686 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4688 struct thread *t = &p->threads[p->thread_id];
4689 struct instruction *ip = t->ip;
4692 __instr_metprefetch_i_exec(p, t, ip);
4699 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4701 struct thread *t = &p->threads[p->thread_id];
4702 struct instruction *ip = t->ip;
4705 __instr_meter_hhm_exec(p, t, ip);
4712 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4714 struct thread *t = &p->threads[p->thread_id];
4715 struct instruction *ip = t->ip;
4718 __instr_meter_hhi_exec(p, t, ip);
4725 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4727 struct thread *t = &p->threads[p->thread_id];
4728 struct instruction *ip = t->ip;
4731 __instr_meter_hmm_exec(p, t, ip);
4738 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4740 struct thread *t = &p->threads[p->thread_id];
4741 struct instruction *ip = t->ip;
4744 __instr_meter_hmi_exec(p, t, ip);
4751 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4753 struct thread *t = &p->threads[p->thread_id];
4754 struct instruction *ip = t->ip;
4757 __instr_meter_mhm_exec(p, t, ip);
4764 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4766 struct thread *t = &p->threads[p->thread_id];
4767 struct instruction *ip = t->ip;
4770 __instr_meter_mhi_exec(p, t, ip);
4777 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4779 struct thread *t = &p->threads[p->thread_id];
4780 struct instruction *ip = t->ip;
4783 __instr_meter_mmm_exec(p, t, ip);
4790 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4792 struct thread *t = &p->threads[p->thread_id];
4793 struct instruction *ip = t->ip;
4796 __instr_meter_mmi_exec(p, t, ip);
4803 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4805 struct thread *t = &p->threads[p->thread_id];
4806 struct instruction *ip = t->ip;
4809 __instr_meter_ihm_exec(p, t, ip);
4816 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4818 struct thread *t = &p->threads[p->thread_id];
4819 struct instruction *ip = t->ip;
4822 __instr_meter_ihi_exec(p, t, ip);
4829 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4831 struct thread *t = &p->threads[p->thread_id];
4832 struct instruction *ip = t->ip;
4835 __instr_meter_imm_exec(p, t, ip);
4842 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4844 struct thread *t = &p->threads[p->thread_id];
4845 struct instruction *ip = t->ip;
4848 __instr_meter_imi_exec(p, t, ip);
4858 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4859 struct action *action __rte_unused,
4862 struct instruction *instr,
4863 struct instruction_data *data)
4865 CHECK(n_tokens == 2, EINVAL);
4867 strcpy(data->jmp_label, tokens[1]);
4869 instr->type = INSTR_JMP;
4870 instr->jmp.ip = NULL; /* Resolved later. */
4875 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4876 struct action *action __rte_unused,
4879 struct instruction *instr,
4880 struct instruction_data *data)
4884 CHECK(n_tokens == 3, EINVAL);
4886 strcpy(data->jmp_label, tokens[1]);
4888 h = header_parse(p, tokens[2]);
4891 instr->type = INSTR_JMP_VALID;
4892 instr->jmp.ip = NULL; /* Resolved later. */
4893 instr->jmp.header_id = h->id;
4898 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4899 struct action *action __rte_unused,
4902 struct instruction *instr,
4903 struct instruction_data *data)
4907 CHECK(n_tokens == 3, EINVAL);
4909 strcpy(data->jmp_label, tokens[1]);
4911 h = header_parse(p, tokens[2]);
4914 instr->type = INSTR_JMP_INVALID;
4915 instr->jmp.ip = NULL; /* Resolved later. */
4916 instr->jmp.header_id = h->id;
4921 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4922 struct action *action,
4925 struct instruction *instr,
4926 struct instruction_data *data)
4928 CHECK(!action, EINVAL);
4929 CHECK(n_tokens == 2, EINVAL);
4931 strcpy(data->jmp_label, tokens[1]);
4933 instr->type = INSTR_JMP_HIT;
4934 instr->jmp.ip = NULL; /* Resolved later. */
4939 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4940 struct action *action,
4943 struct instruction *instr,
4944 struct instruction_data *data)
4946 CHECK(!action, EINVAL);
4947 CHECK(n_tokens == 2, EINVAL);
4949 strcpy(data->jmp_label, tokens[1]);
4951 instr->type = INSTR_JMP_MISS;
4952 instr->jmp.ip = NULL; /* Resolved later. */
4957 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4958 struct action *action,
4961 struct instruction *instr,
4962 struct instruction_data *data)
4966 CHECK(!action, EINVAL);
4967 CHECK(n_tokens == 3, EINVAL);
4969 strcpy(data->jmp_label, tokens[1]);
4971 a = action_find(p, tokens[2]);
4974 instr->type = INSTR_JMP_ACTION_HIT;
4975 instr->jmp.ip = NULL; /* Resolved later. */
4976 instr->jmp.action_id = a->id;
4981 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4982 struct action *action,
4985 struct instruction *instr,
4986 struct instruction_data *data)
4990 CHECK(!action, EINVAL);
4991 CHECK(n_tokens == 3, EINVAL);
4993 strcpy(data->jmp_label, tokens[1]);
4995 a = action_find(p, tokens[2]);
4998 instr->type = INSTR_JMP_ACTION_MISS;
4999 instr->jmp.ip = NULL; /* Resolved later. */
5000 instr->jmp.action_id = a->id;
5005 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
5006 struct action *action,
5009 struct instruction *instr,
5010 struct instruction_data *data)
5012 char *a = tokens[2], *b = tokens[3];
5013 struct field *fa, *fb;
5015 uint32_t a_struct_id, b_struct_id;
5017 CHECK(n_tokens == 4, EINVAL);
5019 strcpy(data->jmp_label, tokens[1]);
5021 fa = struct_field_parse(p, action, a, &a_struct_id);
5023 CHECK(!fa->var_size, EINVAL);
5025 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
5026 fb = struct_field_parse(p, action, b, &b_struct_id);
5028 CHECK(!fb->var_size, EINVAL);
5030 instr->type = INSTR_JMP_EQ;
5031 if (a[0] != 'h' && b[0] == 'h')
5032 instr->type = INSTR_JMP_EQ_MH;
5033 if (a[0] == 'h' && b[0] != 'h')
5034 instr->type = INSTR_JMP_EQ_HM;
5035 if (a[0] == 'h' && b[0] == 'h')
5036 instr->type = INSTR_JMP_EQ_HH;
5037 instr->jmp.ip = NULL; /* Resolved later. */
5039 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5040 instr->jmp.a.n_bits = fa->n_bits;
5041 instr->jmp.a.offset = fa->offset / 8;
5042 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5043 instr->jmp.b.n_bits = fb->n_bits;
5044 instr->jmp.b.offset = fb->offset / 8;
5049 b_val = strtoull(b, &b, 0);
5050 CHECK(!b[0], EINVAL);
5053 b_val = hton64(b_val) >> (64 - fa->n_bits);
5055 instr->type = INSTR_JMP_EQ_I;
5056 instr->jmp.ip = NULL; /* Resolved later. */
5057 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5058 instr->jmp.a.n_bits = fa->n_bits;
5059 instr->jmp.a.offset = fa->offset / 8;
5060 instr->jmp.b_val = b_val;
5065 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
5066 struct action *action,
5069 struct instruction *instr,
5070 struct instruction_data *data)
5072 char *a = tokens[2], *b = tokens[3];
5073 struct field *fa, *fb;
5075 uint32_t a_struct_id, b_struct_id;
5077 CHECK(n_tokens == 4, EINVAL);
5079 strcpy(data->jmp_label, tokens[1]);
5081 fa = struct_field_parse(p, action, a, &a_struct_id);
5083 CHECK(!fa->var_size, EINVAL);
5085 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5086 fb = struct_field_parse(p, action, b, &b_struct_id);
5088 CHECK(!fb->var_size, EINVAL);
5090 instr->type = INSTR_JMP_NEQ;
5091 if (a[0] != 'h' && b[0] == 'h')
5092 instr->type = INSTR_JMP_NEQ_MH;
5093 if (a[0] == 'h' && b[0] != 'h')
5094 instr->type = INSTR_JMP_NEQ_HM;
5095 if (a[0] == 'h' && b[0] == 'h')
5096 instr->type = INSTR_JMP_NEQ_HH;
5097 instr->jmp.ip = NULL; /* Resolved later. */
5099 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5100 instr->jmp.a.n_bits = fa->n_bits;
5101 instr->jmp.a.offset = fa->offset / 8;
5102 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5103 instr->jmp.b.n_bits = fb->n_bits;
5104 instr->jmp.b.offset = fb->offset / 8;
5109 b_val = strtoull(b, &b, 0);
5110 CHECK(!b[0], EINVAL);
5113 b_val = hton64(b_val) >> (64 - fa->n_bits);
5115 instr->type = INSTR_JMP_NEQ_I;
5116 instr->jmp.ip = NULL; /* Resolved later. */
5117 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5118 instr->jmp.a.n_bits = fa->n_bits;
5119 instr->jmp.a.offset = fa->offset / 8;
5120 instr->jmp.b_val = b_val;
5125 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5126 struct action *action,
5129 struct instruction *instr,
5130 struct instruction_data *data)
5132 char *a = tokens[2], *b = tokens[3];
5133 struct field *fa, *fb;
5135 uint32_t a_struct_id, b_struct_id;
5137 CHECK(n_tokens == 4, EINVAL);
5139 strcpy(data->jmp_label, tokens[1]);
5141 fa = struct_field_parse(p, action, a, &a_struct_id);
5143 CHECK(!fa->var_size, EINVAL);
5145 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5146 fb = struct_field_parse(p, action, b, &b_struct_id);
5148 CHECK(!fb->var_size, EINVAL);
5150 instr->type = INSTR_JMP_LT;
5151 if (a[0] == 'h' && b[0] != 'h')
5152 instr->type = INSTR_JMP_LT_HM;
5153 if (a[0] != 'h' && b[0] == 'h')
5154 instr->type = INSTR_JMP_LT_MH;
5155 if (a[0] == 'h' && b[0] == 'h')
5156 instr->type = INSTR_JMP_LT_HH;
5157 instr->jmp.ip = NULL; /* Resolved later. */
5159 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5160 instr->jmp.a.n_bits = fa->n_bits;
5161 instr->jmp.a.offset = fa->offset / 8;
5162 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5163 instr->jmp.b.n_bits = fb->n_bits;
5164 instr->jmp.b.offset = fb->offset / 8;
5168 /* JMP_LT_MI, JMP_LT_HI. */
5169 b_val = strtoull(b, &b, 0);
5170 CHECK(!b[0], EINVAL);
5172 instr->type = INSTR_JMP_LT_MI;
5174 instr->type = INSTR_JMP_LT_HI;
5175 instr->jmp.ip = NULL; /* Resolved later. */
5177 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5178 instr->jmp.a.n_bits = fa->n_bits;
5179 instr->jmp.a.offset = fa->offset / 8;
5180 instr->jmp.b_val = b_val;
5185 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5186 struct action *action,
5189 struct instruction *instr,
5190 struct instruction_data *data)
5192 char *a = tokens[2], *b = tokens[3];
5193 struct field *fa, *fb;
5195 uint32_t a_struct_id, b_struct_id;
5197 CHECK(n_tokens == 4, EINVAL);
5199 strcpy(data->jmp_label, tokens[1]);
5201 fa = struct_field_parse(p, action, a, &a_struct_id);
5203 CHECK(!fa->var_size, EINVAL);
5205 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5206 fb = struct_field_parse(p, action, b, &b_struct_id);
5208 CHECK(!fb->var_size, EINVAL);
5210 instr->type = INSTR_JMP_GT;
5211 if (a[0] == 'h' && b[0] != 'h')
5212 instr->type = INSTR_JMP_GT_HM;
5213 if (a[0] != 'h' && b[0] == 'h')
5214 instr->type = INSTR_JMP_GT_MH;
5215 if (a[0] == 'h' && b[0] == 'h')
5216 instr->type = INSTR_JMP_GT_HH;
5217 instr->jmp.ip = NULL; /* Resolved later. */
5219 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5220 instr->jmp.a.n_bits = fa->n_bits;
5221 instr->jmp.a.offset = fa->offset / 8;
5222 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5223 instr->jmp.b.n_bits = fb->n_bits;
5224 instr->jmp.b.offset = fb->offset / 8;
5228 /* JMP_GT_MI, JMP_GT_HI. */
5229 b_val = strtoull(b, &b, 0);
5230 CHECK(!b[0], EINVAL);
5232 instr->type = INSTR_JMP_GT_MI;
5234 instr->type = INSTR_JMP_GT_HI;
5235 instr->jmp.ip = NULL; /* Resolved later. */
5237 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5238 instr->jmp.a.n_bits = fa->n_bits;
5239 instr->jmp.a.offset = fa->offset / 8;
5240 instr->jmp.b_val = b_val;
5245 instr_jmp_exec(struct rte_swx_pipeline *p)
5247 struct thread *t = &p->threads[p->thread_id];
5248 struct instruction *ip = t->ip;
5250 TRACE("[Thread %2u] jmp\n", p->thread_id);
5252 thread_ip_set(t, ip->jmp.ip);
5256 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5258 struct thread *t = &p->threads[p->thread_id];
5259 struct instruction *ip = t->ip;
5260 uint32_t header_id = ip->jmp.header_id;
5262 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5264 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5268 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5270 struct thread *t = &p->threads[p->thread_id];
5271 struct instruction *ip = t->ip;
5272 uint32_t header_id = ip->jmp.header_id;
5274 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5276 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5280 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5282 struct thread *t = &p->threads[p->thread_id];
5283 struct instruction *ip = t->ip;
5284 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5286 TRACE("[Thread %2u] jmph\n", p->thread_id);
5288 t->ip = ip_next[t->hit];
5292 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5294 struct thread *t = &p->threads[p->thread_id];
5295 struct instruction *ip = t->ip;
5296 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5298 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5300 t->ip = ip_next[t->hit];
5304 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5306 struct thread *t = &p->threads[p->thread_id];
5307 struct instruction *ip = t->ip;
5309 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5311 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5315 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5317 struct thread *t = &p->threads[p->thread_id];
5318 struct instruction *ip = t->ip;
5320 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5322 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5326 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5328 struct thread *t = &p->threads[p->thread_id];
5329 struct instruction *ip = t->ip;
5331 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5337 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5339 struct thread *t = &p->threads[p->thread_id];
5340 struct instruction *ip = t->ip;
5342 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5344 JMP_CMP_MH(t, ip, ==);
5348 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5350 struct thread *t = &p->threads[p->thread_id];
5351 struct instruction *ip = t->ip;
5353 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5355 JMP_CMP_HM(t, ip, ==);
5359 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5361 struct thread *t = &p->threads[p->thread_id];
5362 struct instruction *ip = t->ip;
5364 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5366 JMP_CMP_HH_FAST(t, ip, ==);
5370 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5372 struct thread *t = &p->threads[p->thread_id];
5373 struct instruction *ip = t->ip;
5375 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5377 JMP_CMP_I(t, ip, ==);
5381 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5383 struct thread *t = &p->threads[p->thread_id];
5384 struct instruction *ip = t->ip;
5386 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5392 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5394 struct thread *t = &p->threads[p->thread_id];
5395 struct instruction *ip = t->ip;
5397 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5399 JMP_CMP_MH(t, ip, !=);
5403 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5405 struct thread *t = &p->threads[p->thread_id];
5406 struct instruction *ip = t->ip;
5408 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5410 JMP_CMP_HM(t, ip, !=);
5414 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5416 struct thread *t = &p->threads[p->thread_id];
5417 struct instruction *ip = t->ip;
5419 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5421 JMP_CMP_HH_FAST(t, ip, !=);
5425 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5427 struct thread *t = &p->threads[p->thread_id];
5428 struct instruction *ip = t->ip;
5430 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5432 JMP_CMP_I(t, ip, !=);
5436 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5438 struct thread *t = &p->threads[p->thread_id];
5439 struct instruction *ip = t->ip;
5441 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5447 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5449 struct thread *t = &p->threads[p->thread_id];
5450 struct instruction *ip = t->ip;
5452 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5454 JMP_CMP_MH(t, ip, <);
5458 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5460 struct thread *t = &p->threads[p->thread_id];
5461 struct instruction *ip = t->ip;
5463 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5465 JMP_CMP_HM(t, ip, <);
5469 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5471 struct thread *t = &p->threads[p->thread_id];
5472 struct instruction *ip = t->ip;
5474 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5476 JMP_CMP_HH(t, ip, <);
5480 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5482 struct thread *t = &p->threads[p->thread_id];
5483 struct instruction *ip = t->ip;
5485 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5487 JMP_CMP_MI(t, ip, <);
5491 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5493 struct thread *t = &p->threads[p->thread_id];
5494 struct instruction *ip = t->ip;
5496 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5498 JMP_CMP_HI(t, ip, <);
5502 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5504 struct thread *t = &p->threads[p->thread_id];
5505 struct instruction *ip = t->ip;
5507 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5513 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5515 struct thread *t = &p->threads[p->thread_id];
5516 struct instruction *ip = t->ip;
5518 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5520 JMP_CMP_MH(t, ip, >);
5524 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5526 struct thread *t = &p->threads[p->thread_id];
5527 struct instruction *ip = t->ip;
5529 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5531 JMP_CMP_HM(t, ip, >);
5535 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5537 struct thread *t = &p->threads[p->thread_id];
5538 struct instruction *ip = t->ip;
5540 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5542 JMP_CMP_HH(t, ip, >);
5546 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5548 struct thread *t = &p->threads[p->thread_id];
5549 struct instruction *ip = t->ip;
5551 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5553 JMP_CMP_MI(t, ip, >);
5557 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5559 struct thread *t = &p->threads[p->thread_id];
5560 struct instruction *ip = t->ip;
5562 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5564 JMP_CMP_HI(t, ip, >);
5571 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5572 struct action *action,
5573 char **tokens __rte_unused,
5575 struct instruction *instr,
5576 struct instruction_data *data __rte_unused)
5578 CHECK(action, EINVAL);
5579 CHECK(n_tokens == 1, EINVAL);
5581 instr->type = INSTR_RETURN;
5586 instr_return_exec(struct rte_swx_pipeline *p)
5588 struct thread *t = &p->threads[p->thread_id];
5590 TRACE("[Thread %2u] return\n", p->thread_id);
5596 instr_translate(struct rte_swx_pipeline *p,
5597 struct action *action,
5599 struct instruction *instr,
5600 struct instruction_data *data)
5602 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5603 int n_tokens = 0, tpos = 0;
5605 /* Parse the instruction string into tokens. */
5609 token = strtok_r(string, " \t\v", &string);
5613 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5614 CHECK_NAME(token, EINVAL);
5616 tokens[n_tokens] = token;
5620 CHECK(n_tokens, EINVAL);
5622 /* Handle the optional instruction label. */
5623 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5624 strcpy(data->label, tokens[0]);
5627 CHECK(n_tokens - tpos, EINVAL);
5630 /* Identify the instruction type. */
5631 if (!strcmp(tokens[tpos], "rx"))
5632 return instr_rx_translate(p,
5639 if (!strcmp(tokens[tpos], "tx"))
5640 return instr_tx_translate(p,
5647 if (!strcmp(tokens[tpos], "drop"))
5648 return instr_drop_translate(p,
5655 if (!strcmp(tokens[tpos], "extract"))
5656 return instr_hdr_extract_translate(p,
5663 if (!strcmp(tokens[tpos], "lookahead"))
5664 return instr_hdr_lookahead_translate(p,
5671 if (!strcmp(tokens[tpos], "emit"))
5672 return instr_hdr_emit_translate(p,
5679 if (!strcmp(tokens[tpos], "validate"))
5680 return instr_hdr_validate_translate(p,
5687 if (!strcmp(tokens[tpos], "invalidate"))
5688 return instr_hdr_invalidate_translate(p,
5695 if (!strcmp(tokens[tpos], "mov"))
5696 return instr_mov_translate(p,
5703 if (!strcmp(tokens[tpos], "add"))
5704 return instr_alu_add_translate(p,
5711 if (!strcmp(tokens[tpos], "sub"))
5712 return instr_alu_sub_translate(p,
5719 if (!strcmp(tokens[tpos], "ckadd"))
5720 return instr_alu_ckadd_translate(p,
5727 if (!strcmp(tokens[tpos], "cksub"))
5728 return instr_alu_cksub_translate(p,
5735 if (!strcmp(tokens[tpos], "and"))
5736 return instr_alu_and_translate(p,
5743 if (!strcmp(tokens[tpos], "or"))
5744 return instr_alu_or_translate(p,
5751 if (!strcmp(tokens[tpos], "xor"))
5752 return instr_alu_xor_translate(p,
5759 if (!strcmp(tokens[tpos], "shl"))
5760 return instr_alu_shl_translate(p,
5767 if (!strcmp(tokens[tpos], "shr"))
5768 return instr_alu_shr_translate(p,
5775 if (!strcmp(tokens[tpos], "regprefetch"))
5776 return instr_regprefetch_translate(p,
5783 if (!strcmp(tokens[tpos], "regrd"))
5784 return instr_regrd_translate(p,
5791 if (!strcmp(tokens[tpos], "regwr"))
5792 return instr_regwr_translate(p,
5799 if (!strcmp(tokens[tpos], "regadd"))
5800 return instr_regadd_translate(p,
5807 if (!strcmp(tokens[tpos], "metprefetch"))
5808 return instr_metprefetch_translate(p,
5815 if (!strcmp(tokens[tpos], "meter"))
5816 return instr_meter_translate(p,
5823 if (!strcmp(tokens[tpos], "table"))
5824 return instr_table_translate(p,
5831 if (!strcmp(tokens[tpos], "learn"))
5832 return instr_learn_translate(p,
5839 if (!strcmp(tokens[tpos], "forget"))
5840 return instr_forget_translate(p,
5847 if (!strcmp(tokens[tpos], "extern"))
5848 return instr_extern_translate(p,
5855 if (!strcmp(tokens[tpos], "jmp"))
5856 return instr_jmp_translate(p,
5863 if (!strcmp(tokens[tpos], "jmpv"))
5864 return instr_jmp_valid_translate(p,
5871 if (!strcmp(tokens[tpos], "jmpnv"))
5872 return instr_jmp_invalid_translate(p,
5879 if (!strcmp(tokens[tpos], "jmph"))
5880 return instr_jmp_hit_translate(p,
5887 if (!strcmp(tokens[tpos], "jmpnh"))
5888 return instr_jmp_miss_translate(p,
5895 if (!strcmp(tokens[tpos], "jmpa"))
5896 return instr_jmp_action_hit_translate(p,
5903 if (!strcmp(tokens[tpos], "jmpna"))
5904 return instr_jmp_action_miss_translate(p,
5911 if (!strcmp(tokens[tpos], "jmpeq"))
5912 return instr_jmp_eq_translate(p,
5919 if (!strcmp(tokens[tpos], "jmpneq"))
5920 return instr_jmp_neq_translate(p,
5927 if (!strcmp(tokens[tpos], "jmplt"))
5928 return instr_jmp_lt_translate(p,
5935 if (!strcmp(tokens[tpos], "jmpgt"))
5936 return instr_jmp_gt_translate(p,
5943 if (!strcmp(tokens[tpos], "return"))
5944 return instr_return_translate(p,
5954 static struct instruction_data *
5955 label_find(struct instruction_data *data, uint32_t n, const char *label)
5959 for (i = 0; i < n; i++)
5960 if (!strcmp(label, data[i].label))
5967 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5969 uint32_t count = 0, i;
5974 for (i = 0; i < n; i++)
5975 if (!strcmp(label, data[i].jmp_label))
5982 instr_label_check(struct instruction_data *instruction_data,
5983 uint32_t n_instructions)
5987 /* Check that all instruction labels are unique. */
5988 for (i = 0; i < n_instructions; i++) {
5989 struct instruction_data *data = &instruction_data[i];
5990 char *label = data->label;
5996 for (j = i + 1; j < n_instructions; j++)
5997 CHECK(strcmp(label, instruction_data[j].label), EINVAL);
6000 /* Get users for each instruction label. */
6001 for (i = 0; i < n_instructions; i++) {
6002 struct instruction_data *data = &instruction_data[i];
6003 char *label = data->label;
6005 data->n_users = label_is_used(instruction_data,
6014 instr_jmp_resolve(struct instruction *instructions,
6015 struct instruction_data *instruction_data,
6016 uint32_t n_instructions)
6020 for (i = 0; i < n_instructions; i++) {
6021 struct instruction *instr = &instructions[i];
6022 struct instruction_data *data = &instruction_data[i];
6023 struct instruction_data *found;
6025 if (!instruction_is_jmp(instr))
6028 found = label_find(instruction_data,
6031 CHECK(found, EINVAL);
6033 instr->jmp.ip = &instructions[found - instruction_data];
6040 instr_verify(struct rte_swx_pipeline *p __rte_unused,
6042 struct instruction *instr,
6043 struct instruction_data *data __rte_unused,
6044 uint32_t n_instructions)
6047 enum instruction_type type;
6050 /* Check that the first instruction is rx. */
6051 CHECK(instr[0].type == INSTR_RX, EINVAL);
6053 /* Check that there is at least one tx instruction. */
6054 for (i = 0; i < n_instructions; i++) {
6055 type = instr[i].type;
6057 if (instruction_is_tx(type))
6060 CHECK(i < n_instructions, EINVAL);
6062 /* Check that the last instruction is either tx or unconditional
6065 type = instr[n_instructions - 1].type;
6066 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6070 enum instruction_type type;
6073 /* Check that there is at least one return or tx instruction. */
6074 for (i = 0; i < n_instructions; i++) {
6075 type = instr[i].type;
6077 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6080 CHECK(i < n_instructions, EINVAL);
6087 instr_compact(struct instruction *instructions,
6088 struct instruction_data *instruction_data,
6089 uint32_t n_instructions)
6091 uint32_t i, pos = 0;
6093 /* Eliminate the invalid instructions that have been optimized out. */
6094 for (i = 0; i < n_instructions; i++) {
6095 struct instruction *instr = &instructions[i];
6096 struct instruction_data *data = &instruction_data[i];
6102 memcpy(&instructions[pos], instr, sizeof(*instr));
6103 memcpy(&instruction_data[pos], data, sizeof(*data));
6113 instr_pattern_extract_many_search(struct instruction *instr,
6114 struct instruction_data *data,
6116 uint32_t *n_pattern_instr)
6120 for (i = 0; i < n_instr; i++) {
6121 if (data[i].invalid)
6124 if (instr[i].type != INSTR_HDR_EXTRACT)
6127 if (i == RTE_DIM(instr->io.hdr.header_id))
6130 if (i && data[i].n_users)
6137 *n_pattern_instr = i;
6142 instr_pattern_extract_many_replace(struct instruction *instr,
6143 struct instruction_data *data,
6148 for (i = 1; i < n_instr; i++) {
6150 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6151 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6152 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6154 data[i].invalid = 1;
6159 instr_pattern_extract_many_optimize(struct instruction *instructions,
6160 struct instruction_data *instruction_data,
6161 uint32_t n_instructions)
6165 for (i = 0; i < n_instructions; ) {
6166 struct instruction *instr = &instructions[i];
6167 struct instruction_data *data = &instruction_data[i];
6168 uint32_t n_instr = 0;
6172 detected = instr_pattern_extract_many_search(instr,
6177 instr_pattern_extract_many_replace(instr,
6184 /* No pattern starting at the current instruction. */
6188 /* Eliminate the invalid instructions that have been optimized out. */
6189 n_instructions = instr_compact(instructions,
6193 return n_instructions;
6197 instr_pattern_emit_many_tx_search(struct instruction *instr,
6198 struct instruction_data *data,
6200 uint32_t *n_pattern_instr)
6204 for (i = 0; i < n_instr; i++) {
6205 if (data[i].invalid)
6208 if (instr[i].type != INSTR_HDR_EMIT)
6211 if (i == RTE_DIM(instr->io.hdr.header_id))
6214 if (i && data[i].n_users)
6221 if (instr[i].type != INSTR_TX)
6224 if (data[i].n_users)
6229 *n_pattern_instr = i;
6234 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6235 struct instruction_data *data,
6240 /* Any emit instruction in addition to the first one. */
6241 for (i = 1; i < n_instr - 1; i++) {
6243 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6244 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6245 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6247 data[i].invalid = 1;
6250 /* The TX instruction is the last one in the pattern. */
6252 instr[0].io.io.offset = instr[i].io.io.offset;
6253 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6254 data[i].invalid = 1;
6258 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6259 struct instruction_data *instruction_data,
6260 uint32_t n_instructions)
6264 for (i = 0; i < n_instructions; ) {
6265 struct instruction *instr = &instructions[i];
6266 struct instruction_data *data = &instruction_data[i];
6267 uint32_t n_instr = 0;
6270 /* Emit many + TX. */
6271 detected = instr_pattern_emit_many_tx_search(instr,
6276 instr_pattern_emit_many_tx_replace(instr,
6283 /* No pattern starting at the current instruction. */
6287 /* Eliminate the invalid instructions that have been optimized out. */
6288 n_instructions = instr_compact(instructions,
6292 return n_instructions;
6296 action_arg_src_mov_count(struct action *a,
6298 struct instruction *instructions,
6299 struct instruction_data *instruction_data,
6300 uint32_t n_instructions);
6303 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6305 struct instruction *instr,
6306 struct instruction_data *data,
6308 struct instruction *instructions,
6309 struct instruction_data *instruction_data,
6310 uint32_t n_instructions,
6311 uint32_t *n_pattern_instr)
6314 uint32_t src_field_id, i, j;
6316 /* Prerequisites. */
6320 /* First instruction: MOV_HM. */
6321 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6324 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6325 if (!h || h->st->var_size)
6328 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6329 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6332 if (src_field_id == a->st->n_fields)
6335 if (instr[0].mov.dst.offset ||
6336 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6337 instr[0].mov.src.struct_id ||
6338 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6339 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6342 if ((n_instr < h->st->n_fields + 1) ||
6343 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6346 /* Subsequent instructions: MOV_HM. */
6347 for (i = 1; i < h->st->n_fields; i++)
6348 if (data[i].invalid ||
6350 (instr[i].type != INSTR_MOV_HM) ||
6351 (instr[i].mov.dst.struct_id != h->struct_id) ||
6352 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6353 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6354 instr[i].mov.src.struct_id ||
6355 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6356 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6357 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6360 /* Last instruction: HDR_VALIDATE. */
6361 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6362 (instr[i].valid.header_id != h->id))
6365 /* Check that none of the action args that are used as source for this
6366 * DMA transfer are not used as source in any other mov instruction.
6368 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6371 n_users = action_arg_src_mov_count(a,
6380 *n_pattern_instr = 1 + i;
6385 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6387 struct instruction *instr,
6388 struct instruction_data *data,
6392 uint32_t src_field_id, src_offset, i;
6394 /* Read from the instructions before they are modified. */
6395 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6399 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6400 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6403 if (src_field_id == a->st->n_fields)
6406 src_offset = instr[0].mov.src.offset;
6408 /* Modify the instructions. */
6409 instr[0].type = INSTR_DMA_HT;
6410 instr[0].dma.dst.header_id[0] = h->id;
6411 instr[0].dma.dst.struct_id[0] = h->struct_id;
6412 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6413 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6415 for (i = 1; i < n_instr; i++)
6416 data[i].invalid = 1;
6418 /* Update the endianness of the action arguments to header endianness. */
6419 for (i = 0; i < h->st->n_fields; i++)
6420 a->args_endianness[src_field_id + i] = 1;
6424 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6426 struct instruction *instructions,
6427 struct instruction_data *instruction_data,
6428 uint32_t n_instructions)
6433 return n_instructions;
6435 for (i = 0; i < n_instructions; ) {
6436 struct instruction *instr = &instructions[i];
6437 struct instruction_data *data = &instruction_data[i];
6438 uint32_t n_instr = 0;
6441 /* Mov all + validate. */
6442 detected = instr_pattern_mov_all_validate_search(p,
6452 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6457 /* No pattern starting at the current instruction. */
6461 /* Eliminate the invalid instructions that have been optimized out. */
6462 n_instructions = instr_compact(instructions,
6466 return n_instructions;
6470 instr_pattern_dma_many_search(struct instruction *instr,
6471 struct instruction_data *data,
6473 uint32_t *n_pattern_instr)
6477 for (i = 0; i < n_instr; i++) {
6478 if (data[i].invalid)
6481 if (instr[i].type != INSTR_DMA_HT)
6484 if (i == RTE_DIM(instr->dma.dst.header_id))
6487 if (i && data[i].n_users)
6494 *n_pattern_instr = i;
6499 instr_pattern_dma_many_replace(struct instruction *instr,
6500 struct instruction_data *data,
6505 for (i = 1; i < n_instr; i++) {
6507 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6508 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6509 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6510 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6512 data[i].invalid = 1;
6517 instr_pattern_dma_many_optimize(struct instruction *instructions,
6518 struct instruction_data *instruction_data,
6519 uint32_t n_instructions)
6523 for (i = 0; i < n_instructions; ) {
6524 struct instruction *instr = &instructions[i];
6525 struct instruction_data *data = &instruction_data[i];
6526 uint32_t n_instr = 0;
6530 detected = instr_pattern_dma_many_search(instr,
6535 instr_pattern_dma_many_replace(instr, data, n_instr);
6540 /* No pattern starting at the current instruction. */
6544 /* Eliminate the invalid instructions that have been optimized out. */
6545 n_instructions = instr_compact(instructions,
6549 return n_instructions;
6553 instr_optimize(struct rte_swx_pipeline *p,
6555 struct instruction *instructions,
6556 struct instruction_data *instruction_data,
6557 uint32_t n_instructions)
6560 n_instructions = instr_pattern_extract_many_optimize(instructions,
6564 /* Emit many + TX. */
6565 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6569 /* Mov all + validate. */
6570 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6577 n_instructions = instr_pattern_dma_many_optimize(instructions,
6581 return n_instructions;
6585 instruction_config(struct rte_swx_pipeline *p,
6587 const char **instructions,
6588 uint32_t n_instructions)
6590 struct instruction *instr = NULL;
6591 struct instruction_data *data = NULL;
6595 CHECK(n_instructions, EINVAL);
6596 CHECK(instructions, EINVAL);
6597 for (i = 0; i < n_instructions; i++)
6598 CHECK_INSTRUCTION(instructions[i], EINVAL);
6600 /* Memory allocation. */
6601 instr = calloc(n_instructions, sizeof(struct instruction));
6607 data = calloc(n_instructions, sizeof(struct instruction_data));
6613 for (i = 0; i < n_instructions; i++) {
6614 char *string = strdup(instructions[i]);
6620 err = instr_translate(p, a, string, &instr[i], &data[i]);
6629 err = instr_label_check(data, n_instructions);
6633 err = instr_verify(p, a, instr, data, n_instructions);
6637 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6639 err = instr_jmp_resolve(instr, data, n_instructions);
6644 a->instructions = instr;
6645 a->instruction_data = data;
6646 a->n_instructions = n_instructions;
6648 p->instructions = instr;
6649 p->instruction_data = data;
6650 p->n_instructions = n_instructions;
6661 static instr_exec_t instruction_table[] = {
6662 [INSTR_RX] = instr_rx_exec,
6663 [INSTR_TX] = instr_tx_exec,
6664 [INSTR_TX_I] = instr_tx_i_exec,
6665 [INSTR_DROP] = instr_drop_exec,
6667 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6668 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6669 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6670 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6671 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6672 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6673 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6674 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6675 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6676 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6678 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6679 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6680 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6681 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6682 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6683 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6684 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6685 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6686 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6688 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6689 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6691 [INSTR_MOV] = instr_mov_exec,
6692 [INSTR_MOV_MH] = instr_mov_mh_exec,
6693 [INSTR_MOV_HM] = instr_mov_hm_exec,
6694 [INSTR_MOV_HH] = instr_mov_hh_exec,
6695 [INSTR_MOV_I] = instr_mov_i_exec,
6697 [INSTR_DMA_HT] = instr_dma_ht_exec,
6698 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6699 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6700 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6701 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6702 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6703 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6704 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6706 [INSTR_ALU_ADD] = instr_alu_add_exec,
6707 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6708 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6709 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6710 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6711 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6713 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6714 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6715 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6716 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6717 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6718 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6720 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6721 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6722 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6723 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6725 [INSTR_ALU_AND] = instr_alu_and_exec,
6726 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6727 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6728 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6729 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6731 [INSTR_ALU_OR] = instr_alu_or_exec,
6732 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6733 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6734 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6735 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6737 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6738 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6739 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6740 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6741 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6743 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6744 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6745 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6746 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6747 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6748 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6750 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6751 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6752 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6753 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6754 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6755 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6757 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6758 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6759 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6761 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6762 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6763 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6764 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6765 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6766 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6768 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6769 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6770 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6771 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6772 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6773 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6774 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6775 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6776 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6778 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6779 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6780 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6781 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6782 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6783 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6784 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6785 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6786 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6788 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6789 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6790 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6792 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6793 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6794 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6795 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6796 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6797 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6798 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6799 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6800 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6801 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6802 [INSTR_METER_IMM] = instr_meter_imm_exec,
6803 [INSTR_METER_IMI] = instr_meter_imi_exec,
6805 [INSTR_TABLE] = instr_table_exec,
6806 [INSTR_TABLE_AF] = instr_table_af_exec,
6807 [INSTR_SELECTOR] = instr_selector_exec,
6808 [INSTR_LEARNER] = instr_learner_exec,
6809 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6810 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6811 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6812 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6813 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6815 [INSTR_JMP] = instr_jmp_exec,
6816 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6817 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6818 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6819 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6820 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6821 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6823 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6824 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6825 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6826 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6827 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6829 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6830 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6831 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6832 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6833 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6835 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6836 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6837 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6838 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6839 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6840 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6842 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6843 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6844 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6845 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6846 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6847 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6849 [INSTR_RETURN] = instr_return_exec,
6853 instruction_table_build(struct rte_swx_pipeline *p)
6855 p->instruction_table = calloc(RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX,
6856 sizeof(struct instr_exec_t *));
6857 if (!p->instruction_table)
6860 memcpy(p->instruction_table, instruction_table, sizeof(instruction_table));
6866 instruction_table_build_free(struct rte_swx_pipeline *p)
6868 if (!p->instruction_table)
6871 free(p->instruction_table);
6872 p->instruction_table = NULL;
6876 instruction_table_free(struct rte_swx_pipeline *p)
6878 instruction_table_build_free(p);
6882 instr_exec(struct rte_swx_pipeline *p)
6884 struct thread *t = &p->threads[p->thread_id];
6885 struct instruction *ip = t->ip;
6886 instr_exec_t instr = p->instruction_table[ip->type];
6894 static struct action *
6895 action_find(struct rte_swx_pipeline *p, const char *name)
6897 struct action *elem;
6902 TAILQ_FOREACH(elem, &p->actions, node)
6903 if (strcmp(elem->name, name) == 0)
6909 static struct action *
6910 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6912 struct action *action = NULL;
6914 TAILQ_FOREACH(action, &p->actions, node)
6915 if (action->id == id)
6921 static struct field *
6922 action_field_find(struct action *a, const char *name)
6924 return a->st ? struct_type_field_find(a->st, name) : NULL;
6927 static struct field *
6928 action_field_parse(struct action *action, const char *name)
6930 if (name[0] != 't' || name[1] != '.')
6933 return action_field_find(action, &name[2]);
6937 action_has_nbo_args(struct action *a)
6941 /* Return if the action does not have any args. */
6943 return 0; /* FALSE */
6945 for (i = 0; i < a->st->n_fields; i++)
6946 if (a->args_endianness[i])
6947 return 1; /* TRUE */
6949 return 0; /* FALSE */
6953 action_does_learning(struct action *a)
6957 for (i = 0; i < a->n_instructions; i++)
6958 switch (a->instructions[i].type) {
6959 case INSTR_LEARNER_LEARN:
6960 return 1; /* TRUE */
6962 case INSTR_LEARNER_FORGET:
6963 return 1; /* TRUE */
6969 return 0; /* FALSE */
6973 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6975 const char *args_struct_type_name,
6976 const char **instructions,
6977 uint32_t n_instructions)
6979 struct struct_type *args_struct_type = NULL;
6985 CHECK_NAME(name, EINVAL);
6986 CHECK(!action_find(p, name), EEXIST);
6988 if (args_struct_type_name) {
6989 CHECK_NAME(args_struct_type_name, EINVAL);
6990 args_struct_type = struct_type_find(p, args_struct_type_name);
6991 CHECK(args_struct_type, EINVAL);
6992 CHECK(!args_struct_type->var_size, EINVAL);
6995 /* Node allocation. */
6996 a = calloc(1, sizeof(struct action));
6998 if (args_struct_type) {
6999 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
7000 if (!a->args_endianness) {
7006 /* Node initialization. */
7007 strcpy(a->name, name);
7008 a->st = args_struct_type;
7009 a->id = p->n_actions;
7011 /* Instruction translation. */
7012 err = instruction_config(p, a, instructions, n_instructions);
7014 free(a->args_endianness);
7019 /* Node add to tailq. */
7020 TAILQ_INSERT_TAIL(&p->actions, a, node);
7027 action_build(struct rte_swx_pipeline *p)
7029 struct action *action;
7031 /* p->action_instructions. */
7032 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
7033 CHECK(p->action_instructions, ENOMEM);
7035 TAILQ_FOREACH(action, &p->actions, node)
7036 p->action_instructions[action->id] = action->instructions;
7038 /* p->action_funcs. */
7039 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
7040 CHECK(p->action_funcs, ENOMEM);
7046 action_build_free(struct rte_swx_pipeline *p)
7048 free(p->action_funcs);
7049 p->action_funcs = NULL;
7051 free(p->action_instructions);
7052 p->action_instructions = NULL;
7056 action_free(struct rte_swx_pipeline *p)
7058 action_build_free(p);
7061 struct action *action;
7063 action = TAILQ_FIRST(&p->actions);
7067 TAILQ_REMOVE(&p->actions, action, node);
7068 free(action->instruction_data);
7069 free(action->instructions);
7075 action_arg_src_mov_count(struct action *a,
7077 struct instruction *instructions,
7078 struct instruction_data *instruction_data,
7079 uint32_t n_instructions)
7081 uint32_t offset, n_users = 0, i;
7084 (arg_id >= a->st->n_fields) ||
7086 !instruction_data ||
7090 offset = a->st->fields[arg_id].offset / 8;
7092 for (i = 0; i < n_instructions; i++) {
7093 struct instruction *instr = &instructions[i];
7094 struct instruction_data *data = &instruction_data[i];
7096 if (data->invalid ||
7097 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7098 instr->mov.src.struct_id ||
7099 (instr->mov.src.offset != offset))
7111 static struct table_type *
7112 table_type_find(struct rte_swx_pipeline *p, const char *name)
7114 struct table_type *elem;
7116 TAILQ_FOREACH(elem, &p->table_types, node)
7117 if (strcmp(elem->name, name) == 0)
7123 static struct table_type *
7124 table_type_resolve(struct rte_swx_pipeline *p,
7125 const char *recommended_type_name,
7126 enum rte_swx_table_match_type match_type)
7128 struct table_type *elem;
7130 /* Only consider the recommended type if the match type is correct. */
7131 if (recommended_type_name)
7132 TAILQ_FOREACH(elem, &p->table_types, node)
7133 if (!strcmp(elem->name, recommended_type_name) &&
7134 (elem->match_type == match_type))
7137 /* Ignore the recommended type and get the first element with this match
7140 TAILQ_FOREACH(elem, &p->table_types, node)
7141 if (elem->match_type == match_type)
7147 static struct table *
7148 table_find(struct rte_swx_pipeline *p, const char *name)
7152 TAILQ_FOREACH(elem, &p->tables, node)
7153 if (strcmp(elem->name, name) == 0)
7159 static struct table *
7160 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7162 struct table *table = NULL;
7164 TAILQ_FOREACH(table, &p->tables, node)
7165 if (table->id == id)
7172 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7174 enum rte_swx_table_match_type match_type,
7175 struct rte_swx_table_ops *ops)
7177 struct table_type *elem;
7181 CHECK_NAME(name, EINVAL);
7182 CHECK(!table_type_find(p, name), EEXIST);
7185 CHECK(ops->create, EINVAL);
7186 CHECK(ops->lkp, EINVAL);
7187 CHECK(ops->free, EINVAL);
7189 /* Node allocation. */
7190 elem = calloc(1, sizeof(struct table_type));
7191 CHECK(elem, ENOMEM);
7193 /* Node initialization. */
7194 strcpy(elem->name, name);
7195 elem->match_type = match_type;
7196 memcpy(&elem->ops, ops, sizeof(*ops));
7198 /* Node add to tailq. */
7199 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7205 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7207 enum rte_swx_table_match_type *match_type)
7209 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7211 for (i = 0; i < n_fields; i++) {
7212 struct rte_swx_match_field_params *f = &fields[i];
7214 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7217 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7221 if ((n_fields_lpm > 1) ||
7222 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7225 *match_type = (n_fields_em == n_fields) ?
7226 RTE_SWX_TABLE_MATCH_EXACT :
7227 RTE_SWX_TABLE_MATCH_WILDCARD;
7233 table_match_fields_check(struct rte_swx_pipeline *p,
7234 struct rte_swx_pipeline_table_params *params,
7235 struct header **header)
7237 struct header *h0 = NULL;
7238 struct field *hf, *mf;
7239 uint32_t *offset = NULL, i;
7242 /* Return if no match fields. */
7243 if (!params->n_fields) {
7244 if (params->fields) {
7255 /* Memory allocation. */
7256 offset = calloc(params->n_fields, sizeof(uint32_t));
7262 /* Check that all the match fields belong to either the same header or
7265 hf = header_field_parse(p, params->fields[0].name, &h0);
7266 mf = metadata_field_parse(p, params->fields[0].name);
7267 if ((!hf && !mf) || (hf && hf->var_size)) {
7272 offset[0] = h0 ? hf->offset : mf->offset;
7274 for (i = 1; i < params->n_fields; i++)
7278 hf = header_field_parse(p, params->fields[i].name, &h);
7279 if (!hf || (h->id != h0->id) || hf->var_size) {
7284 offset[i] = hf->offset;
7286 mf = metadata_field_parse(p, params->fields[i].name);
7292 offset[i] = mf->offset;
7295 /* Check that there are no duplicated match fields. */
7296 for (i = 0; i < params->n_fields; i++) {
7299 for (j = 0; j < i; j++)
7300 if (offset[j] == offset[i]) {
7316 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7318 struct rte_swx_pipeline_table_params *params,
7319 const char *recommended_table_type_name,
7323 struct table_type *type;
7324 struct table *t = NULL;
7325 struct action *default_action;
7326 struct header *header = NULL;
7327 uint32_t action_data_size_max = 0, i;
7332 CHECK_NAME(name, EINVAL);
7333 CHECK(!table_find(p, name), EEXIST);
7334 CHECK(!selector_find(p, name), EEXIST);
7335 CHECK(!learner_find(p, name), EEXIST);
7337 CHECK(params, EINVAL);
7340 status = table_match_fields_check(p, params, &header);
7344 /* Action checks. */
7345 CHECK(params->n_actions, EINVAL);
7346 CHECK(params->action_names, EINVAL);
7347 for (i = 0; i < params->n_actions; i++) {
7348 const char *action_name = params->action_names[i];
7350 uint32_t action_data_size;
7351 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7353 CHECK_NAME(action_name, EINVAL);
7355 a = action_find(p, action_name);
7357 CHECK(!action_does_learning(a), EINVAL);
7359 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7360 if (action_data_size > action_data_size_max)
7361 action_data_size_max = action_data_size;
7363 if (params->action_is_for_table_entries)
7364 action_is_for_table_entries = params->action_is_for_table_entries[i];
7365 if (params->action_is_for_default_entry)
7366 action_is_for_default_entry = params->action_is_for_default_entry[i];
7367 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
7370 CHECK_NAME(params->default_action_name, EINVAL);
7371 for (i = 0; i < p->n_actions; i++)
7372 if (!strcmp(params->action_names[i],
7373 params->default_action_name))
7375 CHECK(i < params->n_actions, EINVAL);
7376 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
7379 default_action = action_find(p, params->default_action_name);
7380 CHECK((default_action->st && params->default_action_data) ||
7381 !params->default_action_data, EINVAL);
7383 /* Table type checks. */
7384 if (recommended_table_type_name)
7385 CHECK_NAME(recommended_table_type_name, EINVAL);
7387 if (params->n_fields) {
7388 enum rte_swx_table_match_type match_type;
7390 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7394 type = table_type_resolve(p, recommended_table_type_name, match_type);
7395 CHECK(type, EINVAL);
7400 /* Memory allocation. */
7401 t = calloc(1, sizeof(struct table));
7405 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7409 t->actions = calloc(params->n_actions, sizeof(struct action *));
7413 if (action_data_size_max) {
7414 t->default_action_data = calloc(1, action_data_size_max);
7415 if (!t->default_action_data)
7419 t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
7420 if (!t->action_is_for_table_entries)
7423 t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
7424 if (!t->action_is_for_default_entry)
7427 /* Node initialization. */
7428 strcpy(t->name, name);
7429 if (args && args[0])
7430 strcpy(t->args, args);
7433 for (i = 0; i < params->n_fields; i++) {
7434 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7435 struct match_field *f = &t->fields[i];
7437 f->match_type = field->match_type;
7439 header_field_parse(p, field->name, NULL) :
7440 metadata_field_parse(p, field->name);
7442 t->n_fields = params->n_fields;
7445 for (i = 0; i < params->n_actions; i++) {
7446 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
7448 if (params->action_is_for_table_entries)
7449 action_is_for_table_entries = params->action_is_for_table_entries[i];
7450 if (params->action_is_for_default_entry)
7451 action_is_for_default_entry = params->action_is_for_default_entry[i];
7453 t->actions[i] = action_find(p, params->action_names[i]);
7454 t->action_is_for_table_entries[i] = action_is_for_table_entries;
7455 t->action_is_for_default_entry[i] = action_is_for_default_entry;
7457 t->default_action = default_action;
7458 if (default_action->st)
7459 memcpy(t->default_action_data,
7460 params->default_action_data,
7461 default_action->st->n_bits / 8);
7462 t->n_actions = params->n_actions;
7463 t->default_action_is_const = params->default_action_is_const;
7464 t->action_data_size_max = action_data_size_max;
7467 t->id = p->n_tables;
7469 /* Node add to tailq. */
7470 TAILQ_INSERT_TAIL(&p->tables, t, node);
7479 free(t->action_is_for_default_entry);
7480 free(t->action_is_for_table_entries);
7481 free(t->default_action_data);
7489 static struct rte_swx_table_params *
7490 table_params_get(struct table *table)
7492 struct rte_swx_table_params *params;
7493 struct field *first, *last;
7495 uint32_t key_size, key_offset, action_data_size, i;
7497 /* Memory allocation. */
7498 params = calloc(1, sizeof(struct rte_swx_table_params));
7502 /* Find first (smallest offset) and last (biggest offset) match fields. */
7503 first = table->fields[0].field;
7504 last = table->fields[0].field;
7506 for (i = 0; i < table->n_fields; i++) {
7507 struct field *f = table->fields[i].field;
7509 if (f->offset < first->offset)
7512 if (f->offset > last->offset)
7516 /* Key offset and size. */
7517 key_offset = first->offset / 8;
7518 key_size = (last->offset + last->n_bits - first->offset) / 8;
7520 /* Memory allocation. */
7521 key_mask = calloc(1, key_size);
7528 for (i = 0; i < table->n_fields; i++) {
7529 struct field *f = table->fields[i].field;
7530 uint32_t start = (f->offset - first->offset) / 8;
7531 size_t size = f->n_bits / 8;
7533 memset(&key_mask[start], 0xFF, size);
7536 /* Action data size. */
7537 action_data_size = 0;
7538 for (i = 0; i < table->n_actions; i++) {
7539 struct action *action = table->actions[i];
7540 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7542 if (ads > action_data_size)
7543 action_data_size = ads;
7547 params->match_type = table->type->match_type;
7548 params->key_size = key_size;
7549 params->key_offset = key_offset;
7550 params->key_mask0 = key_mask;
7551 params->action_data_size = action_data_size;
7552 params->n_keys_max = table->size;
7558 table_params_free(struct rte_swx_table_params *params)
7563 free(params->key_mask0);
7568 table_stub_lkp(void *table __rte_unused,
7569 void *mailbox __rte_unused,
7570 uint8_t **key __rte_unused,
7571 uint64_t *action_id __rte_unused,
7572 uint8_t **action_data __rte_unused,
7576 return 1; /* DONE. */
7580 table_build(struct rte_swx_pipeline *p)
7584 /* Per pipeline: table statistics. */
7585 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7586 CHECK(p->table_stats, ENOMEM);
7588 for (i = 0; i < p->n_tables; i++) {
7589 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7590 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7593 /* Per thread: table runt-time. */
7594 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7595 struct thread *t = &p->threads[i];
7596 struct table *table;
7598 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7599 CHECK(t->tables, ENOMEM);
7601 TAILQ_FOREACH(table, &p->tables, node) {
7602 struct table_runtime *r = &t->tables[table->id];
7607 size = table->type->ops.mailbox_size_get();
7610 r->func = table->type->ops.lkp;
7614 r->mailbox = calloc(1, size);
7615 CHECK(r->mailbox, ENOMEM);
7619 r->key = table->header ?
7620 &t->structs[table->header->struct_id] :
7621 &t->structs[p->metadata_struct_id];
7623 r->func = table_stub_lkp;
7632 table_build_free(struct rte_swx_pipeline *p)
7636 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7637 struct thread *t = &p->threads[i];
7643 for (j = 0; j < p->n_tables; j++) {
7644 struct table_runtime *r = &t->tables[j];
7653 if (p->table_stats) {
7654 for (i = 0; i < p->n_tables; i++)
7655 free(p->table_stats[i].n_pkts_action);
7657 free(p->table_stats);
7662 table_free(struct rte_swx_pipeline *p)
7664 table_build_free(p);
7670 elem = TAILQ_FIRST(&p->tables);
7674 TAILQ_REMOVE(&p->tables, elem, node);
7676 free(elem->actions);
7677 free(elem->default_action_data);
7683 struct table_type *elem;
7685 elem = TAILQ_FIRST(&p->table_types);
7689 TAILQ_REMOVE(&p->table_types, elem, node);
7697 static struct selector *
7698 selector_find(struct rte_swx_pipeline *p, const char *name)
7702 TAILQ_FOREACH(s, &p->selectors, node)
7703 if (strcmp(s->name, name) == 0)
7709 static struct selector *
7710 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7712 struct selector *s = NULL;
7714 TAILQ_FOREACH(s, &p->selectors, node)
7722 selector_fields_check(struct rte_swx_pipeline *p,
7723 struct rte_swx_pipeline_selector_params *params,
7724 struct header **header)
7726 struct header *h0 = NULL;
7727 struct field *hf, *mf;
7730 /* Return if no selector fields. */
7731 if (!params->n_selector_fields || !params->selector_field_names)
7734 /* Check that all the selector fields either belong to the same header
7735 * or are all meta-data fields.
7737 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7738 mf = metadata_field_parse(p, params->selector_field_names[0]);
7742 for (i = 1; i < params->n_selector_fields; i++)
7746 hf = header_field_parse(p, params->selector_field_names[i], &h);
7747 if (!hf || (h->id != h0->id))
7750 mf = metadata_field_parse(p, params->selector_field_names[i]);
7755 /* Check that there are no duplicated match fields. */
7756 for (i = 0; i < params->n_selector_fields; i++) {
7757 const char *field_name = params->selector_field_names[i];
7760 for (j = i + 1; j < params->n_selector_fields; j++)
7761 if (!strcmp(params->selector_field_names[j], field_name))
7773 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7775 struct rte_swx_pipeline_selector_params *params)
7778 struct header *selector_header = NULL;
7779 struct field *group_id_field, *member_id_field;
7785 CHECK_NAME(name, EINVAL);
7786 CHECK(!table_find(p, name), EEXIST);
7787 CHECK(!selector_find(p, name), EEXIST);
7788 CHECK(!learner_find(p, name), EEXIST);
7790 CHECK(params, EINVAL);
7792 CHECK_NAME(params->group_id_field_name, EINVAL);
7793 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7794 CHECK(group_id_field, EINVAL);
7796 for (i = 0; i < params->n_selector_fields; i++) {
7797 const char *field_name = params->selector_field_names[i];
7799 CHECK_NAME(field_name, EINVAL);
7801 status = selector_fields_check(p, params, &selector_header);
7805 CHECK_NAME(params->member_id_field_name, EINVAL);
7806 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7807 CHECK(member_id_field, EINVAL);
7809 CHECK(params->n_groups_max, EINVAL);
7811 CHECK(params->n_members_per_group_max, EINVAL);
7813 /* Memory allocation. */
7814 s = calloc(1, sizeof(struct selector));
7820 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7821 if (!s->selector_fields) {
7826 /* Node initialization. */
7827 strcpy(s->name, name);
7829 s->group_id_field = group_id_field;
7831 for (i = 0; i < params->n_selector_fields; i++) {
7832 const char *field_name = params->selector_field_names[i];
7834 s->selector_fields[i] = selector_header ?
7835 header_field_parse(p, field_name, NULL) :
7836 metadata_field_parse(p, field_name);
7839 s->n_selector_fields = params->n_selector_fields;
7841 s->selector_header = selector_header;
7843 s->member_id_field = member_id_field;
7845 s->n_groups_max = params->n_groups_max;
7847 s->n_members_per_group_max = params->n_members_per_group_max;
7849 s->id = p->n_selectors;
7851 /* Node add to tailq. */
7852 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7861 free(s->selector_fields);
7869 selector_params_free(struct rte_swx_table_selector_params *params)
7874 free(params->selector_mask);
7879 static struct rte_swx_table_selector_params *
7880 selector_table_params_get(struct selector *s)
7882 struct rte_swx_table_selector_params *params = NULL;
7883 struct field *first, *last;
7886 /* Memory allocation. */
7887 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7892 params->group_id_offset = s->group_id_field->offset / 8;
7894 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7895 first = s->selector_fields[0];
7896 last = s->selector_fields[0];
7898 for (i = 0; i < s->n_selector_fields; i++) {
7899 struct field *f = s->selector_fields[i];
7901 if (f->offset < first->offset)
7904 if (f->offset > last->offset)
7908 /* Selector offset and size. */
7909 params->selector_offset = first->offset / 8;
7910 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7912 /* Memory allocation. */
7913 params->selector_mask = calloc(1, params->selector_size);
7914 if (!params->selector_mask)
7917 /* Selector mask. */
7918 for (i = 0; i < s->n_selector_fields; i++) {
7919 struct field *f = s->selector_fields[i];
7920 uint32_t start = (f->offset - first->offset) / 8;
7921 size_t size = f->n_bits / 8;
7923 memset(¶ms->selector_mask[start], 0xFF, size);
7927 params->member_id_offset = s->member_id_field->offset / 8;
7929 /* Maximum number of groups. */
7930 params->n_groups_max = s->n_groups_max;
7932 /* Maximum number of members per group. */
7933 params->n_members_per_group_max = s->n_members_per_group_max;
7938 selector_params_free(params);
7943 selector_build_free(struct rte_swx_pipeline *p)
7947 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7948 struct thread *t = &p->threads[i];
7954 for (j = 0; j < p->n_selectors; j++) {
7955 struct selector_runtime *r = &t->selectors[j];
7961 t->selectors = NULL;
7964 free(p->selector_stats);
7965 p->selector_stats = NULL;
7969 selector_build(struct rte_swx_pipeline *p)
7974 /* Per pipeline: selector statistics. */
7975 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7976 if (!p->selector_stats) {
7981 /* Per thread: selector run-time. */
7982 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7983 struct thread *t = &p->threads[i];
7986 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7987 if (!t->selectors) {
7992 TAILQ_FOREACH(s, &p->selectors, node) {
7993 struct selector_runtime *r = &t->selectors[s->id];
7997 size = rte_swx_table_selector_mailbox_size_get();
7999 r->mailbox = calloc(1, size);
8006 /* r->group_id_buffer. */
8007 r->group_id_buffer = &t->structs[p->metadata_struct_id];
8009 /* r->selector_buffer. */
8010 r->selector_buffer = s->selector_header ?
8011 &t->structs[s->selector_header->struct_id] :
8012 &t->structs[p->metadata_struct_id];
8014 /* r->member_id_buffer. */
8015 r->member_id_buffer = &t->structs[p->metadata_struct_id];
8022 selector_build_free(p);
8027 selector_free(struct rte_swx_pipeline *p)
8029 selector_build_free(p);
8031 /* Selector tables. */
8033 struct selector *elem;
8035 elem = TAILQ_FIRST(&p->selectors);
8039 TAILQ_REMOVE(&p->selectors, elem, node);
8040 free(elem->selector_fields);
8048 static struct learner *
8049 learner_find(struct rte_swx_pipeline *p, const char *name)
8053 TAILQ_FOREACH(l, &p->learners, node)
8054 if (!strcmp(l->name, name))
8060 static struct learner *
8061 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8063 struct learner *l = NULL;
8065 TAILQ_FOREACH(l, &p->learners, node)
8073 learner_match_fields_check(struct rte_swx_pipeline *p,
8074 struct rte_swx_pipeline_learner_params *params,
8075 struct header **header)
8077 struct header *h0 = NULL;
8078 struct field *hf, *mf;
8081 /* Return if no match fields. */
8082 if (!params->n_fields || !params->field_names)
8085 /* Check that all the match fields either belong to the same header
8086 * or are all meta-data fields.
8088 hf = header_field_parse(p, params->field_names[0], &h0);
8089 mf = metadata_field_parse(p, params->field_names[0]);
8093 for (i = 1; i < params->n_fields; i++)
8097 hf = header_field_parse(p, params->field_names[i], &h);
8098 if (!hf || (h->id != h0->id))
8101 mf = metadata_field_parse(p, params->field_names[i]);
8106 /* Check that there are no duplicated match fields. */
8107 for (i = 0; i < params->n_fields; i++) {
8108 const char *field_name = params->field_names[i];
8111 for (j = i + 1; j < params->n_fields; j++)
8112 if (!strcmp(params->field_names[j], field_name))
8124 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
8126 struct struct_type *mst = p->metadata_st, *ast = a->st;
8127 struct field *mf, *af;
8137 /* Check that mf_name is the name of a valid meta-data field. */
8138 CHECK_NAME(mf_name, EINVAL);
8139 mf = metadata_field_parse(p, mf_name);
8142 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8143 * all the action arguments.
8145 mf_pos = mf - mst->fields;
8146 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8148 /* Check that the size of each of the identified meta-data fields matches exactly the size
8149 * of the corresponding action argument.
8151 for (i = 0; i < ast->n_fields; i++) {
8152 mf = &mst->fields[mf_pos + i];
8153 af = &ast->fields[i];
8155 CHECK(mf->n_bits == af->n_bits, EINVAL);
8162 learner_action_learning_check(struct rte_swx_pipeline *p,
8163 struct action *action,
8164 const char **action_names,
8169 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8170 * the action passed as argument to the "learn" instruction) is also enabled for the
8171 * current learner table.
8173 for (i = 0; i < action->n_instructions; i++) {
8174 struct instruction *instr = &action->instructions[i];
8175 uint32_t found = 0, j;
8177 if (instr->type != INSTR_LEARNER_LEARN)
8180 for (j = 0; j < n_actions; j++) {
8183 a = action_find(p, action_names[j]);
8187 if (a->id == instr->learn.action_id)
8199 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8201 struct rte_swx_pipeline_learner_params *params,
8205 struct learner *l = NULL;
8206 struct action *default_action;
8207 struct header *header = NULL;
8208 uint32_t action_data_size_max = 0, i;
8213 CHECK_NAME(name, EINVAL);
8214 CHECK(!table_find(p, name), EEXIST);
8215 CHECK(!selector_find(p, name), EEXIST);
8216 CHECK(!learner_find(p, name), EEXIST);
8218 CHECK(params, EINVAL);
8221 status = learner_match_fields_check(p, params, &header);
8225 /* Action checks. */
8226 CHECK(params->n_actions, EINVAL);
8227 CHECK(params->action_names, EINVAL);
8228 for (i = 0; i < params->n_actions; i++) {
8229 const char *action_name = params->action_names[i];
8231 uint32_t action_data_size;
8232 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8234 CHECK_NAME(action_name, EINVAL);
8236 a = action_find(p, action_name);
8239 status = learner_action_learning_check(p,
8241 params->action_names,
8246 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8247 if (action_data_size > action_data_size_max)
8248 action_data_size_max = action_data_size;
8250 if (params->action_is_for_table_entries)
8251 action_is_for_table_entries = params->action_is_for_table_entries[i];
8252 if (params->action_is_for_default_entry)
8253 action_is_for_default_entry = params->action_is_for_default_entry[i];
8254 CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
8257 CHECK_NAME(params->default_action_name, EINVAL);
8258 for (i = 0; i < p->n_actions; i++)
8259 if (!strcmp(params->action_names[i],
8260 params->default_action_name))
8262 CHECK(i < params->n_actions, EINVAL);
8263 CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
8266 default_action = action_find(p, params->default_action_name);
8267 CHECK((default_action->st && params->default_action_data) ||
8268 !params->default_action_data, EINVAL);
8270 /* Any other checks. */
8271 CHECK(size, EINVAL);
8272 CHECK(timeout, EINVAL);
8274 /* Memory allocation. */
8275 l = calloc(1, sizeof(struct learner));
8279 l->fields = calloc(params->n_fields, sizeof(struct field *));
8283 l->actions = calloc(params->n_actions, sizeof(struct action *));
8287 if (action_data_size_max) {
8288 l->default_action_data = calloc(1, action_data_size_max);
8289 if (!l->default_action_data)
8293 l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
8294 if (!l->action_is_for_table_entries)
8297 l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
8298 if (!l->action_is_for_default_entry)
8301 /* Node initialization. */
8302 strcpy(l->name, name);
8304 for (i = 0; i < params->n_fields; i++) {
8305 const char *field_name = params->field_names[i];
8307 l->fields[i] = header ?
8308 header_field_parse(p, field_name, NULL) :
8309 metadata_field_parse(p, field_name);
8312 l->n_fields = params->n_fields;
8316 for (i = 0; i < params->n_actions; i++) {
8317 int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
8319 if (params->action_is_for_table_entries)
8320 action_is_for_table_entries = params->action_is_for_table_entries[i];
8321 if (params->action_is_for_default_entry)
8322 action_is_for_default_entry = params->action_is_for_default_entry[i];
8324 l->actions[i] = action_find(p, params->action_names[i]);
8325 l->action_is_for_table_entries[i] = action_is_for_table_entries;
8326 l->action_is_for_default_entry[i] = action_is_for_default_entry;
8329 l->default_action = default_action;
8331 if (default_action->st)
8332 memcpy(l->default_action_data,
8333 params->default_action_data,
8334 default_action->st->n_bits / 8);
8336 l->n_actions = params->n_actions;
8338 l->default_action_is_const = params->default_action_is_const;
8340 l->action_data_size_max = action_data_size_max;
8344 l->timeout = timeout;
8346 l->id = p->n_learners;
8348 /* Node add to tailq. */
8349 TAILQ_INSERT_TAIL(&p->learners, l, node);
8358 free(l->action_is_for_default_entry);
8359 free(l->action_is_for_table_entries);
8360 free(l->default_action_data);
8369 learner_params_free(struct rte_swx_table_learner_params *params)
8374 free(params->key_mask0);
8379 static struct rte_swx_table_learner_params *
8380 learner_params_get(struct learner *l)
8382 struct rte_swx_table_learner_params *params = NULL;
8383 struct field *first, *last;
8386 /* Memory allocation. */
8387 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8391 /* Find first (smallest offset) and last (biggest offset) match fields. */
8392 first = l->fields[0];
8393 last = l->fields[0];
8395 for (i = 0; i < l->n_fields; i++) {
8396 struct field *f = l->fields[i];
8398 if (f->offset < first->offset)
8401 if (f->offset > last->offset)
8405 /* Key offset and size. */
8406 params->key_offset = first->offset / 8;
8407 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8409 /* Memory allocation. */
8410 params->key_mask0 = calloc(1, params->key_size);
8411 if (!params->key_mask0)
8415 for (i = 0; i < l->n_fields; i++) {
8416 struct field *f = l->fields[i];
8417 uint32_t start = (f->offset - first->offset) / 8;
8418 size_t size = f->n_bits / 8;
8420 memset(¶ms->key_mask0[start], 0xFF, size);
8423 /* Action data size. */
8424 params->action_data_size = l->action_data_size_max;
8426 /* Maximum number of keys. */
8427 params->n_keys_max = l->size;
8430 params->key_timeout = l->timeout;
8435 learner_params_free(params);
8440 learner_build_free(struct rte_swx_pipeline *p)
8444 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8445 struct thread *t = &p->threads[i];
8451 for (j = 0; j < p->n_learners; j++) {
8452 struct learner_runtime *r = &t->learners[j];
8461 if (p->learner_stats) {
8462 for (i = 0; i < p->n_learners; i++)
8463 free(p->learner_stats[i].n_pkts_action);
8465 free(p->learner_stats);
8470 learner_build(struct rte_swx_pipeline *p)
8475 /* Per pipeline: learner statistics. */
8476 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8477 CHECK(p->learner_stats, ENOMEM);
8479 for (i = 0; i < p->n_learners; i++) {
8480 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8481 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8484 /* Per thread: learner run-time. */
8485 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8486 struct thread *t = &p->threads[i];
8489 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8495 TAILQ_FOREACH(l, &p->learners, node) {
8496 struct learner_runtime *r = &t->learners[l->id];
8500 size = rte_swx_table_learner_mailbox_size_get();
8502 r->mailbox = calloc(1, size);
8510 r->key = l->header ?
8511 &t->structs[l->header->struct_id] :
8512 &t->structs[p->metadata_struct_id];
8519 learner_build_free(p);
8524 learner_free(struct rte_swx_pipeline *p)
8526 learner_build_free(p);
8528 /* Learner tables. */
8532 l = TAILQ_FIRST(&p->learners);
8536 TAILQ_REMOVE(&p->learners, l, node);
8539 free(l->default_action_data);
8548 table_state_build(struct rte_swx_pipeline *p)
8550 struct table *table;
8554 p->table_state = calloc(p->n_tables + p->n_selectors,
8555 sizeof(struct rte_swx_table_state));
8556 CHECK(p->table_state, ENOMEM);
8558 TAILQ_FOREACH(table, &p->tables, node) {
8559 struct rte_swx_table_state *ts = &p->table_state[table->id];
8562 struct rte_swx_table_params *params;
8565 params = table_params_get(table);
8566 CHECK(params, ENOMEM);
8568 ts->obj = table->type->ops.create(params,
8573 table_params_free(params);
8574 CHECK(ts->obj, ENODEV);
8577 /* ts->default_action_data. */
8578 if (table->action_data_size_max) {
8579 ts->default_action_data =
8580 malloc(table->action_data_size_max);
8581 CHECK(ts->default_action_data, ENOMEM);
8583 memcpy(ts->default_action_data,
8584 table->default_action_data,
8585 table->action_data_size_max);
8588 /* ts->default_action_id. */
8589 ts->default_action_id = table->default_action->id;
8592 TAILQ_FOREACH(s, &p->selectors, node) {
8593 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8594 struct rte_swx_table_selector_params *params;
8597 params = selector_table_params_get(s);
8598 CHECK(params, ENOMEM);
8600 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8602 selector_params_free(params);
8603 CHECK(ts->obj, ENODEV);
8606 TAILQ_FOREACH(l, &p->learners, node) {
8607 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8608 p->n_selectors + l->id];
8609 struct rte_swx_table_learner_params *params;
8612 params = learner_params_get(l);
8613 CHECK(params, ENOMEM);
8615 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8616 learner_params_free(params);
8617 CHECK(ts->obj, ENODEV);
8619 /* ts->default_action_data. */
8620 if (l->action_data_size_max) {
8621 ts->default_action_data = malloc(l->action_data_size_max);
8622 CHECK(ts->default_action_data, ENOMEM);
8624 memcpy(ts->default_action_data,
8625 l->default_action_data,
8626 l->action_data_size_max);
8629 /* ts->default_action_id. */
8630 ts->default_action_id = l->default_action->id;
8637 table_state_build_free(struct rte_swx_pipeline *p)
8641 if (!p->table_state)
8644 for (i = 0; i < p->n_tables; i++) {
8645 struct rte_swx_table_state *ts = &p->table_state[i];
8646 struct table *table = table_find_by_id(p, i);
8649 if (table->type && ts->obj)
8650 table->type->ops.free(ts->obj);
8652 /* ts->default_action_data. */
8653 free(ts->default_action_data);
8656 for (i = 0; i < p->n_selectors; i++) {
8657 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8661 rte_swx_table_selector_free(ts->obj);
8664 for (i = 0; i < p->n_learners; i++) {
8665 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8669 rte_swx_table_learner_free(ts->obj);
8671 /* ts->default_action_data. */
8672 free(ts->default_action_data);
8675 free(p->table_state);
8676 p->table_state = NULL;
8680 table_state_free(struct rte_swx_pipeline *p)
8682 table_state_build_free(p);
8688 static struct regarray *
8689 regarray_find(struct rte_swx_pipeline *p, const char *name)
8691 struct regarray *elem;
8693 TAILQ_FOREACH(elem, &p->regarrays, node)
8694 if (!strcmp(elem->name, name))
8700 static struct regarray *
8701 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8703 struct regarray *elem = NULL;
8705 TAILQ_FOREACH(elem, &p->regarrays, node)
8713 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8722 CHECK_NAME(name, EINVAL);
8723 CHECK(!regarray_find(p, name), EEXIST);
8725 CHECK(size, EINVAL);
8726 size = rte_align32pow2(size);
8728 /* Memory allocation. */
8729 r = calloc(1, sizeof(struct regarray));
8732 /* Node initialization. */
8733 strcpy(r->name, name);
8734 r->init_val = init_val;
8736 r->id = p->n_regarrays;
8738 /* Node add to tailq. */
8739 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8746 regarray_build(struct rte_swx_pipeline *p)
8748 struct regarray *regarray;
8750 if (!p->n_regarrays)
8753 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8754 CHECK(p->regarray_runtime, ENOMEM);
8756 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8757 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8760 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8761 RTE_CACHE_LINE_SIZE,
8763 CHECK(r->regarray, ENOMEM);
8765 if (regarray->init_val)
8766 for (i = 0; i < regarray->size; i++)
8767 r->regarray[i] = regarray->init_val;
8769 r->size_mask = regarray->size - 1;
8776 regarray_build_free(struct rte_swx_pipeline *p)
8780 if (!p->regarray_runtime)
8783 for (i = 0; i < p->n_regarrays; i++) {
8784 struct regarray *regarray = regarray_find_by_id(p, i);
8785 struct regarray_runtime *r = &p->regarray_runtime[i];
8787 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8790 free(p->regarray_runtime);
8791 p->regarray_runtime = NULL;
8795 regarray_free(struct rte_swx_pipeline *p)
8797 regarray_build_free(p);
8800 struct regarray *elem;
8802 elem = TAILQ_FIRST(&p->regarrays);
8806 TAILQ_REMOVE(&p->regarrays, elem, node);
8814 static struct meter_profile *
8815 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8817 struct meter_profile *elem;
8819 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8820 if (!strcmp(elem->name, name))
8826 static struct metarray *
8827 metarray_find(struct rte_swx_pipeline *p, const char *name)
8829 struct metarray *elem;
8831 TAILQ_FOREACH(elem, &p->metarrays, node)
8832 if (!strcmp(elem->name, name))
8838 static struct metarray *
8839 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8841 struct metarray *elem = NULL;
8843 TAILQ_FOREACH(elem, &p->metarrays, node)
8851 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8859 CHECK_NAME(name, EINVAL);
8860 CHECK(!metarray_find(p, name), EEXIST);
8862 CHECK(size, EINVAL);
8863 size = rte_align32pow2(size);
8865 /* Memory allocation. */
8866 m = calloc(1, sizeof(struct metarray));
8869 /* Node initialization. */
8870 strcpy(m->name, name);
8872 m->id = p->n_metarrays;
8874 /* Node add to tailq. */
8875 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8881 struct meter_profile meter_profile_default = {
8890 .cir_bytes_per_period = 1,
8892 .pir_bytes_per_period = 1,
8899 meter_init(struct meter *m)
8901 memset(m, 0, sizeof(struct meter));
8902 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8903 m->profile = &meter_profile_default;
8904 m->color_mask = RTE_COLOR_GREEN;
8906 meter_profile_default.n_users++;
8910 metarray_build(struct rte_swx_pipeline *p)
8914 if (!p->n_metarrays)
8917 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8918 CHECK(p->metarray_runtime, ENOMEM);
8920 TAILQ_FOREACH(m, &p->metarrays, node) {
8921 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8924 r->metarray = env_malloc(m->size * sizeof(struct meter),
8925 RTE_CACHE_LINE_SIZE,
8927 CHECK(r->metarray, ENOMEM);
8929 for (i = 0; i < m->size; i++)
8930 meter_init(&r->metarray[i]);
8932 r->size_mask = m->size - 1;
8939 metarray_build_free(struct rte_swx_pipeline *p)
8943 if (!p->metarray_runtime)
8946 for (i = 0; i < p->n_metarrays; i++) {
8947 struct metarray *m = metarray_find_by_id(p, i);
8948 struct metarray_runtime *r = &p->metarray_runtime[i];
8950 env_free(r->metarray, m->size * sizeof(struct meter));
8953 free(p->metarray_runtime);
8954 p->metarray_runtime = NULL;
8958 metarray_free(struct rte_swx_pipeline *p)
8960 metarray_build_free(p);
8964 struct metarray *elem;
8966 elem = TAILQ_FIRST(&p->metarrays);
8970 TAILQ_REMOVE(&p->metarrays, elem, node);
8974 /* Meter profiles. */
8976 struct meter_profile *elem;
8978 elem = TAILQ_FIRST(&p->meter_profiles);
8982 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8991 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9000 free(p->instruction_data);
9001 free(p->instructions);
9005 table_state_free(p);
9010 instruction_table_free(p);
9013 extern_func_free(p);
9026 port_in_types_register(struct rte_swx_pipeline *p)
9030 status = rte_swx_pipeline_port_in_type_register(p,
9032 &rte_swx_port_ethdev_reader_ops);
9036 status = rte_swx_pipeline_port_in_type_register(p,
9038 &rte_swx_port_ring_reader_ops);
9042 #ifdef RTE_PORT_PCAP
9043 status = rte_swx_pipeline_port_in_type_register(p,
9045 &rte_swx_port_source_ops);
9050 status = rte_swx_pipeline_port_in_type_register(p,
9052 &rte_swx_port_fd_reader_ops);
9060 port_out_types_register(struct rte_swx_pipeline *p)
9064 status = rte_swx_pipeline_port_out_type_register(p,
9066 &rte_swx_port_ethdev_writer_ops);
9070 status = rte_swx_pipeline_port_out_type_register(p,
9072 &rte_swx_port_ring_writer_ops);
9076 status = rte_swx_pipeline_port_out_type_register(p,
9078 &rte_swx_port_sink_ops);
9082 status = rte_swx_pipeline_port_out_type_register(p,
9084 &rte_swx_port_fd_writer_ops);
9092 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9094 struct rte_swx_pipeline *pipeline = NULL;
9097 /* Check input parameters. */
9100 /* Memory allocation. */
9101 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9107 /* Initialization. */
9108 TAILQ_INIT(&pipeline->struct_types);
9109 TAILQ_INIT(&pipeline->port_in_types);
9110 TAILQ_INIT(&pipeline->ports_in);
9111 TAILQ_INIT(&pipeline->port_out_types);
9112 TAILQ_INIT(&pipeline->ports_out);
9113 TAILQ_INIT(&pipeline->extern_types);
9114 TAILQ_INIT(&pipeline->extern_objs);
9115 TAILQ_INIT(&pipeline->extern_funcs);
9116 TAILQ_INIT(&pipeline->headers);
9117 TAILQ_INIT(&pipeline->actions);
9118 TAILQ_INIT(&pipeline->table_types);
9119 TAILQ_INIT(&pipeline->tables);
9120 TAILQ_INIT(&pipeline->selectors);
9121 TAILQ_INIT(&pipeline->learners);
9122 TAILQ_INIT(&pipeline->regarrays);
9123 TAILQ_INIT(&pipeline->meter_profiles);
9124 TAILQ_INIT(&pipeline->metarrays);
9126 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9127 pipeline->numa_node = numa_node;
9129 status = port_in_types_register(pipeline);
9133 status = port_out_types_register(pipeline);
9141 rte_swx_pipeline_free(pipeline);
9146 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9147 const char **instructions,
9148 uint32_t n_instructions)
9153 err = instruction_config(p, NULL, instructions, n_instructions);
9157 /* Thread instruction pointer reset. */
9158 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9159 struct thread *t = &p->threads[i];
9161 thread_ip_reset(p, t);
9168 pipeline_compile(struct rte_swx_pipeline *p);
9171 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9176 CHECK(p->build_done == 0, EEXIST);
9178 status = port_in_build(p);
9182 status = port_out_build(p);
9186 status = struct_build(p);
9190 status = extern_obj_build(p);
9194 status = extern_func_build(p);
9198 status = header_build(p);
9202 status = metadata_build(p);
9206 status = instruction_table_build(p);
9210 status = action_build(p);
9214 status = table_build(p);
9218 status = selector_build(p);
9222 status = learner_build(p);
9226 status = table_state_build(p);
9230 status = regarray_build(p);
9234 status = metarray_build(p);
9240 pipeline_compile(p);
9245 metarray_build_free(p);
9246 regarray_build_free(p);
9247 table_state_build_free(p);
9248 learner_build_free(p);
9249 selector_build_free(p);
9250 table_build_free(p);
9251 action_build_free(p);
9252 instruction_table_build_free(p);
9253 metadata_build_free(p);
9254 header_build_free(p);
9255 extern_func_build_free(p);
9256 extern_obj_build_free(p);
9257 port_out_build_free(p);
9258 port_in_build_free(p);
9259 struct_build_free(p);
9265 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9269 for (i = 0; i < n_instructions; i++)
9274 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9278 for (i = 0; i < p->n_ports_out; i++) {
9279 struct port_out_runtime *port = &p->out[i];
9282 port->flush(port->obj);
9290 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9291 struct rte_swx_ctl_pipeline_info *pipeline)
9293 struct action *action;
9294 struct table *table;
9295 uint32_t n_actions = 0, n_tables = 0;
9297 if (!p || !pipeline)
9300 TAILQ_FOREACH(action, &p->actions, node)
9303 TAILQ_FOREACH(table, &p->tables, node)
9306 pipeline->n_ports_in = p->n_ports_in;
9307 pipeline->n_ports_out = p->n_ports_out;
9308 pipeline->n_actions = n_actions;
9309 pipeline->n_tables = n_tables;
9310 pipeline->n_selectors = p->n_selectors;
9311 pipeline->n_learners = p->n_learners;
9312 pipeline->n_regarrays = p->n_regarrays;
9313 pipeline->n_metarrays = p->n_metarrays;
9319 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9321 if (!p || !numa_node)
9324 *numa_node = p->numa_node;
9329 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9331 struct rte_swx_ctl_action_info *action)
9333 struct action *a = NULL;
9335 if (!p || (action_id >= p->n_actions) || !action)
9338 a = action_find_by_id(p, action_id);
9342 strcpy(action->name, a->name);
9343 action->n_args = a->st ? a->st->n_fields : 0;
9348 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9350 uint32_t action_arg_id,
9351 struct rte_swx_ctl_action_arg_info *action_arg)
9353 struct action *a = NULL;
9354 struct field *arg = NULL;
9356 if (!p || (action_id >= p->n_actions) || !action_arg)
9359 a = action_find_by_id(p, action_id);
9360 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9363 arg = &a->st->fields[action_arg_id];
9364 strcpy(action_arg->name, arg->name);
9365 action_arg->n_bits = arg->n_bits;
9366 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9372 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9374 struct rte_swx_ctl_table_info *table)
9376 struct table *t = NULL;
9381 t = table_find_by_id(p, table_id);
9385 strcpy(table->name, t->name);
9386 strcpy(table->args, t->args);
9387 table->n_match_fields = t->n_fields;
9388 table->n_actions = t->n_actions;
9389 table->default_action_is_const = t->default_action_is_const;
9390 table->size = t->size;
9395 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9397 uint32_t match_field_id,
9398 struct rte_swx_ctl_table_match_field_info *match_field)
9401 struct match_field *f;
9403 if (!p || (table_id >= p->n_tables) || !match_field)
9406 t = table_find_by_id(p, table_id);
9407 if (!t || (match_field_id >= t->n_fields))
9410 f = &t->fields[match_field_id];
9411 match_field->match_type = f->match_type;
9412 match_field->is_header = t->header ? 1 : 0;
9413 match_field->n_bits = f->field->n_bits;
9414 match_field->offset = f->field->offset;
9420 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9422 uint32_t table_action_id,
9423 struct rte_swx_ctl_table_action_info *table_action)
9427 if (!p || (table_id >= p->n_tables) || !table_action)
9430 t = table_find_by_id(p, table_id);
9431 if (!t || (table_action_id >= t->n_actions))
9434 table_action->action_id = t->actions[table_action_id]->id;
9436 table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
9437 table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
9443 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9445 struct rte_swx_table_ops *table_ops,
9450 if (!p || (table_id >= p->n_tables))
9453 t = table_find_by_id(p, table_id);
9459 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9469 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9470 uint32_t selector_id,
9471 struct rte_swx_ctl_selector_info *selector)
9473 struct selector *s = NULL;
9475 if (!p || !selector)
9478 s = selector_find_by_id(p, selector_id);
9482 strcpy(selector->name, s->name);
9484 selector->n_selector_fields = s->n_selector_fields;
9485 selector->n_groups_max = s->n_groups_max;
9486 selector->n_members_per_group_max = s->n_members_per_group_max;
9492 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9493 uint32_t selector_id,
9494 struct rte_swx_ctl_table_match_field_info *field)
9498 if (!p || (selector_id >= p->n_selectors) || !field)
9501 s = selector_find_by_id(p, selector_id);
9505 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9506 field->is_header = 0;
9507 field->n_bits = s->group_id_field->n_bits;
9508 field->offset = s->group_id_field->offset;
9514 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9515 uint32_t selector_id,
9516 uint32_t selector_field_id,
9517 struct rte_swx_ctl_table_match_field_info *field)
9522 if (!p || (selector_id >= p->n_selectors) || !field)
9525 s = selector_find_by_id(p, selector_id);
9526 if (!s || (selector_field_id >= s->n_selector_fields))
9529 f = s->selector_fields[selector_field_id];
9530 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9531 field->is_header = s->selector_header ? 1 : 0;
9532 field->n_bits = f->n_bits;
9533 field->offset = f->offset;
9539 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9540 uint32_t selector_id,
9541 struct rte_swx_ctl_table_match_field_info *field)
9545 if (!p || (selector_id >= p->n_selectors) || !field)
9548 s = selector_find_by_id(p, selector_id);
9552 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9553 field->is_header = 0;
9554 field->n_bits = s->member_id_field->n_bits;
9555 field->offset = s->member_id_field->offset;
9561 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9562 uint32_t learner_id,
9563 struct rte_swx_ctl_learner_info *learner)
9565 struct learner *l = NULL;
9570 l = learner_find_by_id(p, learner_id);
9574 strcpy(learner->name, l->name);
9576 learner->n_match_fields = l->n_fields;
9577 learner->n_actions = l->n_actions;
9578 learner->default_action_is_const = l->default_action_is_const;
9579 learner->size = l->size;
9585 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9586 uint32_t learner_id,
9587 uint32_t match_field_id,
9588 struct rte_swx_ctl_table_match_field_info *match_field)
9593 if (!p || (learner_id >= p->n_learners) || !match_field)
9596 l = learner_find_by_id(p, learner_id);
9597 if (!l || (match_field_id >= l->n_fields))
9600 f = l->fields[match_field_id];
9601 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9602 match_field->is_header = l->header ? 1 : 0;
9603 match_field->n_bits = f->n_bits;
9604 match_field->offset = f->offset;
9610 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9611 uint32_t learner_id,
9612 uint32_t learner_action_id,
9613 struct rte_swx_ctl_table_action_info *learner_action)
9617 if (!p || (learner_id >= p->n_learners) || !learner_action)
9620 l = learner_find_by_id(p, learner_id);
9621 if (!l || (learner_action_id >= l->n_actions))
9624 learner_action->action_id = l->actions[learner_action_id]->id;
9626 learner_action->action_is_for_table_entries =
9627 l->action_is_for_table_entries[learner_action_id];
9629 learner_action->action_is_for_default_entry =
9630 l->action_is_for_default_entry[learner_action_id];
9636 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9637 struct rte_swx_table_state **table_state)
9639 if (!p || !table_state || !p->build_done)
9642 *table_state = p->table_state;
9647 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9648 struct rte_swx_table_state *table_state)
9650 if (!p || !table_state || !p->build_done)
9653 p->table_state = table_state;
9658 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9660 struct rte_swx_port_in_stats *stats)
9662 struct port_in *port;
9667 port = port_in_find(p, port_id);
9671 port->type->ops.stats_read(port->obj, stats);
9676 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9678 struct rte_swx_port_out_stats *stats)
9680 struct port_out *port;
9685 port = port_out_find(p, port_id);
9689 port->type->ops.stats_read(port->obj, stats);
9694 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9695 const char *table_name,
9696 struct rte_swx_table_stats *stats)
9698 struct table *table;
9699 struct table_statistics *table_stats;
9701 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9704 table = table_find(p, table_name);
9708 table_stats = &p->table_stats[table->id];
9710 memcpy(stats->n_pkts_action,
9711 table_stats->n_pkts_action,
9712 p->n_actions * sizeof(uint64_t));
9714 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9715 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9721 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9722 const char *selector_name,
9723 struct rte_swx_pipeline_selector_stats *stats)
9727 if (!p || !selector_name || !selector_name[0] || !stats)
9730 s = selector_find(p, selector_name);
9734 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9740 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9741 const char *learner_name,
9742 struct rte_swx_learner_stats *stats)
9745 struct learner_statistics *learner_stats;
9747 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9750 l = learner_find(p, learner_name);
9754 learner_stats = &p->learner_stats[l->id];
9756 memcpy(stats->n_pkts_action,
9757 learner_stats->n_pkts_action,
9758 p->n_actions * sizeof(uint64_t));
9760 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9761 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9763 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9764 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9766 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9772 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9773 uint32_t regarray_id,
9774 struct rte_swx_ctl_regarray_info *regarray)
9778 if (!p || !regarray)
9781 r = regarray_find_by_id(p, regarray_id);
9785 strcpy(regarray->name, r->name);
9786 regarray->size = r->size;
9791 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9792 const char *regarray_name,
9793 uint32_t regarray_index,
9796 struct regarray *regarray;
9797 struct regarray_runtime *r;
9799 if (!p || !regarray_name || !value)
9802 regarray = regarray_find(p, regarray_name);
9803 if (!regarray || (regarray_index >= regarray->size))
9806 r = &p->regarray_runtime[regarray->id];
9807 *value = r->regarray[regarray_index];
9812 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9813 const char *regarray_name,
9814 uint32_t regarray_index,
9817 struct regarray *regarray;
9818 struct regarray_runtime *r;
9820 if (!p || !regarray_name)
9823 regarray = regarray_find(p, regarray_name);
9824 if (!regarray || (regarray_index >= regarray->size))
9827 r = &p->regarray_runtime[regarray->id];
9828 r->regarray[regarray_index] = value;
9833 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9834 uint32_t metarray_id,
9835 struct rte_swx_ctl_metarray_info *metarray)
9839 if (!p || !metarray)
9842 m = metarray_find_by_id(p, metarray_id);
9846 strcpy(metarray->name, m->name);
9847 metarray->size = m->size;
9852 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9854 struct rte_meter_trtcm_params *params)
9856 struct meter_profile *mp;
9860 CHECK_NAME(name, EINVAL);
9861 CHECK(params, EINVAL);
9862 CHECK(!meter_profile_find(p, name), EEXIST);
9864 /* Node allocation. */
9865 mp = calloc(1, sizeof(struct meter_profile));
9868 /* Node initialization. */
9869 strcpy(mp->name, name);
9870 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9871 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9877 /* Node add to tailq. */
9878 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9884 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9887 struct meter_profile *mp;
9890 CHECK_NAME(name, EINVAL);
9892 mp = meter_profile_find(p, name);
9894 CHECK(!mp->n_users, EBUSY);
9896 /* Remove node from tailq. */
9897 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9904 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9905 const char *metarray_name,
9906 uint32_t metarray_index)
9908 struct meter_profile *mp_old;
9909 struct metarray *metarray;
9910 struct metarray_runtime *metarray_runtime;
9914 CHECK_NAME(metarray_name, EINVAL);
9916 metarray = metarray_find(p, metarray_name);
9917 CHECK(metarray, EINVAL);
9918 CHECK(metarray_index < metarray->size, EINVAL);
9920 metarray_runtime = &p->metarray_runtime[metarray->id];
9921 m = &metarray_runtime->metarray[metarray_index];
9922 mp_old = m->profile;
9932 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9933 const char *metarray_name,
9934 uint32_t metarray_index,
9935 const char *profile_name)
9937 struct meter_profile *mp, *mp_old;
9938 struct metarray *metarray;
9939 struct metarray_runtime *metarray_runtime;
9943 CHECK_NAME(metarray_name, EINVAL);
9945 metarray = metarray_find(p, metarray_name);
9946 CHECK(metarray, EINVAL);
9947 CHECK(metarray_index < metarray->size, EINVAL);
9949 mp = meter_profile_find(p, profile_name);
9952 metarray_runtime = &p->metarray_runtime[metarray->id];
9953 m = &metarray_runtime->metarray[metarray_index];
9954 mp_old = m->profile;
9956 memset(m, 0, sizeof(struct meter));
9957 rte_meter_trtcm_config(&m->m, &mp->profile);
9959 m->color_mask = RTE_COLORS;
9968 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9969 const char *metarray_name,
9970 uint32_t metarray_index,
9971 struct rte_swx_ctl_meter_stats *stats)
9973 struct metarray *metarray;
9974 struct metarray_runtime *metarray_runtime;
9978 CHECK_NAME(metarray_name, EINVAL);
9980 metarray = metarray_find(p, metarray_name);
9981 CHECK(metarray, EINVAL);
9982 CHECK(metarray_index < metarray->size, EINVAL);
9984 CHECK(stats, EINVAL);
9986 metarray_runtime = &p->metarray_runtime[metarray->id];
9987 m = &metarray_runtime->metarray[metarray_index];
9989 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9990 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
9996 * Pipeline compilation.
9999 instr_type_to_name(struct instruction *instr)
10001 switch (instr->type) {
10002 case INSTR_RX: return "INSTR_RX";
10004 case INSTR_TX: return "INSTR_TX";
10005 case INSTR_TX_I: return "INSTR_TX_I";
10006 case INSTR_DROP: return "INSTR_DROP";
10008 case INSTR_HDR_EXTRACT: return "INSTR_HDR_EXTRACT";
10009 case INSTR_HDR_EXTRACT2: return "INSTR_HDR_EXTRACT2";
10010 case INSTR_HDR_EXTRACT3: return "INSTR_HDR_EXTRACT3";
10011 case INSTR_HDR_EXTRACT4: return "INSTR_HDR_EXTRACT4";
10012 case INSTR_HDR_EXTRACT5: return "INSTR_HDR_EXTRACT5";
10013 case INSTR_HDR_EXTRACT6: return "INSTR_HDR_EXTRACT6";
10014 case INSTR_HDR_EXTRACT7: return "INSTR_HDR_EXTRACT7";
10015 case INSTR_HDR_EXTRACT8: return "INSTR_HDR_EXTRACT8";
10017 case INSTR_HDR_EXTRACT_M: return "INSTR_HDR_EXTRACT_M";
10019 case INSTR_HDR_LOOKAHEAD: return "INSTR_HDR_LOOKAHEAD";
10021 case INSTR_HDR_EMIT: return "INSTR_HDR_EMIT";
10022 case INSTR_HDR_EMIT_TX: return "INSTR_HDR_EMIT_TX";
10023 case INSTR_HDR_EMIT2_TX: return "INSTR_HDR_EMIT2_TX";
10024 case INSTR_HDR_EMIT3_TX: return "INSTR_HDR_EMIT3_TX";
10025 case INSTR_HDR_EMIT4_TX: return "INSTR_HDR_EMIT4_TX";
10026 case INSTR_HDR_EMIT5_TX: return "INSTR_HDR_EMIT5_TX";
10027 case INSTR_HDR_EMIT6_TX: return "INSTR_HDR_EMIT6_TX";
10028 case INSTR_HDR_EMIT7_TX: return "INSTR_HDR_EMIT7_TX";
10029 case INSTR_HDR_EMIT8_TX: return "INSTR_HDR_EMIT8_TX";
10031 case INSTR_HDR_VALIDATE: return "INSTR_HDR_VALIDATE";
10032 case INSTR_HDR_INVALIDATE: return "INSTR_HDR_INVALIDATE";
10034 case INSTR_MOV: return "INSTR_MOV";
10035 case INSTR_MOV_MH: return "INSTR_MOV_MH";
10036 case INSTR_MOV_HM: return "INSTR_MOV_HM";
10037 case INSTR_MOV_HH: return "INSTR_MOV_HH";
10038 case INSTR_MOV_I: return "INSTR_MOV_I";
10040 case INSTR_DMA_HT: return "INSTR_DMA_HT";
10041 case INSTR_DMA_HT2: return "INSTR_DMA_HT2";
10042 case INSTR_DMA_HT3: return "INSTR_DMA_HT3";
10043 case INSTR_DMA_HT4: return "INSTR_DMA_HT4";
10044 case INSTR_DMA_HT5: return "INSTR_DMA_HT5";
10045 case INSTR_DMA_HT6: return "INSTR_DMA_HT6";
10046 case INSTR_DMA_HT7: return "INSTR_DMA_HT7";
10047 case INSTR_DMA_HT8: return "INSTR_DMA_HT8";
10049 case INSTR_ALU_ADD: return "INSTR_ALU_ADD";
10050 case INSTR_ALU_ADD_MH: return "INSTR_ALU_ADD_MH";
10051 case INSTR_ALU_ADD_HM: return "INSTR_ALU_ADD_HM";
10052 case INSTR_ALU_ADD_HH: return "INSTR_ALU_ADD_HH";
10053 case INSTR_ALU_ADD_MI: return "INSTR_ALU_ADD_MI";
10054 case INSTR_ALU_ADD_HI: return "INSTR_ALU_ADD_HI";
10056 case INSTR_ALU_SUB: return "INSTR_ALU_SUB";
10057 case INSTR_ALU_SUB_MH: return "INSTR_ALU_SUB_MH";
10058 case INSTR_ALU_SUB_HM: return "INSTR_ALU_SUB_HM";
10059 case INSTR_ALU_SUB_HH: return "INSTR_ALU_SUB_HH";
10060 case INSTR_ALU_SUB_MI: return "INSTR_ALU_SUB_MI";
10061 case INSTR_ALU_SUB_HI: return "INSTR_ALU_SUB_HI";
10063 case INSTR_ALU_CKADD_FIELD: return "INSTR_ALU_CKADD_FIELD";
10064 case INSTR_ALU_CKADD_STRUCT20: return "INSTR_ALU_CKADD_STRUCT20";
10065 case INSTR_ALU_CKADD_STRUCT: return "INSTR_ALU_CKADD_STRUCT";
10066 case INSTR_ALU_CKSUB_FIELD: return "INSTR_ALU_CKSUB_FIELD";
10068 case INSTR_ALU_AND: return "INSTR_ALU_AND";
10069 case INSTR_ALU_AND_MH: return "INSTR_ALU_AND_MH";
10070 case INSTR_ALU_AND_HM: return "INSTR_ALU_AND_HM";
10071 case INSTR_ALU_AND_HH: return "INSTR_ALU_AND_HH";
10072 case INSTR_ALU_AND_I: return "INSTR_ALU_AND_I";
10074 case INSTR_ALU_OR: return "INSTR_ALU_OR";
10075 case INSTR_ALU_OR_MH: return "INSTR_ALU_OR_MH";
10076 case INSTR_ALU_OR_HM: return "INSTR_ALU_OR_HM";
10077 case INSTR_ALU_OR_HH: return "INSTR_ALU_OR_HH";
10078 case INSTR_ALU_OR_I: return "INSTR_ALU_OR_I";
10080 case INSTR_ALU_XOR: return "INSTR_ALU_XOR";
10081 case INSTR_ALU_XOR_MH: return "INSTR_ALU_XOR_MH";
10082 case INSTR_ALU_XOR_HM: return "INSTR_ALU_XOR_HM";
10083 case INSTR_ALU_XOR_HH: return "INSTR_ALU_XOR_HH";
10084 case INSTR_ALU_XOR_I: return "INSTR_ALU_XOR_I";
10086 case INSTR_ALU_SHL: return "INSTR_ALU_SHL";
10087 case INSTR_ALU_SHL_MH: return "INSTR_ALU_SHL_MH";
10088 case INSTR_ALU_SHL_HM: return "INSTR_ALU_SHL_HM";
10089 case INSTR_ALU_SHL_HH: return "INSTR_ALU_SHL_HH";
10090 case INSTR_ALU_SHL_MI: return "INSTR_ALU_SHL_MI";
10091 case INSTR_ALU_SHL_HI: return "INSTR_ALU_SHL_HI";
10093 case INSTR_ALU_SHR: return "INSTR_ALU_SHR";
10094 case INSTR_ALU_SHR_MH: return "INSTR_ALU_SHR_MH";
10095 case INSTR_ALU_SHR_HM: return "INSTR_ALU_SHR_HM";
10096 case INSTR_ALU_SHR_HH: return "INSTR_ALU_SHR_HH";
10097 case INSTR_ALU_SHR_MI: return "INSTR_ALU_SHR_MI";
10098 case INSTR_ALU_SHR_HI: return "INSTR_ALU_SHR_HI";
10100 case INSTR_REGPREFETCH_RH: return "INSTR_REGPREFETCH_RH";
10101 case INSTR_REGPREFETCH_RM: return "INSTR_REGPREFETCH_RM";
10102 case INSTR_REGPREFETCH_RI: return "INSTR_REGPREFETCH_RI";
10104 case INSTR_REGRD_HRH: return "INSTR_REGRD_HRH";
10105 case INSTR_REGRD_HRM: return "INSTR_REGRD_HRM";
10106 case INSTR_REGRD_HRI: return "INSTR_REGRD_HRI";
10107 case INSTR_REGRD_MRH: return "INSTR_REGRD_MRH";
10108 case INSTR_REGRD_MRM: return "INSTR_REGRD_MRM";
10109 case INSTR_REGRD_MRI: return "INSTR_REGRD_MRI";
10111 case INSTR_REGWR_RHH: return "INSTR_REGWR_RHH";
10112 case INSTR_REGWR_RHM: return "INSTR_REGWR_RHM";
10113 case INSTR_REGWR_RHI: return "INSTR_REGWR_RHI";
10114 case INSTR_REGWR_RMH: return "INSTR_REGWR_RMH";
10115 case INSTR_REGWR_RMM: return "INSTR_REGWR_RMM";
10116 case INSTR_REGWR_RMI: return "INSTR_REGWR_RMI";
10117 case INSTR_REGWR_RIH: return "INSTR_REGWR_RIH";
10118 case INSTR_REGWR_RIM: return "INSTR_REGWR_RIM";
10119 case INSTR_REGWR_RII: return "INSTR_REGWR_RII";
10121 case INSTR_REGADD_RHH: return "INSTR_REGADD_RHH";
10122 case INSTR_REGADD_RHM: return "INSTR_REGADD_RHM";
10123 case INSTR_REGADD_RHI: return "INSTR_REGADD_RHI";
10124 case INSTR_REGADD_RMH: return "INSTR_REGADD_RMH";
10125 case INSTR_REGADD_RMM: return "INSTR_REGADD_RMM";
10126 case INSTR_REGADD_RMI: return "INSTR_REGADD_RMI";
10127 case INSTR_REGADD_RIH: return "INSTR_REGADD_RIH";
10128 case INSTR_REGADD_RIM: return "INSTR_REGADD_RIM";
10129 case INSTR_REGADD_RII: return "INSTR_REGADD_RII";
10131 case INSTR_METPREFETCH_H: return "INSTR_METPREFETCH_H";
10132 case INSTR_METPREFETCH_M: return "INSTR_METPREFETCH_M";
10133 case INSTR_METPREFETCH_I: return "INSTR_METPREFETCH_I";
10135 case INSTR_METER_HHM: return "INSTR_METER_HHM";
10136 case INSTR_METER_HHI: return "INSTR_METER_HHI";
10137 case INSTR_METER_HMM: return "INSTR_METER_HMM";
10138 case INSTR_METER_HMI: return "INSTR_METER_HMI";
10139 case INSTR_METER_MHM: return "INSTR_METER_MHM";
10140 case INSTR_METER_MHI: return "INSTR_METER_MHI";
10141 case INSTR_METER_MMM: return "INSTR_METER_MMM";
10142 case INSTR_METER_MMI: return "INSTR_METER_MMI";
10143 case INSTR_METER_IHM: return "INSTR_METER_IHM";
10144 case INSTR_METER_IHI: return "INSTR_METER_IHI";
10145 case INSTR_METER_IMM: return "INSTR_METER_IMM";
10146 case INSTR_METER_IMI: return "INSTR_METER_IMI";
10148 case INSTR_TABLE: return "INSTR_TABLE";
10149 case INSTR_TABLE_AF: return "INSTR_TABLE_AF";
10150 case INSTR_SELECTOR: return "INSTR_SELECTOR";
10151 case INSTR_LEARNER: return "INSTR_LEARNER";
10152 case INSTR_LEARNER_AF: return "INSTR_LEARNER_AF";
10154 case INSTR_LEARNER_LEARN: return "INSTR_LEARNER_LEARN";
10155 case INSTR_LEARNER_FORGET: return "INSTR_LEARNER_FORGET";
10157 case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
10158 case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
10160 case INSTR_JMP: return "INSTR_JMP";
10161 case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
10162 case INSTR_JMP_INVALID: return "INSTR_JMP_INVALID";
10163 case INSTR_JMP_HIT: return "INSTR_JMP_HIT";
10164 case INSTR_JMP_MISS: return "INSTR_JMP_MISS";
10165 case INSTR_JMP_ACTION_HIT: return "INSTR_JMP_ACTION_HIT";
10166 case INSTR_JMP_ACTION_MISS: return "INSTR_JMP_ACTION_MISS";
10167 case INSTR_JMP_EQ: return "INSTR_JMP_EQ";
10168 case INSTR_JMP_EQ_MH: return "INSTR_JMP_EQ_MH";
10169 case INSTR_JMP_EQ_HM: return "INSTR_JMP_EQ_HM";
10170 case INSTR_JMP_EQ_HH: return "INSTR_JMP_EQ_HH";
10171 case INSTR_JMP_EQ_I: return "INSTR_JMP_EQ_I";
10172 case INSTR_JMP_NEQ: return "INSTR_JMP_NEQ";
10173 case INSTR_JMP_NEQ_MH: return "INSTR_JMP_NEQ_MH";
10174 case INSTR_JMP_NEQ_HM: return "INSTR_JMP_NEQ_HM";
10175 case INSTR_JMP_NEQ_HH: return "INSTR_JMP_NEQ_HH";
10176 case INSTR_JMP_NEQ_I: return "INSTR_JMP_NEQ_I";
10177 case INSTR_JMP_LT: return "INSTR_JMP_LT";
10178 case INSTR_JMP_LT_MH: return "INSTR_JMP_LT_MH";
10179 case INSTR_JMP_LT_HM: return "INSTR_JMP_LT_HM";
10180 case INSTR_JMP_LT_HH: return "INSTR_JMP_LT_HH";
10181 case INSTR_JMP_LT_MI: return "INSTR_JMP_LT_MI";
10182 case INSTR_JMP_LT_HI: return "INSTR_JMP_LT_HI";
10183 case INSTR_JMP_GT: return "INSTR_JMP_GT";
10184 case INSTR_JMP_GT_MH: return "INSTR_JMP_GT_MH";
10185 case INSTR_JMP_GT_HM: return "INSTR_JMP_GT_HM";
10186 case INSTR_JMP_GT_HH: return "INSTR_JMP_GT_HH";
10187 case INSTR_JMP_GT_MI: return "INSTR_JMP_GT_MI";
10188 case INSTR_JMP_GT_HI: return "INSTR_JMP_GT_HI";
10190 case INSTR_RETURN: return "INSTR_RETURN";
10192 default: return "INSTR_UNKNOWN";
10197 (*instruction_export_t)(struct instruction *, FILE *);
10200 instr_io_export(struct instruction *instr, FILE *f)
10202 uint32_t n_io = 0, n_io_imm = 0, n_hdrs = 0, i;
10204 /* n_io, n_io_imm, n_hdrs. */
10205 if (instr->type == INSTR_RX ||
10206 instr->type == INSTR_TX ||
10207 instr->type == INSTR_HDR_EXTRACT_M ||
10208 (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX))
10211 if (instr->type == INSTR_TX_I)
10214 if (instr->type >= INSTR_HDR_EXTRACT && instr->type <= INSTR_HDR_EXTRACT8)
10215 n_hdrs = 1 + (instr->type - INSTR_HDR_EXTRACT);
10217 if (instr->type == INSTR_HDR_EXTRACT_M ||
10218 instr->type == INSTR_HDR_LOOKAHEAD ||
10219 instr->type == INSTR_HDR_EMIT)
10222 if (instr->type >= INSTR_HDR_EMIT_TX && instr->type <= INSTR_HDR_EMIT8_TX)
10223 n_hdrs = 1 + (instr->type - INSTR_HDR_EMIT_TX);
10228 "\t\t.type = %s,\n",
10229 instr_type_to_name(instr));
10232 if (n_io || n_io_imm || n_hdrs)
10240 "\t\t\t\t.offset = %u,\n"
10241 "\t\t\t\t.n_bits = %u,\n"
10243 instr->io.io.offset,
10244 instr->io.io.n_bits);
10249 "\t\t\t\t.val = %u,\n"
10253 /* instr.io.hdr. */
10258 /* instr.io.hdr.header_id. */
10260 "\t\t\t.header_id = {");
10262 for (i = 0; i < n_hdrs; i++)
10265 instr->io.hdr.header_id[i]);
10270 /* instr.io.hdr.struct_id. */
10272 "\t\t\t.struct_id = {");
10274 for (i = 0; i < n_hdrs; i++)
10277 instr->io.hdr.struct_id[i]);
10282 /* instr.io.hdr.n_bytes. */
10284 "\t\t\t.n_bytes = {");
10286 for (i = 0; i < n_hdrs; i++)
10289 instr->io.hdr.n_bytes[i]);
10294 /* instr.io.hdr - closing curly brace. */
10299 /* instr.io - closing curly brace. */
10300 if (n_io || n_io_imm || n_hdrs)
10304 /* instr - closing curly brace. */
10310 instr_hdr_validate_export(struct instruction *instr, FILE *f)
10314 "\t\t.type = %s,\n"
10316 "\t\t\t.header_id = %u,\n"
10319 instr_type_to_name(instr),
10320 instr->valid.header_id);
10324 instr_mov_export(struct instruction *instr, FILE *f)
10326 if (instr->type != INSTR_MOV_I)
10329 "\t\t.type = %s,\n"
10332 "\t\t\t\t.struct_id = %u,\n"
10333 "\t\t\t\t.n_bits = %u,\n"
10334 "\t\t\t\t.offset = %u,\n"
10337 "\t\t\t\t.struct_id = %u,\n"
10338 "\t\t\t\t.n_bits = %u,\n"
10339 "\t\t\t\t.offset = %u,\n"
10343 instr_type_to_name(instr),
10344 instr->mov.dst.struct_id,
10345 instr->mov.dst.n_bits,
10346 instr->mov.dst.offset,
10347 instr->mov.src.struct_id,
10348 instr->mov.src.n_bits,
10349 instr->mov.src.offset);
10353 "\t\t.type = %s,\n"
10356 "\t\t\t\t.struct_id = %u,\n"
10357 "\t\t\t\t.n_bits = %u,\n"
10358 "\t\t\t\t.offset = %u,\n"
10360 "\t\t\t.src_val = %" PRIu64 ",\n"
10363 instr_type_to_name(instr),
10364 instr->mov.dst.struct_id,
10365 instr->mov.dst.n_bits,
10366 instr->mov.dst.offset,
10367 instr->mov.src_val);
10371 instr_dma_ht_export(struct instruction *instr, FILE *f)
10373 uint32_t n_dma = 0, i;
10376 n_dma = 1 + (instr->type - INSTR_DMA_HT);
10381 "\t\t.type = %s,\n",
10382 instr_type_to_name(instr));
10388 /* instr.dma.dst. */
10390 "\t\t\t.dst = {\n");
10392 /* instr.dma.dst.header_id. */
10394 "\t\t\t\t.header_id = {");
10396 for (i = 0; i < n_dma; i++)
10399 instr->dma.dst.header_id[i]);
10404 /* instr.dma.dst.struct_id. */
10406 "\t\t\t\t.struct_id = {");
10408 for (i = 0; i < n_dma; i++)
10411 instr->dma.dst.struct_id[i]);
10416 /* instr.dma.dst - closing curly brace. */
10420 /* instr.dma.src. */
10422 "\t\t\t.src = {\n");
10424 /* instr.dma.src.offset. */
10426 "\t\t\t\t.offset = {");
10428 for (i = 0; i < n_dma; i++)
10431 instr->dma.src.offset[i]);
10436 /* instr.dma.src - closing curly brace. */
10440 /* instr.dma.n_bytes. */
10442 "\t\t\t.n_bytes = {");
10444 for (i = 0; i < n_dma; i++)
10447 instr->dma.n_bytes[i]);
10452 /* instr.dma - closing curly brace. */
10456 /* instr - closing curly brace. */
10462 instr_alu_export(struct instruction *instr, FILE *f)
10466 if (instr->type == INSTR_ALU_ADD_MI ||
10467 instr->type == INSTR_ALU_ADD_HI ||
10468 instr->type == INSTR_ALU_SUB_MI ||
10469 instr->type == INSTR_ALU_SUB_HI ||
10470 instr->type == INSTR_ALU_SHL_MI ||
10471 instr->type == INSTR_ALU_SHL_HI ||
10472 instr->type == INSTR_ALU_SHR_MI ||
10473 instr->type == INSTR_ALU_SHR_HI ||
10474 instr->type == INSTR_ALU_AND_I ||
10475 instr->type == INSTR_ALU_OR_I ||
10476 instr->type == INSTR_ALU_XOR_I)
10482 "\t\t.type = %s,\n"
10485 "\t\t\t\t.struct_id = %u,\n"
10486 "\t\t\t\t.n_bits = %u,\n"
10487 "\t\t\t\t.offset = %u,\n"
10490 "\t\t\t\t.struct_id = %u,\n"
10491 "\t\t\t\t.n_bits = %u,\n"
10492 "\t\t\t\t.offset = %u,\n"
10496 instr_type_to_name(instr),
10497 instr->alu.dst.struct_id,
10498 instr->alu.dst.n_bits,
10499 instr->alu.dst.offset,
10500 instr->alu.src.struct_id,
10501 instr->alu.src.n_bits,
10502 instr->alu.src.offset);
10506 "\t\t.type = %s,\n"
10509 "\t\t\t\t.struct_id = %u,\n"
10510 "\t\t\t\t.n_bits = %u,\n"
10511 "\t\t\t\t.offset = %u,\n"
10513 "\t\t\t.src_val = %" PRIu64 ",\n"
10516 instr_type_to_name(instr),
10517 instr->alu.dst.struct_id,
10518 instr->alu.dst.n_bits,
10519 instr->alu.dst.offset,
10520 instr->alu.src_val);
10524 instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10526 int prefetch = 0, idx_imm = 0, src_imm = 0;
10528 if (instr->type == INSTR_REGPREFETCH_RH ||
10529 instr->type == INSTR_REGPREFETCH_RM ||
10530 instr->type == INSTR_REGPREFETCH_RI)
10533 /* index is the 3rd operand for the regrd instruction and the 2nd
10534 * operand for the regwr and regadd instructions.
10536 if (instr->type == INSTR_REGPREFETCH_RI ||
10537 instr->type == INSTR_REGRD_HRI ||
10538 instr->type == INSTR_REGRD_MRI ||
10539 instr->type == INSTR_REGWR_RIH ||
10540 instr->type == INSTR_REGWR_RIM ||
10541 instr->type == INSTR_REGWR_RII ||
10542 instr->type == INSTR_REGADD_RIH ||
10543 instr->type == INSTR_REGADD_RIM ||
10544 instr->type == INSTR_REGADD_RII)
10547 /* src is the 3rd operand for the regwr and regadd instructions. */
10548 if (instr->type == INSTR_REGWR_RHI ||
10549 instr->type == INSTR_REGWR_RMI ||
10550 instr->type == INSTR_REGWR_RII ||
10551 instr->type == INSTR_REGADD_RHI ||
10552 instr->type == INSTR_REGADD_RMI ||
10553 instr->type == INSTR_REGADD_RII)
10556 /* instr.regarray.regarray_id. */
10559 "\t\t.type = %s,\n"
10560 "\t\t.regarray = {\n"
10561 "\t\t\t.regarray_id = %u,\n",
10562 instr_type_to_name(instr),
10563 instr->regarray.regarray_id);
10565 /* instr.regarray.idx / instr.regarray.idx_val. */
10568 "\t\t\t\t.idx = {\n"
10569 "\t\t\t\t\t.struct_id = %u,\n"
10570 "\t\t\t\t\t.n_bits = %u,\n"
10571 "\t\t\t\t\t.offset = %u,\n"
10573 instr->regarray.idx.struct_id,
10574 instr->regarray.idx.n_bits,
10575 instr->regarray.idx.offset);
10578 "\t\t\t\t.idx_val = %u,\n",
10579 instr->regarray.idx_val);
10581 /* instr.regarray.dstsrc / instr.regarray.dstsrc_val. */
10585 "\t\t\t\t.dstsrc = {\n"
10586 "\t\t\t\t\t.struct_id = %u,\n"
10587 "\t\t\t\t\t.n_bits = %u,\n"
10588 "\t\t\t\t\t.offset = %u,\n"
10590 instr->regarray.dstsrc.struct_id,
10591 instr->regarray.dstsrc.n_bits,
10592 instr->regarray.dstsrc.offset);
10595 "\t\t\t\t.dstsrc_val = %" PRIu64 ",\n",
10596 instr->regarray.dstsrc_val);
10599 /* instr.regarray and instr - closing curly braces. */
10606 instr_meter_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
10608 int prefetch = 0, idx_imm = 0, color_in_imm = 0;
10610 if (instr->type == INSTR_METPREFETCH_H ||
10611 instr->type == INSTR_METPREFETCH_M ||
10612 instr->type == INSTR_METPREFETCH_I)
10616 if (instr->type == INSTR_METPREFETCH_I ||
10617 instr->type == INSTR_METER_IHM ||
10618 instr->type == INSTR_METER_IHI ||
10619 instr->type == INSTR_METER_IMM ||
10620 instr->type == INSTR_METER_IMI)
10623 /* color_in_imm. */
10624 if (instr->type == INSTR_METER_HHI ||
10625 instr->type == INSTR_METER_HMI ||
10626 instr->type == INSTR_METER_MHI ||
10627 instr->type == INSTR_METER_MMI ||
10628 instr->type == INSTR_METER_IHI ||
10629 instr->type == INSTR_METER_IMI)
10632 /* instr.meter.metarray_id. */
10635 "\t\t.type = %s,\n"
10637 "\t\t\t.metarray_id = %u,\n",
10638 instr_type_to_name(instr),
10639 instr->meter.metarray_id);
10641 /* instr.meter.idx / instr.meter.idx_val. */
10645 "\t\t\t\t.struct_id = %u,\n"
10646 "\t\t\t\t.n_bits = %u,\n"
10647 "\t\t\t\t.offset = %u,\n"
10649 instr->meter.idx.struct_id,
10650 instr->meter.idx.n_bits,
10651 instr->meter.idx.offset);
10654 "\t\t\t.idx_val = %u,\n",
10655 instr->meter.idx_val);
10658 /* instr.meter.length. */
10660 "\t\t\t.length = {\n"
10661 "\t\t\t\t.struct_id = %u,\n"
10662 "\t\t\t\t.n_bits = %u,\n"
10663 "\t\t\t\t.offset = %u,\n"
10665 instr->meter.length.struct_id,
10666 instr->meter.length.n_bits,
10667 instr->meter.length.offset);
10669 /* instr.meter.color_in / instr.meter.color_in_val. */
10672 "\t\t\t.color_in = {\n"
10673 "\t\t\t\t.struct_id = %u,\n"
10674 "\t\t\t\t.n_bits = %u,\n"
10675 "\t\t\t\t.offset = %u,\n"
10677 instr->meter.color_in.struct_id,
10678 instr->meter.color_in.n_bits,
10679 instr->meter.color_in.offset);
10682 "\t\t\t.color_in_val = %u,\n",
10683 (uint32_t)instr->meter.color_in_val);
10685 /* instr.meter.color_out. */
10687 "\t\t\t.color_out = {\n"
10688 "\t\t\t\t.struct_id = %u,\n"
10689 "\t\t\t\t.n_bits = %u,\n"
10690 "\t\t\t\t.offset = %u,\n"
10692 instr->meter.color_out.struct_id,
10693 instr->meter.color_out.n_bits,
10694 instr->meter.color_out.offset);
10697 /* instr.meter and instr - closing curly braces. */
10704 instr_table_export(struct instruction *instr,
10709 "\t\t.type = %s,\n"
10711 "\t\t\t.table_id = %u,\n"
10714 instr_type_to_name(instr),
10715 instr->table.table_id);
10719 instr_learn_export(struct instruction *instr, FILE *f)
10723 "\t\t.type = %s,\n"
10725 "\t\t\t\t.action_id = %u,\n"
10728 instr_type_to_name(instr),
10729 instr->learn.action_id);
10733 instr_forget_export(struct instruction *instr, FILE *f)
10737 "\t\t.type = %s,\n"
10739 instr_type_to_name(instr));
10743 instr_extern_export(struct instruction *instr, FILE *f)
10745 if (instr->type == INSTR_EXTERN_OBJ)
10748 "\t\t.type = %s,\n"
10749 "\t\t.ext_obj = {\n"
10750 "\t\t\t.ext_obj_id = %u,\n"
10751 "\t\t\t.func_id = %u,\n"
10754 instr_type_to_name(instr),
10755 instr->ext_obj.ext_obj_id,
10756 instr->ext_obj.func_id);
10760 "\t\t.type = %s,\n"
10761 "\t\t.ext_func = {\n"
10762 "\t\t\t.ext_func_id = %u,\n"
10765 instr_type_to_name(instr),
10766 instr->ext_func.ext_func_id);
10770 instr_jmp_export(struct instruction *instr, FILE *f __rte_unused)
10774 "\t\t.type = %s,\n"
10776 "\t\t\t.ip = NULL,\n",
10777 instr_type_to_name(instr));
10779 switch (instr->type) {
10780 case INSTR_JMP_VALID:
10781 case INSTR_JMP_INVALID:
10783 "\t\t\t.header_id = %u,\n",
10784 instr->jmp.header_id);
10787 case INSTR_JMP_ACTION_HIT:
10788 case INSTR_JMP_ACTION_MISS:
10790 "\t\t\t.action_id = %u,\n",
10791 instr->jmp.action_id);
10795 case INSTR_JMP_EQ_MH:
10796 case INSTR_JMP_EQ_HM:
10797 case INSTR_JMP_EQ_HH:
10798 case INSTR_JMP_NEQ:
10799 case INSTR_JMP_NEQ_MH:
10800 case INSTR_JMP_NEQ_HM:
10801 case INSTR_JMP_NEQ_HH:
10803 case INSTR_JMP_LT_MH:
10804 case INSTR_JMP_LT_HM:
10805 case INSTR_JMP_LT_HH:
10807 case INSTR_JMP_GT_MH:
10808 case INSTR_JMP_GT_HM:
10809 case INSTR_JMP_GT_HH:
10812 "\t\t\t\t.struct_id = %u,\n"
10813 "\t\t\t\t.n_bits = %u,\n"
10814 "\t\t\t\t.offset = %u,\n"
10817 "\t\t\t\t.struct_id = %u,\n"
10818 "\t\t\t\t.n_bits = %u,\n"
10819 "\t\t\t\t.offset = %u,\n"
10821 instr->jmp.a.struct_id,
10822 instr->jmp.a.n_bits,
10823 instr->jmp.a.offset,
10824 instr->jmp.b.struct_id,
10825 instr->jmp.b.n_bits,
10826 instr->jmp.b.offset);
10829 case INSTR_JMP_EQ_I:
10830 case INSTR_JMP_NEQ_I:
10831 case INSTR_JMP_LT_MI:
10832 case INSTR_JMP_LT_HI:
10833 case INSTR_JMP_GT_MI:
10834 case INSTR_JMP_GT_HI:
10837 "\t\t\t\t.struct_id = %u,\n"
10838 "\t\t\t\t.n_bits = %u,\n"
10839 "\t\t\t\t.offset = %u,\n"
10841 "\t\t\t.b_val = %" PRIu64 ",\n",
10842 instr->jmp.a.struct_id,
10843 instr->jmp.a.n_bits,
10844 instr->jmp.a.offset,
10858 instr_return_export(struct instruction *instr,
10863 "\t\t.type = %s,\n",
10864 instr_type_to_name(instr));
10870 static instruction_export_t export_table[] = {
10871 [INSTR_RX] = instr_io_export,
10873 [INSTR_TX] = instr_io_export,
10874 [INSTR_TX_I] = instr_io_export,
10875 [INSTR_DROP] = instr_io_export,
10877 [INSTR_HDR_EXTRACT] = instr_io_export,
10878 [INSTR_HDR_EXTRACT2] = instr_io_export,
10879 [INSTR_HDR_EXTRACT3] = instr_io_export,
10880 [INSTR_HDR_EXTRACT4] = instr_io_export,
10881 [INSTR_HDR_EXTRACT5] = instr_io_export,
10882 [INSTR_HDR_EXTRACT6] = instr_io_export,
10883 [INSTR_HDR_EXTRACT7] = instr_io_export,
10884 [INSTR_HDR_EXTRACT8] = instr_io_export,
10886 [INSTR_HDR_EXTRACT_M] = instr_io_export,
10888 [INSTR_HDR_LOOKAHEAD] = instr_io_export,
10890 [INSTR_HDR_EMIT] = instr_io_export,
10891 [INSTR_HDR_EMIT_TX] = instr_io_export,
10892 [INSTR_HDR_EMIT2_TX] = instr_io_export,
10893 [INSTR_HDR_EMIT3_TX] = instr_io_export,
10894 [INSTR_HDR_EMIT4_TX] = instr_io_export,
10895 [INSTR_HDR_EMIT5_TX] = instr_io_export,
10896 [INSTR_HDR_EMIT6_TX] = instr_io_export,
10897 [INSTR_HDR_EMIT7_TX] = instr_io_export,
10898 [INSTR_HDR_EMIT8_TX] = instr_io_export,
10900 [INSTR_HDR_VALIDATE] = instr_hdr_validate_export,
10901 [INSTR_HDR_INVALIDATE] = instr_hdr_validate_export,
10903 [INSTR_MOV] = instr_mov_export,
10904 [INSTR_MOV_MH] = instr_mov_export,
10905 [INSTR_MOV_HM] = instr_mov_export,
10906 [INSTR_MOV_HH] = instr_mov_export,
10907 [INSTR_MOV_I] = instr_mov_export,
10909 [INSTR_DMA_HT] = instr_dma_ht_export,
10910 [INSTR_DMA_HT2] = instr_dma_ht_export,
10911 [INSTR_DMA_HT3] = instr_dma_ht_export,
10912 [INSTR_DMA_HT4] = instr_dma_ht_export,
10913 [INSTR_DMA_HT5] = instr_dma_ht_export,
10914 [INSTR_DMA_HT6] = instr_dma_ht_export,
10915 [INSTR_DMA_HT7] = instr_dma_ht_export,
10916 [INSTR_DMA_HT8] = instr_dma_ht_export,
10918 [INSTR_ALU_ADD] = instr_alu_export,
10919 [INSTR_ALU_ADD_MH] = instr_alu_export,
10920 [INSTR_ALU_ADD_HM] = instr_alu_export,
10921 [INSTR_ALU_ADD_HH] = instr_alu_export,
10922 [INSTR_ALU_ADD_MI] = instr_alu_export,
10923 [INSTR_ALU_ADD_HI] = instr_alu_export,
10925 [INSTR_ALU_SUB] = instr_alu_export,
10926 [INSTR_ALU_SUB_MH] = instr_alu_export,
10927 [INSTR_ALU_SUB_HM] = instr_alu_export,
10928 [INSTR_ALU_SUB_HH] = instr_alu_export,
10929 [INSTR_ALU_SUB_MI] = instr_alu_export,
10930 [INSTR_ALU_SUB_HI] = instr_alu_export,
10932 [INSTR_ALU_CKADD_FIELD] = instr_alu_export,
10933 [INSTR_ALU_CKADD_STRUCT] = instr_alu_export,
10934 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_export,
10935 [INSTR_ALU_CKSUB_FIELD] = instr_alu_export,
10937 [INSTR_ALU_AND] = instr_alu_export,
10938 [INSTR_ALU_AND_MH] = instr_alu_export,
10939 [INSTR_ALU_AND_HM] = instr_alu_export,
10940 [INSTR_ALU_AND_HH] = instr_alu_export,
10941 [INSTR_ALU_AND_I] = instr_alu_export,
10943 [INSTR_ALU_OR] = instr_alu_export,
10944 [INSTR_ALU_OR_MH] = instr_alu_export,
10945 [INSTR_ALU_OR_HM] = instr_alu_export,
10946 [INSTR_ALU_OR_HH] = instr_alu_export,
10947 [INSTR_ALU_OR_I] = instr_alu_export,
10949 [INSTR_ALU_XOR] = instr_alu_export,
10950 [INSTR_ALU_XOR_MH] = instr_alu_export,
10951 [INSTR_ALU_XOR_HM] = instr_alu_export,
10952 [INSTR_ALU_XOR_HH] = instr_alu_export,
10953 [INSTR_ALU_XOR_I] = instr_alu_export,
10955 [INSTR_ALU_SHL] = instr_alu_export,
10956 [INSTR_ALU_SHL_MH] = instr_alu_export,
10957 [INSTR_ALU_SHL_HM] = instr_alu_export,
10958 [INSTR_ALU_SHL_HH] = instr_alu_export,
10959 [INSTR_ALU_SHL_MI] = instr_alu_export,
10960 [INSTR_ALU_SHL_HI] = instr_alu_export,
10962 [INSTR_ALU_SHR] = instr_alu_export,
10963 [INSTR_ALU_SHR_MH] = instr_alu_export,
10964 [INSTR_ALU_SHR_HM] = instr_alu_export,
10965 [INSTR_ALU_SHR_HH] = instr_alu_export,
10966 [INSTR_ALU_SHR_MI] = instr_alu_export,
10967 [INSTR_ALU_SHR_HI] = instr_alu_export,
10969 [INSTR_REGPREFETCH_RH] = instr_reg_export,
10970 [INSTR_REGPREFETCH_RM] = instr_reg_export,
10971 [INSTR_REGPREFETCH_RI] = instr_reg_export,
10973 [INSTR_REGRD_HRH] = instr_reg_export,
10974 [INSTR_REGRD_HRM] = instr_reg_export,
10975 [INSTR_REGRD_MRH] = instr_reg_export,
10976 [INSTR_REGRD_MRM] = instr_reg_export,
10977 [INSTR_REGRD_HRI] = instr_reg_export,
10978 [INSTR_REGRD_MRI] = instr_reg_export,
10980 [INSTR_REGWR_RHH] = instr_reg_export,
10981 [INSTR_REGWR_RHM] = instr_reg_export,
10982 [INSTR_REGWR_RMH] = instr_reg_export,
10983 [INSTR_REGWR_RMM] = instr_reg_export,
10984 [INSTR_REGWR_RHI] = instr_reg_export,
10985 [INSTR_REGWR_RMI] = instr_reg_export,
10986 [INSTR_REGWR_RIH] = instr_reg_export,
10987 [INSTR_REGWR_RIM] = instr_reg_export,
10988 [INSTR_REGWR_RII] = instr_reg_export,
10990 [INSTR_REGADD_RHH] = instr_reg_export,
10991 [INSTR_REGADD_RHM] = instr_reg_export,
10992 [INSTR_REGADD_RMH] = instr_reg_export,
10993 [INSTR_REGADD_RMM] = instr_reg_export,
10994 [INSTR_REGADD_RHI] = instr_reg_export,
10995 [INSTR_REGADD_RMI] = instr_reg_export,
10996 [INSTR_REGADD_RIH] = instr_reg_export,
10997 [INSTR_REGADD_RIM] = instr_reg_export,
10998 [INSTR_REGADD_RII] = instr_reg_export,
11000 [INSTR_METPREFETCH_H] = instr_meter_export,
11001 [INSTR_METPREFETCH_M] = instr_meter_export,
11002 [INSTR_METPREFETCH_I] = instr_meter_export,
11004 [INSTR_METER_HHM] = instr_meter_export,
11005 [INSTR_METER_HHI] = instr_meter_export,
11006 [INSTR_METER_HMM] = instr_meter_export,
11007 [INSTR_METER_HMI] = instr_meter_export,
11008 [INSTR_METER_MHM] = instr_meter_export,
11009 [INSTR_METER_MHI] = instr_meter_export,
11010 [INSTR_METER_MMM] = instr_meter_export,
11011 [INSTR_METER_MMI] = instr_meter_export,
11012 [INSTR_METER_IHM] = instr_meter_export,
11013 [INSTR_METER_IHI] = instr_meter_export,
11014 [INSTR_METER_IMM] = instr_meter_export,
11015 [INSTR_METER_IMI] = instr_meter_export,
11017 [INSTR_TABLE] = instr_table_export,
11018 [INSTR_TABLE_AF] = instr_table_export,
11019 [INSTR_SELECTOR] = instr_table_export,
11020 [INSTR_LEARNER] = instr_table_export,
11021 [INSTR_LEARNER_AF] = instr_table_export,
11023 [INSTR_LEARNER_LEARN] = instr_learn_export,
11024 [INSTR_LEARNER_FORGET] = instr_forget_export,
11026 [INSTR_EXTERN_OBJ] = instr_extern_export,
11027 [INSTR_EXTERN_FUNC] = instr_extern_export,
11029 [INSTR_JMP] = instr_jmp_export,
11030 [INSTR_JMP_VALID] = instr_jmp_export,
11031 [INSTR_JMP_INVALID] = instr_jmp_export,
11032 [INSTR_JMP_HIT] = instr_jmp_export,
11033 [INSTR_JMP_MISS] = instr_jmp_export,
11034 [INSTR_JMP_ACTION_HIT] = instr_jmp_export,
11035 [INSTR_JMP_ACTION_MISS] = instr_jmp_export,
11037 [INSTR_JMP_EQ] = instr_jmp_export,
11038 [INSTR_JMP_EQ_MH] = instr_jmp_export,
11039 [INSTR_JMP_EQ_HM] = instr_jmp_export,
11040 [INSTR_JMP_EQ_HH] = instr_jmp_export,
11041 [INSTR_JMP_EQ_I] = instr_jmp_export,
11043 [INSTR_JMP_NEQ] = instr_jmp_export,
11044 [INSTR_JMP_NEQ_MH] = instr_jmp_export,
11045 [INSTR_JMP_NEQ_HM] = instr_jmp_export,
11046 [INSTR_JMP_NEQ_HH] = instr_jmp_export,
11047 [INSTR_JMP_NEQ_I] = instr_jmp_export,
11049 [INSTR_JMP_LT] = instr_jmp_export,
11050 [INSTR_JMP_LT_MH] = instr_jmp_export,
11051 [INSTR_JMP_LT_HM] = instr_jmp_export,
11052 [INSTR_JMP_LT_HH] = instr_jmp_export,
11053 [INSTR_JMP_LT_MI] = instr_jmp_export,
11054 [INSTR_JMP_LT_HI] = instr_jmp_export,
11056 [INSTR_JMP_GT] = instr_jmp_export,
11057 [INSTR_JMP_GT_MH] = instr_jmp_export,
11058 [INSTR_JMP_GT_HM] = instr_jmp_export,
11059 [INSTR_JMP_GT_HH] = instr_jmp_export,
11060 [INSTR_JMP_GT_MI] = instr_jmp_export,
11061 [INSTR_JMP_GT_HI] = instr_jmp_export,
11063 [INSTR_RETURN] = instr_return_export,
11067 action_data_codegen(struct action *a, FILE *f)
11072 "static const struct instruction action_%s_instructions[] = {\n",
11075 for (i = 0; i < a->n_instructions; i++) {
11076 struct instruction *instr = &a->instructions[i];
11077 instruction_export_t func = export_table[instr->type];
11082 fprintf(f, "};\n");
11085 static const char *
11086 instr_type_to_func(struct instruction *instr)
11088 switch (instr->type) {
11089 case INSTR_RX: return NULL;
11091 case INSTR_TX: return "__instr_tx_exec";
11092 case INSTR_TX_I: return "__instr_tx_i_exec";
11093 case INSTR_DROP: return "__instr_drop_exec";
11095 case INSTR_HDR_EXTRACT: return "__instr_hdr_extract_exec";
11096 case INSTR_HDR_EXTRACT2: return "__instr_hdr_extract2_exec";
11097 case INSTR_HDR_EXTRACT3: return "__instr_hdr_extract3_exec";
11098 case INSTR_HDR_EXTRACT4: return "__instr_hdr_extract4_exec";
11099 case INSTR_HDR_EXTRACT5: return "__instr_hdr_extract5_exec";
11100 case INSTR_HDR_EXTRACT6: return "__instr_hdr_extract6_exec";
11101 case INSTR_HDR_EXTRACT7: return "__instr_hdr_extract7_exec";
11102 case INSTR_HDR_EXTRACT8: return "__instr_hdr_extract8_exec";
11104 case INSTR_HDR_EXTRACT_M: return "__instr_hdr_extract_m_exec";
11106 case INSTR_HDR_LOOKAHEAD: return "__instr_hdr_lookahead_exec";
11108 case INSTR_HDR_EMIT: return "__instr_hdr_emit_exec";
11109 case INSTR_HDR_EMIT_TX: return "__instr_hdr_emit_tx_exec";
11110 case INSTR_HDR_EMIT2_TX: return "__instr_hdr_emit2_tx_exec";
11111 case INSTR_HDR_EMIT3_TX: return "__instr_hdr_emit3_tx_exec";
11112 case INSTR_HDR_EMIT4_TX: return "__instr_hdr_emit4_tx_exec";
11113 case INSTR_HDR_EMIT5_TX: return "__instr_hdr_emit5_tx_exec";
11114 case INSTR_HDR_EMIT6_TX: return "__instr_hdr_emit6_tx_exec";
11115 case INSTR_HDR_EMIT7_TX: return "__instr_hdr_emit7_tx_exec";
11116 case INSTR_HDR_EMIT8_TX: return "__instr_hdr_emit8_tx_exec";
11118 case INSTR_HDR_VALIDATE: return "__instr_hdr_validate_exec";
11119 case INSTR_HDR_INVALIDATE: return "__instr_hdr_invalidate_exec";
11121 case INSTR_MOV: return "__instr_mov_exec";
11122 case INSTR_MOV_MH: return "__instr_mov_mh_exec";
11123 case INSTR_MOV_HM: return "__instr_mov_hm_exec";
11124 case INSTR_MOV_HH: return "__instr_mov_hh_exec";
11125 case INSTR_MOV_I: return "__instr_mov_i_exec";
11127 case INSTR_DMA_HT: return "__instr_dma_ht_exec";
11128 case INSTR_DMA_HT2: return "__instr_dma_ht2_exec";
11129 case INSTR_DMA_HT3: return "__instr_dma_ht3_exec";
11130 case INSTR_DMA_HT4: return "__instr_dma_ht4_exec";
11131 case INSTR_DMA_HT5: return "__instr_dma_ht5_exec";
11132 case INSTR_DMA_HT6: return "__instr_dma_ht6_exec";
11133 case INSTR_DMA_HT7: return "__instr_dma_ht7_exec";
11134 case INSTR_DMA_HT8: return "__instr_dma_ht8_exec";
11136 case INSTR_ALU_ADD: return "__instr_alu_add_exec";
11137 case INSTR_ALU_ADD_MH: return "__instr_alu_add_mh_exec";
11138 case INSTR_ALU_ADD_HM: return "__instr_alu_add_hm_exec";
11139 case INSTR_ALU_ADD_HH: return "__instr_alu_add_hh_exec";
11140 case INSTR_ALU_ADD_MI: return "__instr_alu_add_mi_exec";
11141 case INSTR_ALU_ADD_HI: return "__instr_alu_add_hi_exec";
11143 case INSTR_ALU_SUB: return "__instr_alu_sub_exec";
11144 case INSTR_ALU_SUB_MH: return "__instr_alu_sub_mh_exec";
11145 case INSTR_ALU_SUB_HM: return "__instr_alu_sub_hm_exec";
11146 case INSTR_ALU_SUB_HH: return "__instr_alu_sub_hh_exec";
11147 case INSTR_ALU_SUB_MI: return "__instr_alu_sub_mi_exec";
11148 case INSTR_ALU_SUB_HI: return "__instr_alu_sub_hi_exec";
11150 case INSTR_ALU_CKADD_FIELD: return "__instr_alu_ckadd_field_exec";
11151 case INSTR_ALU_CKADD_STRUCT20: return "__instr_alu_ckadd_struct20_exec";
11152 case INSTR_ALU_CKADD_STRUCT: return "__instr_alu_ckadd_struct_exec";
11153 case INSTR_ALU_CKSUB_FIELD: return "__instr_alu_cksub_field_exec";
11155 case INSTR_ALU_AND: return "__instr_alu_and_exec";
11156 case INSTR_ALU_AND_MH: return "__instr_alu_and_mh_exec";
11157 case INSTR_ALU_AND_HM: return "__instr_alu_and_hm_exec";
11158 case INSTR_ALU_AND_HH: return "__instr_alu_and_hh_exec";
11159 case INSTR_ALU_AND_I: return "__instr_alu_and_i_exec";
11161 case INSTR_ALU_OR: return "__instr_alu_or_exec";
11162 case INSTR_ALU_OR_MH: return "__instr_alu_or_mh_exec";
11163 case INSTR_ALU_OR_HM: return "__instr_alu_or_hm_exec";
11164 case INSTR_ALU_OR_HH: return "__instr_alu_or_hh_exec";
11165 case INSTR_ALU_OR_I: return "__instr_alu_or_i_exec";
11167 case INSTR_ALU_XOR: return "__instr_alu_xor_exec";
11168 case INSTR_ALU_XOR_MH: return "__instr_alu_xor_mh_exec";
11169 case INSTR_ALU_XOR_HM: return "__instr_alu_xor_hm_exec";
11170 case INSTR_ALU_XOR_HH: return "__instr_alu_xor_hh_exec";
11171 case INSTR_ALU_XOR_I: return "__instr_alu_xor_i_exec";
11173 case INSTR_ALU_SHL: return "__instr_alu_shl_exec";
11174 case INSTR_ALU_SHL_MH: return "__instr_alu_shl_mh_exec";
11175 case INSTR_ALU_SHL_HM: return "__instr_alu_shl_hm_exec";
11176 case INSTR_ALU_SHL_HH: return "__instr_alu_shl_hh_exec";
11177 case INSTR_ALU_SHL_MI: return "__instr_alu_shl_mi_exec";
11178 case INSTR_ALU_SHL_HI: return "__instr_alu_shl_hi_exec";
11180 case INSTR_ALU_SHR: return "__instr_alu_shr_exec";
11181 case INSTR_ALU_SHR_MH: return "__instr_alu_shr_mh_exec";
11182 case INSTR_ALU_SHR_HM: return "__instr_alu_shr_hm_exec";
11183 case INSTR_ALU_SHR_HH: return "__instr_alu_shr_hh_exec";
11184 case INSTR_ALU_SHR_MI: return "__instr_alu_shr_mi_exec";
11185 case INSTR_ALU_SHR_HI: return "__instr_alu_shr_hi_exec";
11187 case INSTR_REGPREFETCH_RH: return "__instr_regprefetch_rh_exec";
11188 case INSTR_REGPREFETCH_RM: return "__instr_regprefetch_rm_exec";
11189 case INSTR_REGPREFETCH_RI: return "__instr_regprefetch_ri_exec";
11191 case INSTR_REGRD_HRH: return "__instr_regrd_hrh_exec";
11192 case INSTR_REGRD_HRM: return "__instr_regrd_hrm_exec";
11193 case INSTR_REGRD_HRI: return "__instr_regrd_hri_exec";
11194 case INSTR_REGRD_MRH: return "__instr_regrd_mrh_exec";
11195 case INSTR_REGRD_MRM: return "__instr_regrd_mrm_exec";
11196 case INSTR_REGRD_MRI: return "__instr_regrd_mri_exec";
11198 case INSTR_REGWR_RHH: return "__instr_regwr_rhh_exec";
11199 case INSTR_REGWR_RHM: return "__instr_regwr_rhm_exec";
11200 case INSTR_REGWR_RHI: return "__instr_regwr_rhi_exec";
11201 case INSTR_REGWR_RMH: return "__instr_regwr_rmh_exec";
11202 case INSTR_REGWR_RMM: return "__instr_regwr_rmm_exec";
11203 case INSTR_REGWR_RMI: return "__instr_regwr_rmi_exec";
11204 case INSTR_REGWR_RIH: return "__instr_regwr_rih_exec";
11205 case INSTR_REGWR_RIM: return "__instr_regwr_rim_exec";
11206 case INSTR_REGWR_RII: return "__instr_regwr_rii_exec";
11208 case INSTR_REGADD_RHH: return "__instr_regadd_rhh_exec";
11209 case INSTR_REGADD_RHM: return "__instr_regadd_rhm_exec";
11210 case INSTR_REGADD_RHI: return "__instr_regadd_rhi_exec";
11211 case INSTR_REGADD_RMH: return "__instr_regadd_rmh_exec";
11212 case INSTR_REGADD_RMM: return "__instr_regadd_rmm_exec";
11213 case INSTR_REGADD_RMI: return "__instr_regadd_rmi_exec";
11214 case INSTR_REGADD_RIH: return "__instr_regadd_rih_exec";
11215 case INSTR_REGADD_RIM: return "__instr_regadd_rim_exec";
11216 case INSTR_REGADD_RII: return "__instr_regadd_rii_exec";
11218 case INSTR_METPREFETCH_H: return "__instr_metprefetch_h_exec";
11219 case INSTR_METPREFETCH_M: return "__instr_metprefetch_m_exec";
11220 case INSTR_METPREFETCH_I: return "__instr_metprefetch_i_exec";
11222 case INSTR_METER_HHM: return "__instr_meter_hhm_exec";
11223 case INSTR_METER_HHI: return "__instr_meter_hhi_exec";
11224 case INSTR_METER_HMM: return "__instr_meter_hmm_exec";
11225 case INSTR_METER_HMI: return "__instr_meter_hmi_exec";
11226 case INSTR_METER_MHM: return "__instr_meter_mhm_exec";
11227 case INSTR_METER_MHI: return "__instr_meter_mhi_exec";
11228 case INSTR_METER_MMM: return "__instr_meter_mmm_exec";
11229 case INSTR_METER_MMI: return "__instr_meter_mmi_exec";
11230 case INSTR_METER_IHM: return "__instr_meter_ihm_exec";
11231 case INSTR_METER_IHI: return "__instr_meter_ihi_exec";
11232 case INSTR_METER_IMM: return "__instr_meter_imm_exec";
11233 case INSTR_METER_IMI: return "__instr_meter_imi_exec";
11235 case INSTR_TABLE: return NULL;
11236 case INSTR_TABLE_AF: return NULL;
11237 case INSTR_SELECTOR: return NULL;
11238 case INSTR_LEARNER: return NULL;
11239 case INSTR_LEARNER_AF: return NULL;
11241 case INSTR_LEARNER_LEARN: return "__instr_learn_exec";
11242 case INSTR_LEARNER_FORGET: return "__instr_forget_exec";
11244 case INSTR_EXTERN_OBJ: return NULL;
11245 case INSTR_EXTERN_FUNC: return NULL;
11247 case INSTR_JMP: return NULL;
11248 case INSTR_JMP_VALID: return NULL;
11249 case INSTR_JMP_INVALID: return NULL;
11250 case INSTR_JMP_HIT: return NULL;
11251 case INSTR_JMP_MISS: return NULL;
11252 case INSTR_JMP_ACTION_HIT: return NULL;
11253 case INSTR_JMP_ACTION_MISS: return NULL;
11254 case INSTR_JMP_EQ: return NULL;
11255 case INSTR_JMP_EQ_MH: return NULL;
11256 case INSTR_JMP_EQ_HM: return NULL;
11257 case INSTR_JMP_EQ_HH: return NULL;
11258 case INSTR_JMP_EQ_I: return NULL;
11259 case INSTR_JMP_NEQ: return NULL;
11260 case INSTR_JMP_NEQ_MH: return NULL;
11261 case INSTR_JMP_NEQ_HM: return NULL;
11262 case INSTR_JMP_NEQ_HH: return NULL;
11263 case INSTR_JMP_NEQ_I: return NULL;
11264 case INSTR_JMP_LT: return NULL;
11265 case INSTR_JMP_LT_MH: return NULL;
11266 case INSTR_JMP_LT_HM: return NULL;
11267 case INSTR_JMP_LT_HH: return NULL;
11268 case INSTR_JMP_LT_MI: return NULL;
11269 case INSTR_JMP_LT_HI: return NULL;
11270 case INSTR_JMP_GT: return NULL;
11271 case INSTR_JMP_GT_MH: return NULL;
11272 case INSTR_JMP_GT_HM: return NULL;
11273 case INSTR_JMP_GT_HH: return NULL;
11274 case INSTR_JMP_GT_MI: return NULL;
11275 case INSTR_JMP_GT_HI: return NULL;
11277 case INSTR_RETURN: return NULL;
11279 default: return NULL;
11284 action_instr_does_tx_codegen(struct action *a,
11285 uint32_t instr_pos,
11286 struct instruction *instr,
11290 "%s(p, t, &action_%s_instructions[%u]);\n"
11291 "\tthread_ip_reset(p, t);\n"
11292 "\tinstr_rx_exec(p);\n"
11294 instr_type_to_func(instr),
11300 action_instr_extern_obj_codegen(struct action *a,
11301 uint32_t instr_pos,
11305 "while (!__instr_extern_obj_exec(p, t, &action_%s_instructions[%u]));\n",
11311 action_instr_extern_func_codegen(struct action *a,
11312 uint32_t instr_pos,
11316 "while (!__instr_extern_func_exec(p, t, &action_%s_instructions[%u]));\n",
11322 action_instr_jmp_codegen(struct action *a,
11323 uint32_t instr_pos,
11324 struct instruction *instr,
11325 struct instruction_data *data,
11328 switch (instr->type) {
11335 case INSTR_JMP_VALID:
11337 "if (HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11344 case INSTR_JMP_INVALID:
11346 "if (!HEADER_VALID(t, action_%s_instructions[%u].jmp.header_id))\n"
11353 case INSTR_JMP_HIT:
11360 case INSTR_JMP_MISS:
11367 case INSTR_JMP_ACTION_HIT:
11369 "if (t->action_id == action_%s_instructions[%u].jmp.action_id)\n"
11376 case INSTR_JMP_ACTION_MISS:
11378 "if (t->action_id != action_%s_instructions[%u].jmp.action_id)\n"
11387 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11388 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11397 case INSTR_JMP_EQ_MH:
11399 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11400 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11409 case INSTR_JMP_EQ_HM:
11411 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11412 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11421 case INSTR_JMP_EQ_HH:
11423 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) == "
11424 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11433 case INSTR_JMP_EQ_I:
11435 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) == "
11436 "action_%s_instructions[%u].jmp.b_val)\n"
11445 case INSTR_JMP_NEQ:
11447 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11448 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11457 case INSTR_JMP_NEQ_MH:
11459 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11460 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11469 case INSTR_JMP_NEQ_HM:
11471 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11472 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11481 case INSTR_JMP_NEQ_HH:
11483 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) != "
11484 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11493 case INSTR_JMP_NEQ_I:
11495 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) != "
11496 "action_%s_instructions[%u].jmp.b_val)\n"
11507 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11508 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11517 case INSTR_JMP_LT_MH:
11519 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11520 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11529 case INSTR_JMP_LT_HM:
11531 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11532 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11541 case INSTR_JMP_LT_HH:
11543 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11544 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11553 case INSTR_JMP_LT_MI:
11555 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) < "
11556 "action_%s_instructions[%u].jmp.b_val)\n"
11565 case INSTR_JMP_LT_HI:
11567 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) < "
11568 "action_%s_instructions[%u].jmp.b_val)\n"
11579 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11580 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11589 case INSTR_JMP_GT_MH:
11591 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11592 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11601 case INSTR_JMP_GT_HM:
11603 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11604 "instr_operand_hbo(t, &action_%s_instructions[%u].jmp.b))\n"
11613 case INSTR_JMP_GT_HH:
11615 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11616 "instr_operand_nbo(t, &action_%s_instructions[%u].jmp.b))\n"
11625 case INSTR_JMP_GT_MI:
11627 "if (instr_operand_hbo(t, &action_%s_instructions[%u].jmp.a) > "
11628 "action_%s_instructions[%u].jmp.b_val)\n"
11637 case INSTR_JMP_GT_HI:
11639 "if (instr_operand_nbo(t, &action_%s_instructions[%u].jmp.a) > "
11640 "action_%s_instructions[%u].jmp.b_val)\n"
11655 action_instr_return_codegen(FILE *f)
11662 action_instr_codegen(struct action *a, FILE *f)
11668 "action_%s_run(struct rte_swx_pipeline *p)\n"
11670 "\tstruct thread *t = &p->threads[p->thread_id];\n"
11674 for (i = 0; i < a->n_instructions; i++) {
11675 struct instruction *instr = &a->instructions[i];
11676 struct instruction_data *data = &a->instruction_data[i];
11678 /* Label, if present. */
11679 if (data->label[0])
11680 fprintf(f, "\n%s : ", data->label);
11682 fprintf(f, "\n\t");
11684 /* TX instruction type. */
11685 if (instruction_does_tx(instr)) {
11686 action_instr_does_tx_codegen(a, i, instr, f);
11690 /* Extern object/function instruction type. */
11691 if (instr->type == INSTR_EXTERN_OBJ) {
11692 action_instr_extern_obj_codegen(a, i, f);
11696 if (instr->type == INSTR_EXTERN_FUNC) {
11697 action_instr_extern_func_codegen(a, i, f);
11701 /* Jump instruction type. */
11702 if (instruction_is_jmp(instr)) {
11703 action_instr_jmp_codegen(a, i, instr, data, f);
11707 /* Return instruction type. */
11708 if (instr->type == INSTR_RETURN) {
11709 action_instr_return_codegen(f);
11713 /* Any other instruction type. */
11715 "%s(p, t, &action_%s_instructions[%u]);\n",
11716 instr_type_to_func(instr),
11721 fprintf(f, "}\n\n");
11724 struct instruction_group {
11725 TAILQ_ENTRY(instruction_group) node;
11729 uint32_t first_instr_id;
11731 uint32_t last_instr_id;
11736 TAILQ_HEAD(instruction_group_list, instruction_group);
11738 static struct instruction_group *
11739 instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
11741 struct instruction_group *g;
11743 TAILQ_FOREACH(g, igl, node)
11744 if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
11751 instruction_group_list_free(struct instruction_group_list *igl)
11757 struct instruction_group *g;
11759 g = TAILQ_FIRST(igl);
11763 TAILQ_REMOVE(igl, g, node);
11770 static struct instruction_group_list *
11771 instruction_group_list_create(struct rte_swx_pipeline *p)
11773 struct instruction_group_list *igl = NULL;
11774 struct instruction_group *g = NULL;
11775 uint32_t n_groups = 0, i;
11777 if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
11781 igl = calloc(1, sizeof(struct instruction_group_list));
11787 /* Allocate the first group. */
11788 g = calloc(1, sizeof(struct instruction_group));
11792 /* Iteration 1: Separate the instructions into groups based on the thread yield
11793 * instructions. Do not worry about the jump instructions at this point.
11795 for (i = 0; i < p->n_instructions; i++) {
11796 struct instruction *instr = &p->instructions[i];
11798 /* Check for thread yield instructions. */
11799 if (!instruction_does_thread_yield(instr))
11802 /* If the current group contains at least one instruction, then finalize it (with
11803 * the previous instruction), add it to the list and allocate a new group (that
11804 * starts with the current instruction).
11806 if (i - g->first_instr_id) {
11807 /* Finalize the group. */
11808 g->last_instr_id = i - 1;
11810 /* Add the group to the list. Advance the number of groups. */
11811 TAILQ_INSERT_TAIL(igl, g, node);
11814 /* Allocate a new group. */
11815 g = calloc(1, sizeof(struct instruction_group));
11819 /* Initialize the new group. */
11820 g->group_id = n_groups;
11821 g->first_instr_id = i;
11824 /* Finalize the current group (with the current instruction, therefore this group
11825 * contains just the current thread yield instruction), add it to the list and
11826 * allocate a new group (that starts with the next instruction).
11829 /* Finalize the group. */
11830 g->last_instr_id = i;
11832 /* Add the group to the list. Advance the number of groups. */
11833 TAILQ_INSERT_TAIL(igl, g, node);
11836 /* Allocate a new group. */
11837 g = calloc(1, sizeof(struct instruction_group));
11841 /* Initialize the new group. */
11842 g->group_id = n_groups;
11843 g->first_instr_id = i + 1;
11846 /* Handle the last group. */
11847 if (i - g->first_instr_id) {
11848 /* Finalize the group. */
11849 g->last_instr_id = i - 1;
11851 /* Add the group to the list. Advance the number of groups. */
11852 TAILQ_INSERT_TAIL(igl, g, node);
11859 /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
11860 * the destination of a jump instruction located in a different group ("far jump"), then the
11861 * current group has to be split, so that the instruction representing the far jump
11862 * destination is at the start of its group.
11865 int is_modified = 0;
11867 for (i = 0; i < p->n_instructions; i++) {
11868 struct instruction_data *data = &p->instruction_data[i];
11869 struct instruction_group *g;
11872 /* Continue when the current instruction is not a jump destination. */
11873 if (!data->n_users)
11876 g = instruction_group_list_group_find(igl, i);
11880 /* Find out all the jump instructions with this destination. */
11881 for (j = 0; j < p->n_instructions; j++) {
11882 struct instruction *jmp_instr = &p->instructions[j];
11883 struct instruction_data *jmp_data = &p->instruction_data[j];
11884 struct instruction_group *jmp_g, *new_g;
11886 /* Continue when not a jump instruction. Even when jump instruction,
11887 * continue when the jump destination is not this instruction.
11889 if (!instruction_is_jmp(jmp_instr) ||
11890 strcmp(jmp_data->jmp_label, data->label))
11893 jmp_g = instruction_group_list_group_find(igl, j);
11897 /* Continue when both the jump instruction and the jump destination
11898 * instruction are in the same group. Even when in different groups,
11899 * still continue if the jump destination instruction is already the
11900 * first instruction of its group.
11902 if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
11905 /* Split the group of the current jump destination instruction to
11906 * make this instruction the first instruction of a new group.
11908 new_g = calloc(1, sizeof(struct instruction_group));
11912 new_g->group_id = n_groups;
11913 new_g->first_instr_id = i;
11914 new_g->last_instr_id = g->last_instr_id;
11916 g->last_instr_id = i - 1;
11918 TAILQ_INSERT_AFTER(igl, g, new_g, node);
11922 /* The decision to split this group (to make the current instruction
11923 * the first instruction of a new group) is already taken and fully
11924 * implemented, so no need to search for more reasons to do it.
11930 /* Re-evaluate everything, as at least one group got split, so some jumps that were
11931 * previously considered local (i.e. the jump destination is in the same group as
11932 * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
11933 * different group than the jump instruction). Wost case scenario: each instruction
11934 * that is a jump destination ends up as the first instruction of its group.
11940 /* Re-assign the group IDs to be in incremental order. */
11942 TAILQ_FOREACH(g, igl, node) {
11951 instruction_group_list_free(igl);
11959 pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
11960 uint32_t instr_pos,
11961 struct instruction *instr,
11965 "%s(p, t, &pipeline_instructions[%u]);\n"
11966 "\tthread_ip_reset(p, t);\n"
11967 "\tinstr_rx_exec(p);\n"
11969 instr_type_to_func(instr),
11974 pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
11975 struct instruction_group_list *igl,
11976 uint32_t jmp_instr_id,
11977 struct instruction *jmp_instr,
11978 struct instruction_data *jmp_data,
11981 struct instruction_group *jmp_g, *g;
11982 struct instruction_data *data;
11985 switch (jmp_instr->type) {
11989 case INSTR_JMP_VALID:
11991 "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
11995 case INSTR_JMP_INVALID:
11997 "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
12001 case INSTR_JMP_HIT:
12006 case INSTR_JMP_MISS:
12011 case INSTR_JMP_ACTION_HIT:
12013 "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
12017 case INSTR_JMP_ACTION_MISS:
12019 "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
12025 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12026 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12031 case INSTR_JMP_EQ_MH:
12033 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12034 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12039 case INSTR_JMP_EQ_HM:
12041 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12042 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12047 case INSTR_JMP_EQ_HH:
12049 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
12050 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12055 case INSTR_JMP_EQ_I:
12057 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
12058 "pipeline_instructions[%u].jmp.b_val)",
12063 case INSTR_JMP_NEQ:
12065 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12066 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12071 case INSTR_JMP_NEQ_MH:
12073 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12074 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12079 case INSTR_JMP_NEQ_HM:
12081 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12082 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12087 case INSTR_JMP_NEQ_HH:
12089 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
12090 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12095 case INSTR_JMP_NEQ_I:
12097 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
12098 "pipeline_instructions[%u].jmp.b_val)",
12105 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12106 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12111 case INSTR_JMP_LT_MH:
12113 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12114 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12119 case INSTR_JMP_LT_HM:
12121 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12122 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12127 case INSTR_JMP_LT_HH:
12129 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12130 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12135 case INSTR_JMP_LT_MI:
12137 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
12138 "pipeline_instructions[%u].jmp.b_val)",
12143 case INSTR_JMP_LT_HI:
12145 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
12146 "pipeline_instructions[%u].jmp.b_val)",
12153 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12154 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12159 case INSTR_JMP_GT_MH:
12161 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12162 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12167 case INSTR_JMP_GT_HM:
12169 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12170 "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
12175 case INSTR_JMP_GT_HH:
12177 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12178 "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
12183 case INSTR_JMP_GT_MI:
12185 "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
12186 "pipeline_instructions[%u].jmp.b_val)",
12191 case INSTR_JMP_GT_HI:
12193 "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
12194 "pipeline_instructions[%u].jmp.b_val)",
12203 /* Find the instruction group of the jump instruction. */
12204 jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
12208 /* Find the instruction group of the jump destination instruction. */
12209 data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
12213 instr_id = data - p->instruction_data;
12215 g = instruction_group_list_group_find(igl, instr_id);
12219 /* Code generation for "near" jump (same instruction group) or "far" jump (different
12220 * instruction group).
12222 if (g->group_id == jmp_g->group_id)
12224 "\n\t\tgoto %s;\n",
12225 jmp_data->jmp_label);
12229 "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
12238 instruction_group_list_codegen(struct instruction_group_list *igl,
12239 struct rte_swx_pipeline *p,
12242 struct instruction_group *g;
12244 int is_required = 0;
12246 /* Check if code generation is required. */
12247 TAILQ_FOREACH(g, igl, node)
12248 if (g->first_instr_id < g->last_instr_id)
12254 /* Generate the code for the pipeline instruction array. */
12256 "static const struct instruction pipeline_instructions[] = {\n");
12258 for (i = 0; i < p->n_instructions; i++) {
12259 struct instruction *instr = &p->instructions[i];
12260 instruction_export_t func = export_table[instr->type];
12265 fprintf(f, "};\n\n");
12267 /* Generate the code for the pipeline functions: one function for each instruction group
12268 * that contains more than one instruction.
12270 TAILQ_FOREACH(g, igl, node) {
12271 struct instruction *last_instr;
12274 /* Skip if group contains a single instruction. */
12275 if (g->last_instr_id == g->first_instr_id)
12278 /* Generate new pipeline function. */
12281 "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
12283 "\tstruct thread *t = &p->threads[p->thread_id];\n"
12287 /* Generate the code for each pipeline instruction. */
12288 for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
12289 struct instruction *instr = &p->instructions[j];
12290 struct instruction_data *data = &p->instruction_data[j];
12292 /* Label, if present. */
12293 if (data->label[0])
12294 fprintf(f, "\n%s : ", data->label);
12296 fprintf(f, "\n\t");
12298 /* TX instruction type. */
12299 if (instruction_does_tx(instr)) {
12300 pipeline_instr_does_tx_codegen(p, j, instr, f);
12304 /* Jump instruction type. */
12305 if (instruction_is_jmp(instr)) {
12306 pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
12310 /* Any other instruction type. */
12312 "%s(p, t, &pipeline_instructions[%u]);\n",
12313 instr_type_to_func(instr),
12317 /* Finalize the generated pipeline function. For some instructions such as TX,
12318 * emit-many-and-TX and unconditional jump, the next instruction has been already
12319 * decided unconditionally and the instruction pointer of the current thread set
12320 * accordingly; for all the other instructions, the instruction pointer must be
12323 last_instr = &p->instructions[g->last_instr_id];
12325 if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
12327 "thread_ip_inc(p);\n");
12336 instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
12338 struct instruction_group *g;
12339 uint32_t n_custom_instr = 0;
12341 /* Groups with a single instruction: no function is generated for this group, the group
12342 * keeps its current instruction. Groups with more than two instructions: one function and
12343 * the associated custom instruction get generated for each such group.
12345 TAILQ_FOREACH(g, igl, node) {
12346 if (g->first_instr_id == g->last_instr_id)
12352 return n_custom_instr;
12356 pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12361 /* Create the .c file. */
12362 f = fopen("/tmp/pipeline.c", "w");
12366 /* Include the .h file. */
12367 fprintf(f, "#include \"rte_swx_pipeline_internal.h\"\n");
12369 /* Add the code for each action. */
12370 TAILQ_FOREACH(a, &p->actions, node) {
12371 fprintf(f, "/**\n * Action %s\n */\n\n", a->name);
12373 action_data_codegen(a, f);
12377 action_instr_codegen(a, f);
12382 /* Add the pipeline code. */
12383 instruction_group_list_codegen(igl, p, f);
12385 /* Close the .c file. */
12391 #ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
12392 #define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
12396 pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12399 struct instruction_group *g;
12400 char *dir_in, *buffer = NULL;
12401 const char *dir_out;
12404 /* Get the environment variables. */
12405 dir_in = getenv("RTE_INSTALL_DIR");
12413 /* Memory allocation for the command buffer. */
12414 buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
12421 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12422 "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
12423 "-I %s/lib/pipeline "
12424 "-I %s/lib/eal/include "
12425 "-I %s/lib/eal/x86/include "
12426 "-I %s/lib/eal/include/generic "
12430 "-I %s/lib/pipeline "
12433 "-I %s/lib/eal/linux/include "
12434 ">%s/pipeline.log 2>&1 "
12436 "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
12437 ">>%s/pipeline.log 2>&1",
12456 /* Build the shared object library. */
12457 status = system(buffer);
12461 /* Open library. */
12463 RTE_SWX_PIPELINE_CMD_MAX_SIZE,
12464 "%s/libpipeline.so",
12467 p->lib = dlopen(buffer, RTLD_LAZY);
12473 /* Get the action function symbols. */
12474 TAILQ_FOREACH(a, &p->actions, node) {
12475 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
12477 p->action_funcs[a->id] = dlsym(p->lib, buffer);
12478 if (!p->action_funcs[a->id]) {
12484 /* Get the pipeline function symbols. */
12485 TAILQ_FOREACH(g, igl, node) {
12486 if (g->first_instr_id == g->last_instr_id)
12489 snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
12491 g->func = dlsym(p->lib, buffer);
12499 if (status && p->lib) {
12510 pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
12511 struct instruction_group_list *igl)
12513 uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
12515 /* Check that enough space is available within the pipeline instruction table to store all
12516 * the custom instructions.
12518 if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
12525 pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
12527 struct instruction_group *g;
12530 /* Pipeline table instructions. */
12531 for (i = 0; i < p->n_instructions; i++) {
12532 struct instruction *instr = &p->instructions[i];
12534 if (instr->type == INSTR_TABLE)
12535 instr->type = INSTR_TABLE_AF;
12537 if (instr->type == INSTR_LEARNER)
12538 instr->type = INSTR_LEARNER_AF;
12541 /* Pipeline custom instructions. */
12543 TAILQ_FOREACH(g, igl, node) {
12544 struct instruction *instr = &p->instructions[g->first_instr_id];
12547 if (g->first_instr_id == g->last_instr_id)
12550 /* Install a new custom instruction. */
12551 p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
12553 /* First instruction of the group: change its type to the new custom instruction. */
12554 instr->type = INSTR_CUSTOM_0 + i;
12556 /* All the subsequent instructions of the group: invalidate. */
12557 for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
12558 struct instruction_data *data = &p->instruction_data[j];
12566 /* Remove the invalidated instructions. */
12567 p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
12569 /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
12570 * instructions that are the only instruction within their group, so they were left
12573 instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
12577 pipeline_compile(struct rte_swx_pipeline *p)
12579 struct instruction_group_list *igl = NULL;
12582 igl = instruction_group_list_create(p);
12588 /* Code generation. */
12589 status = pipeline_codegen(p, igl);
12593 /* Build and load the shared object library. */
12594 status = pipeline_libload(p, igl);
12598 /* Adjust instructions. */
12599 status = pipeline_adjust_check(p, igl);
12603 pipeline_adjust(p, igl);
12606 instruction_group_list_free(igl);