1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
9 #include "rte_swx_pipeline_internal.h"
11 #define CHECK(condition, err_code) \
17 #define CHECK_NAME(name, err_code) \
20 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
23 #define CHECK_INSTRUCTION(instr, err_code) \
26 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
27 RTE_SWX_INSTRUCTION_SIZE), \
33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
35 #include <rte_malloc.h>
38 env_malloc(size_t size, size_t alignment, int numa_node)
40 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
44 env_free(void *start, size_t size __rte_unused)
54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
58 if (numa_available() == -1)
61 start = numa_alloc_onnode(size, numa_node);
65 memset(start, 0, size);
70 env_free(void *start, size_t size)
72 if (numa_available() == -1)
75 numa_free(start, size);
83 static struct struct_type *
84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
86 struct struct_type *elem;
88 TAILQ_FOREACH(elem, &p->struct_types, node)
89 if (strcmp(elem->name, name) == 0)
96 struct_type_field_find(struct struct_type *st, const char *name)
100 for (i = 0; i < st->n_fields; i++) {
101 struct field *f = &st->fields[i];
103 if (strcmp(f->name, name) == 0)
111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
113 struct rte_swx_field_params *fields,
115 int last_field_has_variable_size)
117 struct struct_type *st;
121 CHECK_NAME(name, EINVAL);
122 CHECK(fields, EINVAL);
123 CHECK(n_fields, EINVAL);
125 for (i = 0; i < n_fields; i++) {
126 struct rte_swx_field_params *f = &fields[i];
127 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
130 CHECK_NAME(f->name, EINVAL);
131 CHECK(f->n_bits, EINVAL);
132 CHECK((f->n_bits <= 64) || var_size, EINVAL);
133 CHECK((f->n_bits & 7) == 0, EINVAL);
135 for (j = 0; j < i; j++) {
136 struct rte_swx_field_params *f_prev = &fields[j];
138 CHECK(strcmp(f->name, f_prev->name), EINVAL);
142 CHECK(!struct_type_find(p, name), EEXIST);
144 /* Node allocation. */
145 st = calloc(1, sizeof(struct struct_type));
148 st->fields = calloc(n_fields, sizeof(struct field));
154 /* Node initialization. */
155 strcpy(st->name, name);
156 for (i = 0; i < n_fields; i++) {
157 struct field *dst = &st->fields[i];
158 struct rte_swx_field_params *src = &fields[i];
159 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
161 strcpy(dst->name, src->name);
162 dst->n_bits = src->n_bits;
163 dst->offset = st->n_bits;
164 dst->var_size = var_size;
166 st->n_bits += src->n_bits;
167 st->n_bits_min += var_size ? 0 : src->n_bits;
169 st->n_fields = n_fields;
170 st->var_size = last_field_has_variable_size;
172 /* Node add to tailq. */
173 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
179 struct_build(struct rte_swx_pipeline *p)
183 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
184 struct thread *t = &p->threads[i];
186 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
187 CHECK(t->structs, ENOMEM);
194 struct_build_free(struct rte_swx_pipeline *p)
198 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
199 struct thread *t = &p->threads[i];
207 struct_free(struct rte_swx_pipeline *p)
209 struct_build_free(p);
213 struct struct_type *elem;
215 elem = TAILQ_FIRST(&p->struct_types);
219 TAILQ_REMOVE(&p->struct_types, elem, node);
228 static struct port_in_type *
229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
231 struct port_in_type *elem;
236 TAILQ_FOREACH(elem, &p->port_in_types, node)
237 if (strcmp(elem->name, name) == 0)
244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
246 struct rte_swx_port_in_ops *ops)
248 struct port_in_type *elem;
251 CHECK_NAME(name, EINVAL);
253 CHECK(ops->create, EINVAL);
254 CHECK(ops->free, EINVAL);
255 CHECK(ops->pkt_rx, EINVAL);
256 CHECK(ops->stats_read, EINVAL);
258 CHECK(!port_in_type_find(p, name), EEXIST);
260 /* Node allocation. */
261 elem = calloc(1, sizeof(struct port_in_type));
264 /* Node initialization. */
265 strcpy(elem->name, name);
266 memcpy(&elem->ops, ops, sizeof(*ops));
268 /* Node add to tailq. */
269 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
274 static struct port_in *
275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
277 struct port_in *port;
279 TAILQ_FOREACH(port, &p->ports_in, node)
280 if (port->id == port_id)
287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
289 const char *port_type_name,
292 struct port_in_type *type = NULL;
293 struct port_in *port = NULL;
298 CHECK(!port_in_find(p, port_id), EINVAL);
300 CHECK_NAME(port_type_name, EINVAL);
301 type = port_in_type_find(p, port_type_name);
304 obj = type->ops.create(args);
307 /* Node allocation. */
308 port = calloc(1, sizeof(struct port_in));
311 /* Node initialization. */
316 /* Node add to tailq. */
317 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
318 if (p->n_ports_in < port_id + 1)
319 p->n_ports_in = port_id + 1;
325 port_in_build(struct rte_swx_pipeline *p)
327 struct port_in *port;
330 CHECK(p->n_ports_in, EINVAL);
331 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
333 for (i = 0; i < p->n_ports_in; i++)
334 CHECK(port_in_find(p, i), EINVAL);
336 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
337 CHECK(p->in, ENOMEM);
339 TAILQ_FOREACH(port, &p->ports_in, node) {
340 struct port_in_runtime *in = &p->in[port->id];
342 in->pkt_rx = port->type->ops.pkt_rx;
350 port_in_build_free(struct rte_swx_pipeline *p)
357 port_in_free(struct rte_swx_pipeline *p)
359 port_in_build_free(p);
363 struct port_in *port;
365 port = TAILQ_FIRST(&p->ports_in);
369 TAILQ_REMOVE(&p->ports_in, port, node);
370 port->type->ops.free(port->obj);
374 /* Input port types. */
376 struct port_in_type *elem;
378 elem = TAILQ_FIRST(&p->port_in_types);
382 TAILQ_REMOVE(&p->port_in_types, elem, node);
390 static struct port_out_type *
391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
393 struct port_out_type *elem;
398 TAILQ_FOREACH(elem, &p->port_out_types, node)
399 if (!strcmp(elem->name, name))
406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
408 struct rte_swx_port_out_ops *ops)
410 struct port_out_type *elem;
413 CHECK_NAME(name, EINVAL);
415 CHECK(ops->create, EINVAL);
416 CHECK(ops->free, EINVAL);
417 CHECK(ops->pkt_tx, EINVAL);
418 CHECK(ops->stats_read, EINVAL);
420 CHECK(!port_out_type_find(p, name), EEXIST);
422 /* Node allocation. */
423 elem = calloc(1, sizeof(struct port_out_type));
426 /* Node initialization. */
427 strcpy(elem->name, name);
428 memcpy(&elem->ops, ops, sizeof(*ops));
430 /* Node add to tailq. */
431 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
436 static struct port_out *
437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
439 struct port_out *port;
441 TAILQ_FOREACH(port, &p->ports_out, node)
442 if (port->id == port_id)
449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
451 const char *port_type_name,
454 struct port_out_type *type = NULL;
455 struct port_out *port = NULL;
460 CHECK(!port_out_find(p, port_id), EINVAL);
462 CHECK_NAME(port_type_name, EINVAL);
463 type = port_out_type_find(p, port_type_name);
466 obj = type->ops.create(args);
469 /* Node allocation. */
470 port = calloc(1, sizeof(struct port_out));
473 /* Node initialization. */
478 /* Node add to tailq. */
479 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
480 if (p->n_ports_out < port_id + 1)
481 p->n_ports_out = port_id + 1;
487 port_out_build(struct rte_swx_pipeline *p)
489 struct port_out *port;
492 CHECK(p->n_ports_out, EINVAL);
494 for (i = 0; i < p->n_ports_out; i++)
495 CHECK(port_out_find(p, i), EINVAL);
497 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
498 CHECK(p->out, ENOMEM);
500 TAILQ_FOREACH(port, &p->ports_out, node) {
501 struct port_out_runtime *out = &p->out[port->id];
503 out->pkt_tx = port->type->ops.pkt_tx;
504 out->flush = port->type->ops.flush;
505 out->obj = port->obj;
512 port_out_build_free(struct rte_swx_pipeline *p)
519 port_out_free(struct rte_swx_pipeline *p)
521 port_out_build_free(p);
525 struct port_out *port;
527 port = TAILQ_FIRST(&p->ports_out);
531 TAILQ_REMOVE(&p->ports_out, port, node);
532 port->type->ops.free(port->obj);
536 /* Output port types. */
538 struct port_out_type *elem;
540 elem = TAILQ_FIRST(&p->port_out_types);
544 TAILQ_REMOVE(&p->port_out_types, elem, node);
552 static struct extern_type *
553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
555 struct extern_type *elem;
557 TAILQ_FOREACH(elem, &p->extern_types, node)
558 if (strcmp(elem->name, name) == 0)
564 static struct extern_type_member_func *
565 extern_type_member_func_find(struct extern_type *type, const char *name)
567 struct extern_type_member_func *elem;
569 TAILQ_FOREACH(elem, &type->funcs, node)
570 if (strcmp(elem->name, name) == 0)
576 static struct extern_obj *
577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
579 struct extern_obj *elem;
581 TAILQ_FOREACH(elem, &p->extern_objs, node)
582 if (strcmp(elem->name, name) == 0)
588 static struct extern_type_member_func *
589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
591 struct extern_obj **obj)
593 struct extern_obj *object;
594 struct extern_type_member_func *func;
595 char *object_name, *func_name;
597 if (name[0] != 'e' || name[1] != '.')
600 object_name = strdup(&name[2]);
604 func_name = strchr(object_name, '.');
613 object = extern_obj_find(p, object_name);
619 func = extern_type_member_func_find(object->type, func_name);
632 static struct field *
633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
635 struct extern_obj **object)
637 struct extern_obj *obj;
639 char *obj_name, *field_name;
641 if ((name[0] != 'e') || (name[1] != '.'))
644 obj_name = strdup(&name[2]);
648 field_name = strchr(obj_name, '.');
657 obj = extern_obj_find(p, obj_name);
663 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
679 const char *mailbox_struct_type_name,
680 rte_swx_extern_type_constructor_t constructor,
681 rte_swx_extern_type_destructor_t destructor)
683 struct extern_type *elem;
684 struct struct_type *mailbox_struct_type;
688 CHECK_NAME(name, EINVAL);
689 CHECK(!extern_type_find(p, name), EEXIST);
691 CHECK_NAME(mailbox_struct_type_name, EINVAL);
692 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
693 CHECK(mailbox_struct_type, EINVAL);
694 CHECK(!mailbox_struct_type->var_size, EINVAL);
696 CHECK(constructor, EINVAL);
697 CHECK(destructor, EINVAL);
699 /* Node allocation. */
700 elem = calloc(1, sizeof(struct extern_type));
703 /* Node initialization. */
704 strcpy(elem->name, name);
705 elem->mailbox_struct_type = mailbox_struct_type;
706 elem->constructor = constructor;
707 elem->destructor = destructor;
708 TAILQ_INIT(&elem->funcs);
710 /* Node add to tailq. */
711 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
718 const char *extern_type_name,
720 rte_swx_extern_type_member_func_t member_func)
722 struct extern_type *type;
723 struct extern_type_member_func *type_member;
727 CHECK_NAME(extern_type_name, EINVAL);
728 type = extern_type_find(p, extern_type_name);
730 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
732 CHECK_NAME(name, EINVAL);
733 CHECK(!extern_type_member_func_find(type, name), EEXIST);
735 CHECK(member_func, EINVAL);
737 /* Node allocation. */
738 type_member = calloc(1, sizeof(struct extern_type_member_func));
739 CHECK(type_member, ENOMEM);
741 /* Node initialization. */
742 strcpy(type_member->name, name);
743 type_member->func = member_func;
744 type_member->id = type->n_funcs;
746 /* Node add to tailq. */
747 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
755 const char *extern_type_name,
759 struct extern_type *type;
760 struct extern_obj *obj;
765 CHECK_NAME(extern_type_name, EINVAL);
766 type = extern_type_find(p, extern_type_name);
769 CHECK_NAME(name, EINVAL);
770 CHECK(!extern_obj_find(p, name), EEXIST);
772 /* Node allocation. */
773 obj = calloc(1, sizeof(struct extern_obj));
776 /* Object construction. */
777 obj_handle = type->constructor(args);
783 /* Node initialization. */
784 strcpy(obj->name, name);
786 obj->obj = obj_handle;
787 obj->struct_id = p->n_structs;
788 obj->id = p->n_extern_objs;
790 /* Node add to tailq. */
791 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
799 extern_obj_build(struct rte_swx_pipeline *p)
803 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
804 struct thread *t = &p->threads[i];
805 struct extern_obj *obj;
807 t->extern_objs = calloc(p->n_extern_objs,
808 sizeof(struct extern_obj_runtime));
809 CHECK(t->extern_objs, ENOMEM);
811 TAILQ_FOREACH(obj, &p->extern_objs, node) {
812 struct extern_obj_runtime *r =
813 &t->extern_objs[obj->id];
814 struct extern_type_member_func *func;
815 uint32_t mailbox_size =
816 obj->type->mailbox_struct_type->n_bits / 8;
820 r->mailbox = calloc(1, mailbox_size);
821 CHECK(r->mailbox, ENOMEM);
823 TAILQ_FOREACH(func, &obj->type->funcs, node)
824 r->funcs[func->id] = func->func;
826 t->structs[obj->struct_id] = r->mailbox;
834 extern_obj_build_free(struct rte_swx_pipeline *p)
838 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
839 struct thread *t = &p->threads[i];
845 for (j = 0; j < p->n_extern_objs; j++) {
846 struct extern_obj_runtime *r = &t->extern_objs[j];
851 free(t->extern_objs);
852 t->extern_objs = NULL;
857 extern_obj_free(struct rte_swx_pipeline *p)
859 extern_obj_build_free(p);
861 /* Extern objects. */
863 struct extern_obj *elem;
865 elem = TAILQ_FIRST(&p->extern_objs);
869 TAILQ_REMOVE(&p->extern_objs, elem, node);
871 elem->type->destructor(elem->obj);
877 struct extern_type *elem;
879 elem = TAILQ_FIRST(&p->extern_types);
883 TAILQ_REMOVE(&p->extern_types, elem, node);
886 struct extern_type_member_func *func;
888 func = TAILQ_FIRST(&elem->funcs);
892 TAILQ_REMOVE(&elem->funcs, func, node);
903 static struct extern_func *
904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
906 struct extern_func *elem;
908 TAILQ_FOREACH(elem, &p->extern_funcs, node)
909 if (strcmp(elem->name, name) == 0)
915 static struct extern_func *
916 extern_func_parse(struct rte_swx_pipeline *p,
919 if (name[0] != 'f' || name[1] != '.')
922 return extern_func_find(p, &name[2]);
925 static struct field *
926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
928 struct extern_func **function)
930 struct extern_func *func;
932 char *func_name, *field_name;
934 if ((name[0] != 'f') || (name[1] != '.'))
937 func_name = strdup(&name[2]);
941 field_name = strchr(func_name, '.');
950 func = extern_func_find(p, func_name);
956 f = struct_type_field_find(func->mailbox_struct_type, field_name);
970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
972 const char *mailbox_struct_type_name,
973 rte_swx_extern_func_t func)
975 struct extern_func *f;
976 struct struct_type *mailbox_struct_type;
980 CHECK_NAME(name, EINVAL);
981 CHECK(!extern_func_find(p, name), EEXIST);
983 CHECK_NAME(mailbox_struct_type_name, EINVAL);
984 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
985 CHECK(mailbox_struct_type, EINVAL);
986 CHECK(!mailbox_struct_type->var_size, EINVAL);
990 /* Node allocation. */
991 f = calloc(1, sizeof(struct extern_func));
994 /* Node initialization. */
995 strcpy(f->name, name);
996 f->mailbox_struct_type = mailbox_struct_type;
998 f->struct_id = p->n_structs;
999 f->id = p->n_extern_funcs;
1001 /* Node add to tailq. */
1002 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003 p->n_extern_funcs++;
1010 extern_func_build(struct rte_swx_pipeline *p)
1014 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015 struct thread *t = &p->threads[i];
1016 struct extern_func *func;
1018 /* Memory allocation. */
1019 t->extern_funcs = calloc(p->n_extern_funcs,
1020 sizeof(struct extern_func_runtime));
1021 CHECK(t->extern_funcs, ENOMEM);
1023 /* Extern function. */
1024 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025 struct extern_func_runtime *r =
1026 &t->extern_funcs[func->id];
1027 uint32_t mailbox_size =
1028 func->mailbox_struct_type->n_bits / 8;
1030 r->func = func->func;
1032 r->mailbox = calloc(1, mailbox_size);
1033 CHECK(r->mailbox, ENOMEM);
1035 t->structs[func->struct_id] = r->mailbox;
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1047 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048 struct thread *t = &p->threads[i];
1051 if (!t->extern_funcs)
1054 for (j = 0; j < p->n_extern_funcs; j++) {
1055 struct extern_func_runtime *r = &t->extern_funcs[j];
1060 free(t->extern_funcs);
1061 t->extern_funcs = NULL;
1066 extern_func_free(struct rte_swx_pipeline *p)
1068 extern_func_build_free(p);
1071 struct extern_func *elem;
1073 elem = TAILQ_FIRST(&p->extern_funcs);
1077 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1088 struct header *elem;
1090 TAILQ_FOREACH(elem, &p->headers, node)
1091 if (strcmp(elem->name, name) == 0)
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 struct header *elem;
1102 TAILQ_FOREACH(elem, &p->headers, node)
1103 if (elem->struct_id == struct_id)
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1113 if (name[0] != 'h' || name[1] != '.')
1116 return header_find(p, &name[2]);
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1122 struct header **header)
1126 char *header_name, *field_name;
1128 if ((name[0] != 'h') || (name[1] != '.'))
1131 header_name = strdup(&name[2]);
1135 field_name = strchr(header_name, '.');
1144 h = header_find(p, header_name);
1150 f = struct_type_field_find(h->st, field_name);
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166 const char *struct_type_name)
1168 struct struct_type *st;
1170 size_t n_headers_max;
1173 CHECK_NAME(name, EINVAL);
1174 CHECK_NAME(struct_type_name, EINVAL);
1176 CHECK(!header_find(p, name), EEXIST);
1178 st = struct_type_find(p, struct_type_name);
1181 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182 CHECK(p->n_headers < n_headers_max, ENOSPC);
1184 /* Node allocation. */
1185 h = calloc(1, sizeof(struct header));
1188 /* Node initialization. */
1189 strcpy(h->name, name);
1191 h->struct_id = p->n_structs;
1192 h->id = p->n_headers;
1194 /* Node add to tailq. */
1195 TAILQ_INSERT_TAIL(&p->headers, h, node);
1203 header_build(struct rte_swx_pipeline *p)
1206 uint32_t n_bytes = 0, i;
1208 TAILQ_FOREACH(h, &p->headers, node) {
1209 n_bytes += h->st->n_bits / 8;
1212 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213 struct thread *t = &p->threads[i];
1214 uint32_t offset = 0;
1216 t->headers = calloc(p->n_headers,
1217 sizeof(struct header_runtime));
1218 CHECK(t->headers, ENOMEM);
1220 t->headers_out = calloc(p->n_headers,
1221 sizeof(struct header_out_runtime));
1222 CHECK(t->headers_out, ENOMEM);
1224 t->header_storage = calloc(1, n_bytes);
1225 CHECK(t->header_storage, ENOMEM);
1227 t->header_out_storage = calloc(1, n_bytes);
1228 CHECK(t->header_out_storage, ENOMEM);
1230 TAILQ_FOREACH(h, &p->headers, node) {
1231 uint8_t *header_storage;
1232 uint32_t n_bytes = h->st->n_bits / 8;
1234 header_storage = &t->header_storage[offset];
1237 t->headers[h->id].ptr0 = header_storage;
1238 t->headers[h->id].n_bytes = n_bytes;
1240 t->structs[h->struct_id] = header_storage;
1248 header_build_free(struct rte_swx_pipeline *p)
1252 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253 struct thread *t = &p->threads[i];
1255 free(t->headers_out);
1256 t->headers_out = NULL;
1261 free(t->header_out_storage);
1262 t->header_out_storage = NULL;
1264 free(t->header_storage);
1265 t->header_storage = NULL;
1270 header_free(struct rte_swx_pipeline *p)
1272 header_build_free(p);
1275 struct header *elem;
1277 elem = TAILQ_FIRST(&p->headers);
1281 TAILQ_REMOVE(&p->headers, elem, node);
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 if (!p->metadata_st)
1295 if (name[0] != 'm' || name[1] != '.')
1298 return struct_type_field_find(p->metadata_st, &name[2]);
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303 const char *struct_type_name)
1305 struct struct_type *st = NULL;
1309 CHECK_NAME(struct_type_name, EINVAL);
1310 st = struct_type_find(p, struct_type_name);
1312 CHECK(!st->var_size, EINVAL);
1313 CHECK(!p->metadata_st, EINVAL);
1315 p->metadata_st = st;
1316 p->metadata_struct_id = p->n_structs;
1324 metadata_build(struct rte_swx_pipeline *p)
1326 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1329 /* Thread-level initialization. */
1330 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331 struct thread *t = &p->threads[i];
1334 metadata = calloc(1, n_bytes);
1335 CHECK(metadata, ENOMEM);
1337 t->metadata = metadata;
1338 t->structs[p->metadata_struct_id] = metadata;
1345 metadata_build_free(struct rte_swx_pipeline *p)
1349 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350 struct thread *t = &p->threads[i];
1358 metadata_free(struct rte_swx_pipeline *p)
1360 metadata_build_free(p);
1367 instruction_is_tx(enum instruction_type type)
1380 instruction_is_jmp(struct instruction *instr)
1382 switch (instr->type) {
1384 case INSTR_JMP_VALID:
1385 case INSTR_JMP_INVALID:
1387 case INSTR_JMP_MISS:
1388 case INSTR_JMP_ACTION_HIT:
1389 case INSTR_JMP_ACTION_MISS:
1391 case INSTR_JMP_EQ_MH:
1392 case INSTR_JMP_EQ_HM:
1393 case INSTR_JMP_EQ_HH:
1394 case INSTR_JMP_EQ_I:
1396 case INSTR_JMP_NEQ_MH:
1397 case INSTR_JMP_NEQ_HM:
1398 case INSTR_JMP_NEQ_HH:
1399 case INSTR_JMP_NEQ_I:
1401 case INSTR_JMP_LT_MH:
1402 case INSTR_JMP_LT_HM:
1403 case INSTR_JMP_LT_HH:
1404 case INSTR_JMP_LT_MI:
1405 case INSTR_JMP_LT_HI:
1407 case INSTR_JMP_GT_MH:
1408 case INSTR_JMP_GT_HM:
1409 case INSTR_JMP_GT_HH:
1410 case INSTR_JMP_GT_MI:
1411 case INSTR_JMP_GT_HI:
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424 struct action *action,
1426 uint32_t *struct_id)
1433 struct header *header;
1435 f = header_field_parse(p, name, &header);
1439 *struct_id = header->struct_id;
1445 f = metadata_field_parse(p, name);
1449 *struct_id = p->metadata_struct_id;
1458 f = action_field_parse(action, name);
1468 struct extern_obj *obj;
1470 f = extern_obj_mailbox_field_parse(p, name, &obj);
1474 *struct_id = obj->struct_id;
1480 struct extern_func *func;
1482 f = extern_func_mailbox_field_parse(p, name, &func);
1486 *struct_id = func->struct_id;
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500 struct action *action,
1503 struct instruction *instr,
1504 struct instruction_data *data __rte_unused)
1508 CHECK(!action, EINVAL);
1509 CHECK(n_tokens == 2, EINVAL);
1511 f = metadata_field_parse(p, tokens[1]);
1514 instr->type = INSTR_RX;
1515 instr->io.io.offset = f->offset / 8;
1516 instr->io.io.n_bits = f->n_bits;
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525 struct action *action __rte_unused,
1528 struct instruction *instr,
1529 struct instruction_data *data __rte_unused)
1531 char *port = tokens[1];
1535 CHECK(n_tokens == 2, EINVAL);
1537 f = metadata_field_parse(p, port);
1539 instr->type = INSTR_TX;
1540 instr->io.io.offset = f->offset / 8;
1541 instr->io.io.n_bits = f->n_bits;
1546 port_val = strtoul(port, &port, 0);
1547 CHECK(!port[0], EINVAL);
1549 instr->type = INSTR_TX_I;
1550 instr->io.io.val = port_val;
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556 struct action *action __rte_unused,
1557 char **tokens __rte_unused,
1559 struct instruction *instr,
1560 struct instruction_data *data __rte_unused)
1562 CHECK(n_tokens == 1, EINVAL);
1565 instr->type = INSTR_TX_I;
1566 instr->io.io.val = p->n_ports_out - 1;
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1573 struct thread *t = &p->threads[p->thread_id];
1574 struct instruction *ip = t->ip;
1576 __instr_tx_exec(p, t, ip);
1579 thread_ip_reset(p, t);
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1586 struct thread *t = &p->threads[p->thread_id];
1587 struct instruction *ip = t->ip;
1589 __instr_tx_i_exec(p, t, ip);
1592 thread_ip_reset(p, t);
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601 struct action *action,
1604 struct instruction *instr,
1605 struct instruction_data *data __rte_unused)
1609 CHECK(!action, EINVAL);
1610 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1612 h = header_parse(p, tokens[1]);
1615 if (n_tokens == 2) {
1616 CHECK(!h->st->var_size, EINVAL);
1618 instr->type = INSTR_HDR_EXTRACT;
1619 instr->io.hdr.header_id[0] = h->id;
1620 instr->io.hdr.struct_id[0] = h->struct_id;
1621 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1625 CHECK(h->st->var_size, EINVAL);
1627 mf = metadata_field_parse(p, tokens[2]);
1629 CHECK(!mf->var_size, EINVAL);
1631 instr->type = INSTR_HDR_EXTRACT_M;
1632 instr->io.io.offset = mf->offset / 8;
1633 instr->io.io.n_bits = mf->n_bits;
1634 instr->io.hdr.header_id[0] = h->id;
1635 instr->io.hdr.struct_id[0] = h->struct_id;
1636 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644 struct action *action,
1647 struct instruction *instr,
1648 struct instruction_data *data __rte_unused)
1652 CHECK(!action, EINVAL);
1653 CHECK(n_tokens == 2, EINVAL);
1655 h = header_parse(p, tokens[1]);
1657 CHECK(!h->st->var_size, EINVAL);
1659 instr->type = INSTR_HDR_LOOKAHEAD;
1660 instr->io.hdr.header_id[0] = h->id;
1661 instr->io.hdr.struct_id[0] = h->struct_id;
1662 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1670 struct thread *t = &p->threads[p->thread_id];
1671 struct instruction *ip = t->ip;
1673 __instr_hdr_extract_exec(p, t, ip);
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1682 struct thread *t = &p->threads[p->thread_id];
1683 struct instruction *ip = t->ip;
1685 __instr_hdr_extract2_exec(p, t, ip);
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1694 struct thread *t = &p->threads[p->thread_id];
1695 struct instruction *ip = t->ip;
1697 __instr_hdr_extract3_exec(p, t, ip);
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1706 struct thread *t = &p->threads[p->thread_id];
1707 struct instruction *ip = t->ip;
1709 __instr_hdr_extract4_exec(p, t, ip);
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1718 struct thread *t = &p->threads[p->thread_id];
1719 struct instruction *ip = t->ip;
1721 __instr_hdr_extract5_exec(p, t, ip);
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1730 struct thread *t = &p->threads[p->thread_id];
1731 struct instruction *ip = t->ip;
1733 __instr_hdr_extract6_exec(p, t, ip);
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1742 struct thread *t = &p->threads[p->thread_id];
1743 struct instruction *ip = t->ip;
1745 __instr_hdr_extract7_exec(p, t, ip);
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1754 struct thread *t = &p->threads[p->thread_id];
1755 struct instruction *ip = t->ip;
1757 __instr_hdr_extract8_exec(p, t, ip);
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1766 struct thread *t = &p->threads[p->thread_id];
1767 struct instruction *ip = t->ip;
1769 __instr_hdr_extract_m_exec(p, t, ip);
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1778 struct thread *t = &p->threads[p->thread_id];
1779 struct instruction *ip = t->ip;
1781 __instr_hdr_lookahead_exec(p, t, ip);
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792 struct action *action __rte_unused,
1795 struct instruction *instr,
1796 struct instruction_data *data __rte_unused)
1800 CHECK(n_tokens == 2, EINVAL);
1802 h = header_parse(p, tokens[1]);
1805 instr->type = INSTR_HDR_EMIT;
1806 instr->io.hdr.header_id[0] = h->id;
1807 instr->io.hdr.struct_id[0] = h->struct_id;
1808 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1815 struct thread *t = &p->threads[p->thread_id];
1816 struct instruction *ip = t->ip;
1818 __instr_hdr_emit_exec(p, t, ip);
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1827 struct thread *t = &p->threads[p->thread_id];
1828 struct instruction *ip = t->ip;
1830 __instr_hdr_emit_tx_exec(p, t, ip);
1833 thread_ip_reset(p, t);
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1840 struct thread *t = &p->threads[p->thread_id];
1841 struct instruction *ip = t->ip;
1843 __instr_hdr_emit2_tx_exec(p, t, ip);
1846 thread_ip_reset(p, t);
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1853 struct thread *t = &p->threads[p->thread_id];
1854 struct instruction *ip = t->ip;
1856 __instr_hdr_emit3_tx_exec(p, t, ip);
1859 thread_ip_reset(p, t);
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1866 struct thread *t = &p->threads[p->thread_id];
1867 struct instruction *ip = t->ip;
1869 __instr_hdr_emit4_tx_exec(p, t, ip);
1872 thread_ip_reset(p, t);
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1879 struct thread *t = &p->threads[p->thread_id];
1880 struct instruction *ip = t->ip;
1882 __instr_hdr_emit5_tx_exec(p, t, ip);
1885 thread_ip_reset(p, t);
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1892 struct thread *t = &p->threads[p->thread_id];
1893 struct instruction *ip = t->ip;
1895 __instr_hdr_emit6_tx_exec(p, t, ip);
1898 thread_ip_reset(p, t);
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1905 struct thread *t = &p->threads[p->thread_id];
1906 struct instruction *ip = t->ip;
1908 __instr_hdr_emit7_tx_exec(p, t, ip);
1911 thread_ip_reset(p, t);
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1918 struct thread *t = &p->threads[p->thread_id];
1919 struct instruction *ip = t->ip;
1921 __instr_hdr_emit8_tx_exec(p, t, ip);
1924 thread_ip_reset(p, t);
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933 struct action *action __rte_unused,
1936 struct instruction *instr,
1937 struct instruction_data *data __rte_unused)
1941 CHECK(n_tokens == 2, EINVAL);
1943 h = header_parse(p, tokens[1]);
1946 instr->type = INSTR_HDR_VALIDATE;
1947 instr->valid.header_id = h->id;
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1954 struct thread *t = &p->threads[p->thread_id];
1955 struct instruction *ip = t->ip;
1957 __instr_hdr_validate_exec(p, t, ip);
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968 struct action *action __rte_unused,
1971 struct instruction *instr,
1972 struct instruction_data *data __rte_unused)
1976 CHECK(n_tokens == 2, EINVAL);
1978 h = header_parse(p, tokens[1]);
1981 instr->type = INSTR_HDR_INVALIDATE;
1982 instr->valid.header_id = h->id;
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1989 struct thread *t = &p->threads[p->thread_id];
1990 struct instruction *ip = t->ip;
1992 __instr_hdr_invalidate_exec(p, t, ip);
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012 struct action *action,
2015 struct instruction *instr,
2016 struct instruction_data *data __rte_unused)
2022 CHECK(!action, EINVAL);
2023 CHECK(n_tokens == 2, EINVAL);
2025 t = table_find(p, tokens[1]);
2027 instr->type = INSTR_TABLE;
2028 instr->table.table_id = t->id;
2032 s = selector_find(p, tokens[1]);
2034 instr->type = INSTR_SELECTOR;
2035 instr->table.table_id = s->id;
2039 l = learner_find(p, tokens[1]);
2041 instr->type = INSTR_LEARNER;
2042 instr->table.table_id = l->id;
2050 instr_table_exec(struct rte_swx_pipeline *p)
2052 struct thread *t = &p->threads[p->thread_id];
2053 struct instruction *ip = t->ip;
2054 uint32_t table_id = ip->table.table_id;
2055 struct rte_swx_table_state *ts = &t->table_state[table_id];
2056 struct table_runtime *table = &t->tables[table_id];
2057 struct table_statistics *stats = &p->table_stats[table_id];
2058 uint64_t action_id, n_pkts_hit, n_pkts_action;
2059 uint8_t *action_data;
2063 done = table->func(ts->obj,
2071 TRACE("[Thread %2u] table %u (not finalized)\n",
2079 action_id = hit ? action_id : ts->default_action_id;
2080 action_data = hit ? action_data : ts->default_action_data;
2081 n_pkts_hit = stats->n_pkts_hit[hit];
2082 n_pkts_action = stats->n_pkts_action[action_id];
2084 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2087 hit ? "hit" : "miss",
2088 (uint32_t)action_id);
2090 t->action_id = action_id;
2091 t->structs[0] = action_data;
2093 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2097 thread_ip_action_call(p, t, action_id);
2101 instr_table_af_exec(struct rte_swx_pipeline *p)
2103 struct thread *t = &p->threads[p->thread_id];
2104 struct instruction *ip = t->ip;
2105 uint32_t table_id = ip->table.table_id;
2106 struct rte_swx_table_state *ts = &t->table_state[table_id];
2107 struct table_runtime *table = &t->tables[table_id];
2108 struct table_statistics *stats = &p->table_stats[table_id];
2109 uint64_t action_id, n_pkts_hit, n_pkts_action;
2110 uint8_t *action_data;
2111 action_func_t action_func;
2115 done = table->func(ts->obj,
2123 TRACE("[Thread %2u] table %u (not finalized)\n",
2131 action_id = hit ? action_id : ts->default_action_id;
2132 action_data = hit ? action_data : ts->default_action_data;
2133 action_func = p->action_funcs[action_id];
2134 n_pkts_hit = stats->n_pkts_hit[hit];
2135 n_pkts_action = stats->n_pkts_action[action_id];
2137 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2140 hit ? "hit" : "miss",
2141 (uint32_t)action_id);
2143 t->action_id = action_id;
2144 t->structs[0] = action_data;
2146 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2147 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2157 instr_selector_exec(struct rte_swx_pipeline *p)
2159 struct thread *t = &p->threads[p->thread_id];
2160 struct instruction *ip = t->ip;
2161 uint32_t selector_id = ip->table.table_id;
2162 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2163 struct selector_runtime *selector = &t->selectors[selector_id];
2164 struct selector_statistics *stats = &p->selector_stats[selector_id];
2165 uint64_t n_pkts = stats->n_pkts;
2169 done = rte_swx_table_selector_select(ts->obj,
2171 selector->group_id_buffer,
2172 selector->selector_buffer,
2173 selector->member_id_buffer);
2176 TRACE("[Thread %2u] selector %u (not finalized)\n",
2185 TRACE("[Thread %2u] selector %u\n",
2189 stats->n_pkts = n_pkts + 1;
2196 instr_learner_exec(struct rte_swx_pipeline *p)
2198 struct thread *t = &p->threads[p->thread_id];
2199 struct instruction *ip = t->ip;
2200 uint32_t learner_id = ip->table.table_id;
2201 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2202 p->n_selectors + learner_id];
2203 struct learner_runtime *l = &t->learners[learner_id];
2204 struct learner_statistics *stats = &p->learner_stats[learner_id];
2205 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2206 uint8_t *action_data;
2210 time = rte_get_tsc_cycles();
2212 done = rte_swx_table_learner_lookup(ts->obj,
2221 TRACE("[Thread %2u] learner %u (not finalized)\n",
2229 action_id = hit ? action_id : ts->default_action_id;
2230 action_data = hit ? action_data : ts->default_action_data;
2231 n_pkts_hit = stats->n_pkts_hit[hit];
2232 n_pkts_action = stats->n_pkts_action[action_id];
2234 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2237 hit ? "hit" : "miss",
2238 (uint32_t)action_id);
2240 t->action_id = action_id;
2241 t->structs[0] = action_data;
2243 t->learner_id = learner_id;
2245 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2246 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2249 thread_ip_action_call(p, t, action_id);
2253 instr_learner_af_exec(struct rte_swx_pipeline *p)
2255 struct thread *t = &p->threads[p->thread_id];
2256 struct instruction *ip = t->ip;
2257 uint32_t learner_id = ip->table.table_id;
2258 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2259 p->n_selectors + learner_id];
2260 struct learner_runtime *l = &t->learners[learner_id];
2261 struct learner_statistics *stats = &p->learner_stats[learner_id];
2262 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2263 uint8_t *action_data;
2264 action_func_t action_func;
2268 time = rte_get_tsc_cycles();
2270 done = rte_swx_table_learner_lookup(ts->obj,
2279 TRACE("[Thread %2u] learner %u (not finalized)\n",
2287 action_id = hit ? action_id : ts->default_action_id;
2288 action_data = hit ? action_data : ts->default_action_data;
2289 action_func = p->action_funcs[action_id];
2290 n_pkts_hit = stats->n_pkts_hit[hit];
2291 n_pkts_action = stats->n_pkts_action[action_id];
2293 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2296 hit ? "hit" : "miss",
2297 (uint32_t)action_id);
2299 t->action_id = action_id;
2300 t->structs[0] = action_data;
2302 t->learner_id = learner_id;
2304 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2305 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2308 thread_ip_action_call(p, t, action_id);
2317 static struct action *
2318 action_find(struct rte_swx_pipeline *p, const char *name);
2321 action_has_nbo_args(struct action *a);
2324 instr_learn_translate(struct rte_swx_pipeline *p,
2325 struct action *action,
2328 struct instruction *instr,
2329 struct instruction_data *data __rte_unused)
2333 CHECK(action, EINVAL);
2334 CHECK(n_tokens == 2, EINVAL);
2336 a = action_find(p, tokens[1]);
2338 CHECK(!action_has_nbo_args(a), EINVAL);
2340 instr->type = INSTR_LEARNER_LEARN;
2341 instr->learn.action_id = a->id;
2347 instr_learn_exec(struct rte_swx_pipeline *p)
2349 struct thread *t = &p->threads[p->thread_id];
2350 struct instruction *ip = t->ip;
2352 __instr_learn_exec(p, t, ip);
2362 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2363 struct action *action,
2364 char **tokens __rte_unused,
2366 struct instruction *instr,
2367 struct instruction_data *data __rte_unused)
2369 CHECK(action, EINVAL);
2370 CHECK(n_tokens == 1, EINVAL);
2372 instr->type = INSTR_LEARNER_FORGET;
2378 instr_forget_exec(struct rte_swx_pipeline *p)
2380 struct thread *t = &p->threads[p->thread_id];
2381 struct instruction *ip = t->ip;
2383 __instr_forget_exec(p, t, ip);
2393 instr_extern_translate(struct rte_swx_pipeline *p,
2394 struct action *action __rte_unused,
2397 struct instruction *instr,
2398 struct instruction_data *data __rte_unused)
2400 char *token = tokens[1];
2402 CHECK(n_tokens == 2, EINVAL);
2404 if (token[0] == 'e') {
2405 struct extern_obj *obj;
2406 struct extern_type_member_func *func;
2408 func = extern_obj_member_func_parse(p, token, &obj);
2409 CHECK(func, EINVAL);
2411 instr->type = INSTR_EXTERN_OBJ;
2412 instr->ext_obj.ext_obj_id = obj->id;
2413 instr->ext_obj.func_id = func->id;
2418 if (token[0] == 'f') {
2419 struct extern_func *func;
2421 func = extern_func_parse(p, token);
2422 CHECK(func, EINVAL);
2424 instr->type = INSTR_EXTERN_FUNC;
2425 instr->ext_func.ext_func_id = func->id;
2434 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2436 struct thread *t = &p->threads[p->thread_id];
2437 struct instruction *ip = t->ip;
2440 /* Extern object member function execute. */
2441 done = __instr_extern_obj_exec(p, t, ip);
2444 thread_ip_inc_cond(t, done);
2445 thread_yield_cond(p, done ^ 1);
2449 instr_extern_func_exec(struct rte_swx_pipeline *p)
2451 struct thread *t = &p->threads[p->thread_id];
2452 struct instruction *ip = t->ip;
2455 /* Extern function execute. */
2456 done = __instr_extern_func_exec(p, t, ip);
2459 thread_ip_inc_cond(t, done);
2460 thread_yield_cond(p, done ^ 1);
2467 instr_mov_translate(struct rte_swx_pipeline *p,
2468 struct action *action,
2471 struct instruction *instr,
2472 struct instruction_data *data __rte_unused)
2474 char *dst = tokens[1], *src = tokens[2];
2475 struct field *fdst, *fsrc;
2477 uint32_t dst_struct_id = 0, src_struct_id = 0;
2479 CHECK(n_tokens == 3, EINVAL);
2481 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2482 CHECK(fdst, EINVAL);
2483 CHECK(!fdst->var_size, EINVAL);
2485 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2486 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2488 CHECK(!fsrc->var_size, EINVAL);
2490 instr->type = INSTR_MOV;
2491 if (dst[0] != 'h' && src[0] == 'h')
2492 instr->type = INSTR_MOV_MH;
2493 if (dst[0] == 'h' && src[0] != 'h')
2494 instr->type = INSTR_MOV_HM;
2495 if (dst[0] == 'h' && src[0] == 'h')
2496 instr->type = INSTR_MOV_HH;
2498 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2499 instr->mov.dst.n_bits = fdst->n_bits;
2500 instr->mov.dst.offset = fdst->offset / 8;
2501 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2502 instr->mov.src.n_bits = fsrc->n_bits;
2503 instr->mov.src.offset = fsrc->offset / 8;
2508 src_val = strtoull(src, &src, 0);
2509 CHECK(!src[0], EINVAL);
2512 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2514 instr->type = INSTR_MOV_I;
2515 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2516 instr->mov.dst.n_bits = fdst->n_bits;
2517 instr->mov.dst.offset = fdst->offset / 8;
2518 instr->mov.src_val = src_val;
2523 instr_mov_exec(struct rte_swx_pipeline *p)
2525 struct thread *t = &p->threads[p->thread_id];
2526 struct instruction *ip = t->ip;
2528 __instr_mov_exec(p, t, ip);
2535 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2537 struct thread *t = &p->threads[p->thread_id];
2538 struct instruction *ip = t->ip;
2540 __instr_mov_mh_exec(p, t, ip);
2547 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2549 struct thread *t = &p->threads[p->thread_id];
2550 struct instruction *ip = t->ip;
2552 __instr_mov_hm_exec(p, t, ip);
2559 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2561 struct thread *t = &p->threads[p->thread_id];
2562 struct instruction *ip = t->ip;
2564 __instr_mov_hh_exec(p, t, ip);
2571 instr_mov_i_exec(struct rte_swx_pipeline *p)
2573 struct thread *t = &p->threads[p->thread_id];
2574 struct instruction *ip = t->ip;
2576 __instr_mov_i_exec(p, t, ip);
2586 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2588 struct thread *t = &p->threads[p->thread_id];
2589 struct instruction *ip = t->ip;
2591 __instr_dma_ht_exec(p, t, ip);
2598 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2600 struct thread *t = &p->threads[p->thread_id];
2601 struct instruction *ip = t->ip;
2603 __instr_dma_ht2_exec(p, t, ip);
2610 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2612 struct thread *t = &p->threads[p->thread_id];
2613 struct instruction *ip = t->ip;
2615 __instr_dma_ht3_exec(p, t, ip);
2622 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2624 struct thread *t = &p->threads[p->thread_id];
2625 struct instruction *ip = t->ip;
2627 __instr_dma_ht4_exec(p, t, ip);
2634 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2636 struct thread *t = &p->threads[p->thread_id];
2637 struct instruction *ip = t->ip;
2639 __instr_dma_ht5_exec(p, t, ip);
2646 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2648 struct thread *t = &p->threads[p->thread_id];
2649 struct instruction *ip = t->ip;
2651 __instr_dma_ht6_exec(p, t, ip);
2658 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2660 struct thread *t = &p->threads[p->thread_id];
2661 struct instruction *ip = t->ip;
2663 __instr_dma_ht7_exec(p, t, ip);
2670 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2672 struct thread *t = &p->threads[p->thread_id];
2673 struct instruction *ip = t->ip;
2675 __instr_dma_ht8_exec(p, t, ip);
2685 instr_alu_add_translate(struct rte_swx_pipeline *p,
2686 struct action *action,
2689 struct instruction *instr,
2690 struct instruction_data *data __rte_unused)
2692 char *dst = tokens[1], *src = tokens[2];
2693 struct field *fdst, *fsrc;
2695 uint32_t dst_struct_id = 0, src_struct_id = 0;
2697 CHECK(n_tokens == 3, EINVAL);
2699 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2700 CHECK(fdst, EINVAL);
2701 CHECK(!fdst->var_size, EINVAL);
2703 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2704 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2706 CHECK(!fsrc->var_size, EINVAL);
2708 instr->type = INSTR_ALU_ADD;
2709 if (dst[0] == 'h' && src[0] != 'h')
2710 instr->type = INSTR_ALU_ADD_HM;
2711 if (dst[0] != 'h' && src[0] == 'h')
2712 instr->type = INSTR_ALU_ADD_MH;
2713 if (dst[0] == 'h' && src[0] == 'h')
2714 instr->type = INSTR_ALU_ADD_HH;
2716 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2717 instr->alu.dst.n_bits = fdst->n_bits;
2718 instr->alu.dst.offset = fdst->offset / 8;
2719 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2720 instr->alu.src.n_bits = fsrc->n_bits;
2721 instr->alu.src.offset = fsrc->offset / 8;
2725 /* ADD_MI, ADD_HI. */
2726 src_val = strtoull(src, &src, 0);
2727 CHECK(!src[0], EINVAL);
2729 instr->type = INSTR_ALU_ADD_MI;
2731 instr->type = INSTR_ALU_ADD_HI;
2733 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2734 instr->alu.dst.n_bits = fdst->n_bits;
2735 instr->alu.dst.offset = fdst->offset / 8;
2736 instr->alu.src_val = src_val;
2741 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2742 struct action *action,
2745 struct instruction *instr,
2746 struct instruction_data *data __rte_unused)
2748 char *dst = tokens[1], *src = tokens[2];
2749 struct field *fdst, *fsrc;
2751 uint32_t dst_struct_id = 0, src_struct_id = 0;
2753 CHECK(n_tokens == 3, EINVAL);
2755 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2756 CHECK(fdst, EINVAL);
2757 CHECK(!fdst->var_size, EINVAL);
2759 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2760 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2762 CHECK(!fsrc->var_size, EINVAL);
2764 instr->type = INSTR_ALU_SUB;
2765 if (dst[0] == 'h' && src[0] != 'h')
2766 instr->type = INSTR_ALU_SUB_HM;
2767 if (dst[0] != 'h' && src[0] == 'h')
2768 instr->type = INSTR_ALU_SUB_MH;
2769 if (dst[0] == 'h' && src[0] == 'h')
2770 instr->type = INSTR_ALU_SUB_HH;
2772 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2773 instr->alu.dst.n_bits = fdst->n_bits;
2774 instr->alu.dst.offset = fdst->offset / 8;
2775 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2776 instr->alu.src.n_bits = fsrc->n_bits;
2777 instr->alu.src.offset = fsrc->offset / 8;
2781 /* SUB_MI, SUB_HI. */
2782 src_val = strtoull(src, &src, 0);
2783 CHECK(!src[0], EINVAL);
2785 instr->type = INSTR_ALU_SUB_MI;
2787 instr->type = INSTR_ALU_SUB_HI;
2789 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2790 instr->alu.dst.n_bits = fdst->n_bits;
2791 instr->alu.dst.offset = fdst->offset / 8;
2792 instr->alu.src_val = src_val;
2797 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2798 struct action *action __rte_unused,
2801 struct instruction *instr,
2802 struct instruction_data *data __rte_unused)
2804 char *dst = tokens[1], *src = tokens[2];
2805 struct header *hdst, *hsrc;
2806 struct field *fdst, *fsrc;
2808 CHECK(n_tokens == 3, EINVAL);
2810 fdst = header_field_parse(p, dst, &hdst);
2811 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2812 CHECK(!fdst->var_size, EINVAL);
2815 fsrc = header_field_parse(p, src, &hsrc);
2817 CHECK(!fsrc->var_size, EINVAL);
2819 instr->type = INSTR_ALU_CKADD_FIELD;
2820 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2821 instr->alu.dst.n_bits = fdst->n_bits;
2822 instr->alu.dst.offset = fdst->offset / 8;
2823 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2824 instr->alu.src.n_bits = fsrc->n_bits;
2825 instr->alu.src.offset = fsrc->offset / 8;
2829 /* CKADD_STRUCT, CKADD_STRUCT20. */
2830 hsrc = header_parse(p, src);
2831 CHECK(hsrc, EINVAL);
2832 CHECK(!hsrc->st->var_size, EINVAL);
2834 instr->type = INSTR_ALU_CKADD_STRUCT;
2835 if ((hsrc->st->n_bits / 8) == 20)
2836 instr->type = INSTR_ALU_CKADD_STRUCT20;
2838 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2839 instr->alu.dst.n_bits = fdst->n_bits;
2840 instr->alu.dst.offset = fdst->offset / 8;
2841 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2842 instr->alu.src.n_bits = hsrc->st->n_bits;
2843 instr->alu.src.offset = 0; /* Unused. */
2848 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2849 struct action *action __rte_unused,
2852 struct instruction *instr,
2853 struct instruction_data *data __rte_unused)
2855 char *dst = tokens[1], *src = tokens[2];
2856 struct header *hdst, *hsrc;
2857 struct field *fdst, *fsrc;
2859 CHECK(n_tokens == 3, EINVAL);
2861 fdst = header_field_parse(p, dst, &hdst);
2862 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2863 CHECK(!fdst->var_size, EINVAL);
2865 fsrc = header_field_parse(p, src, &hsrc);
2866 CHECK(fsrc, EINVAL);
2867 CHECK(!fsrc->var_size, EINVAL);
2869 instr->type = INSTR_ALU_CKSUB_FIELD;
2870 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2871 instr->alu.dst.n_bits = fdst->n_bits;
2872 instr->alu.dst.offset = fdst->offset / 8;
2873 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2874 instr->alu.src.n_bits = fsrc->n_bits;
2875 instr->alu.src.offset = fsrc->offset / 8;
2880 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2881 struct action *action,
2884 struct instruction *instr,
2885 struct instruction_data *data __rte_unused)
2887 char *dst = tokens[1], *src = tokens[2];
2888 struct field *fdst, *fsrc;
2890 uint32_t dst_struct_id = 0, src_struct_id = 0;
2892 CHECK(n_tokens == 3, EINVAL);
2894 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2895 CHECK(fdst, EINVAL);
2896 CHECK(!fdst->var_size, EINVAL);
2898 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2899 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2901 CHECK(!fsrc->var_size, EINVAL);
2903 instr->type = INSTR_ALU_SHL;
2904 if (dst[0] == 'h' && src[0] != 'h')
2905 instr->type = INSTR_ALU_SHL_HM;
2906 if (dst[0] != 'h' && src[0] == 'h')
2907 instr->type = INSTR_ALU_SHL_MH;
2908 if (dst[0] == 'h' && src[0] == 'h')
2909 instr->type = INSTR_ALU_SHL_HH;
2911 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2912 instr->alu.dst.n_bits = fdst->n_bits;
2913 instr->alu.dst.offset = fdst->offset / 8;
2914 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2915 instr->alu.src.n_bits = fsrc->n_bits;
2916 instr->alu.src.offset = fsrc->offset / 8;
2920 /* SHL_MI, SHL_HI. */
2921 src_val = strtoull(src, &src, 0);
2922 CHECK(!src[0], EINVAL);
2924 instr->type = INSTR_ALU_SHL_MI;
2926 instr->type = INSTR_ALU_SHL_HI;
2928 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2929 instr->alu.dst.n_bits = fdst->n_bits;
2930 instr->alu.dst.offset = fdst->offset / 8;
2931 instr->alu.src_val = src_val;
2936 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2937 struct action *action,
2940 struct instruction *instr,
2941 struct instruction_data *data __rte_unused)
2943 char *dst = tokens[1], *src = tokens[2];
2944 struct field *fdst, *fsrc;
2946 uint32_t dst_struct_id = 0, src_struct_id = 0;
2948 CHECK(n_tokens == 3, EINVAL);
2950 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2951 CHECK(fdst, EINVAL);
2952 CHECK(!fdst->var_size, EINVAL);
2954 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2955 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2957 CHECK(!fsrc->var_size, EINVAL);
2959 instr->type = INSTR_ALU_SHR;
2960 if (dst[0] == 'h' && src[0] != 'h')
2961 instr->type = INSTR_ALU_SHR_HM;
2962 if (dst[0] != 'h' && src[0] == 'h')
2963 instr->type = INSTR_ALU_SHR_MH;
2964 if (dst[0] == 'h' && src[0] == 'h')
2965 instr->type = INSTR_ALU_SHR_HH;
2967 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2968 instr->alu.dst.n_bits = fdst->n_bits;
2969 instr->alu.dst.offset = fdst->offset / 8;
2970 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2971 instr->alu.src.n_bits = fsrc->n_bits;
2972 instr->alu.src.offset = fsrc->offset / 8;
2976 /* SHR_MI, SHR_HI. */
2977 src_val = strtoull(src, &src, 0);
2978 CHECK(!src[0], EINVAL);
2980 instr->type = INSTR_ALU_SHR_MI;
2982 instr->type = INSTR_ALU_SHR_HI;
2984 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2985 instr->alu.dst.n_bits = fdst->n_bits;
2986 instr->alu.dst.offset = fdst->offset / 8;
2987 instr->alu.src_val = src_val;
2992 instr_alu_and_translate(struct rte_swx_pipeline *p,
2993 struct action *action,
2996 struct instruction *instr,
2997 struct instruction_data *data __rte_unused)
2999 char *dst = tokens[1], *src = tokens[2];
3000 struct field *fdst, *fsrc;
3002 uint32_t dst_struct_id = 0, src_struct_id = 0;
3004 CHECK(n_tokens == 3, EINVAL);
3006 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3007 CHECK(fdst, EINVAL);
3008 CHECK(!fdst->var_size, EINVAL);
3010 /* AND, AND_MH, AND_HM, AND_HH. */
3011 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3013 CHECK(!fsrc->var_size, EINVAL);
3015 instr->type = INSTR_ALU_AND;
3016 if (dst[0] != 'h' && src[0] == 'h')
3017 instr->type = INSTR_ALU_AND_MH;
3018 if (dst[0] == 'h' && src[0] != 'h')
3019 instr->type = INSTR_ALU_AND_HM;
3020 if (dst[0] == 'h' && src[0] == 'h')
3021 instr->type = INSTR_ALU_AND_HH;
3023 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3024 instr->alu.dst.n_bits = fdst->n_bits;
3025 instr->alu.dst.offset = fdst->offset / 8;
3026 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3027 instr->alu.src.n_bits = fsrc->n_bits;
3028 instr->alu.src.offset = fsrc->offset / 8;
3033 src_val = strtoull(src, &src, 0);
3034 CHECK(!src[0], EINVAL);
3037 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3039 instr->type = INSTR_ALU_AND_I;
3040 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3041 instr->alu.dst.n_bits = fdst->n_bits;
3042 instr->alu.dst.offset = fdst->offset / 8;
3043 instr->alu.src_val = src_val;
3048 instr_alu_or_translate(struct rte_swx_pipeline *p,
3049 struct action *action,
3052 struct instruction *instr,
3053 struct instruction_data *data __rte_unused)
3055 char *dst = tokens[1], *src = tokens[2];
3056 struct field *fdst, *fsrc;
3058 uint32_t dst_struct_id = 0, src_struct_id = 0;
3060 CHECK(n_tokens == 3, EINVAL);
3062 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3063 CHECK(fdst, EINVAL);
3064 CHECK(!fdst->var_size, EINVAL);
3066 /* OR, OR_MH, OR_HM, OR_HH. */
3067 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3069 CHECK(!fsrc->var_size, EINVAL);
3071 instr->type = INSTR_ALU_OR;
3072 if (dst[0] != 'h' && src[0] == 'h')
3073 instr->type = INSTR_ALU_OR_MH;
3074 if (dst[0] == 'h' && src[0] != 'h')
3075 instr->type = INSTR_ALU_OR_HM;
3076 if (dst[0] == 'h' && src[0] == 'h')
3077 instr->type = INSTR_ALU_OR_HH;
3079 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3080 instr->alu.dst.n_bits = fdst->n_bits;
3081 instr->alu.dst.offset = fdst->offset / 8;
3082 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3083 instr->alu.src.n_bits = fsrc->n_bits;
3084 instr->alu.src.offset = fsrc->offset / 8;
3089 src_val = strtoull(src, &src, 0);
3090 CHECK(!src[0], EINVAL);
3093 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3095 instr->type = INSTR_ALU_OR_I;
3096 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3097 instr->alu.dst.n_bits = fdst->n_bits;
3098 instr->alu.dst.offset = fdst->offset / 8;
3099 instr->alu.src_val = src_val;
3104 instr_alu_xor_translate(struct rte_swx_pipeline *p,
3105 struct action *action,
3108 struct instruction *instr,
3109 struct instruction_data *data __rte_unused)
3111 char *dst = tokens[1], *src = tokens[2];
3112 struct field *fdst, *fsrc;
3114 uint32_t dst_struct_id = 0, src_struct_id = 0;
3116 CHECK(n_tokens == 3, EINVAL);
3118 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3119 CHECK(fdst, EINVAL);
3120 CHECK(!fdst->var_size, EINVAL);
3122 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3123 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3125 CHECK(!fsrc->var_size, EINVAL);
3127 instr->type = INSTR_ALU_XOR;
3128 if (dst[0] != 'h' && src[0] == 'h')
3129 instr->type = INSTR_ALU_XOR_MH;
3130 if (dst[0] == 'h' && src[0] != 'h')
3131 instr->type = INSTR_ALU_XOR_HM;
3132 if (dst[0] == 'h' && src[0] == 'h')
3133 instr->type = INSTR_ALU_XOR_HH;
3135 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3136 instr->alu.dst.n_bits = fdst->n_bits;
3137 instr->alu.dst.offset = fdst->offset / 8;
3138 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3139 instr->alu.src.n_bits = fsrc->n_bits;
3140 instr->alu.src.offset = fsrc->offset / 8;
3145 src_val = strtoull(src, &src, 0);
3146 CHECK(!src[0], EINVAL);
3149 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3151 instr->type = INSTR_ALU_XOR_I;
3152 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3153 instr->alu.dst.n_bits = fdst->n_bits;
3154 instr->alu.dst.offset = fdst->offset / 8;
3155 instr->alu.src_val = src_val;
3160 instr_alu_add_exec(struct rte_swx_pipeline *p)
3162 struct thread *t = &p->threads[p->thread_id];
3163 struct instruction *ip = t->ip;
3166 __instr_alu_add_exec(p, t, ip);
3173 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3175 struct thread *t = &p->threads[p->thread_id];
3176 struct instruction *ip = t->ip;
3179 __instr_alu_add_mh_exec(p, t, ip);
3186 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3188 struct thread *t = &p->threads[p->thread_id];
3189 struct instruction *ip = t->ip;
3192 __instr_alu_add_hm_exec(p, t, ip);
3199 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3201 struct thread *t = &p->threads[p->thread_id];
3202 struct instruction *ip = t->ip;
3205 __instr_alu_add_hh_exec(p, t, ip);
3212 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3214 struct thread *t = &p->threads[p->thread_id];
3215 struct instruction *ip = t->ip;
3218 __instr_alu_add_mi_exec(p, t, ip);
3225 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3227 struct thread *t = &p->threads[p->thread_id];
3228 struct instruction *ip = t->ip;
3231 __instr_alu_add_hi_exec(p, t, ip);
3238 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3240 struct thread *t = &p->threads[p->thread_id];
3241 struct instruction *ip = t->ip;
3244 __instr_alu_sub_exec(p, t, ip);
3251 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3253 struct thread *t = &p->threads[p->thread_id];
3254 struct instruction *ip = t->ip;
3257 __instr_alu_sub_mh_exec(p, t, ip);
3264 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3266 struct thread *t = &p->threads[p->thread_id];
3267 struct instruction *ip = t->ip;
3270 __instr_alu_sub_hm_exec(p, t, ip);
3277 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3279 struct thread *t = &p->threads[p->thread_id];
3280 struct instruction *ip = t->ip;
3283 __instr_alu_sub_hh_exec(p, t, ip);
3290 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3292 struct thread *t = &p->threads[p->thread_id];
3293 struct instruction *ip = t->ip;
3296 __instr_alu_sub_mi_exec(p, t, ip);
3303 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3305 struct thread *t = &p->threads[p->thread_id];
3306 struct instruction *ip = t->ip;
3309 __instr_alu_sub_hi_exec(p, t, ip);
3316 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3318 struct thread *t = &p->threads[p->thread_id];
3319 struct instruction *ip = t->ip;
3322 __instr_alu_shl_exec(p, t, ip);
3329 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3331 struct thread *t = &p->threads[p->thread_id];
3332 struct instruction *ip = t->ip;
3335 __instr_alu_shl_mh_exec(p, t, ip);
3342 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3344 struct thread *t = &p->threads[p->thread_id];
3345 struct instruction *ip = t->ip;
3348 __instr_alu_shl_hm_exec(p, t, ip);
3355 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3357 struct thread *t = &p->threads[p->thread_id];
3358 struct instruction *ip = t->ip;
3361 __instr_alu_shl_hh_exec(p, t, ip);
3368 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3370 struct thread *t = &p->threads[p->thread_id];
3371 struct instruction *ip = t->ip;
3374 __instr_alu_shl_mi_exec(p, t, ip);
3381 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3383 struct thread *t = &p->threads[p->thread_id];
3384 struct instruction *ip = t->ip;
3387 __instr_alu_shl_hi_exec(p, t, ip);
3394 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3396 struct thread *t = &p->threads[p->thread_id];
3397 struct instruction *ip = t->ip;
3400 __instr_alu_shr_exec(p, t, ip);
3407 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3409 struct thread *t = &p->threads[p->thread_id];
3410 struct instruction *ip = t->ip;
3413 __instr_alu_shr_mh_exec(p, t, ip);
3420 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3422 struct thread *t = &p->threads[p->thread_id];
3423 struct instruction *ip = t->ip;
3426 __instr_alu_shr_hm_exec(p, t, ip);
3433 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3435 struct thread *t = &p->threads[p->thread_id];
3436 struct instruction *ip = t->ip;
3439 __instr_alu_shr_hh_exec(p, t, ip);
3446 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3448 struct thread *t = &p->threads[p->thread_id];
3449 struct instruction *ip = t->ip;
3452 __instr_alu_shr_mi_exec(p, t, ip);
3459 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3461 struct thread *t = &p->threads[p->thread_id];
3462 struct instruction *ip = t->ip;
3465 __instr_alu_shr_hi_exec(p, t, ip);
3472 instr_alu_and_exec(struct rte_swx_pipeline *p)
3474 struct thread *t = &p->threads[p->thread_id];
3475 struct instruction *ip = t->ip;
3478 __instr_alu_and_exec(p, t, ip);
3485 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3487 struct thread *t = &p->threads[p->thread_id];
3488 struct instruction *ip = t->ip;
3491 __instr_alu_and_mh_exec(p, t, ip);
3498 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3500 struct thread *t = &p->threads[p->thread_id];
3501 struct instruction *ip = t->ip;
3504 __instr_alu_and_hm_exec(p, t, ip);
3511 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3513 struct thread *t = &p->threads[p->thread_id];
3514 struct instruction *ip = t->ip;
3517 __instr_alu_and_hh_exec(p, t, ip);
3524 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3526 struct thread *t = &p->threads[p->thread_id];
3527 struct instruction *ip = t->ip;
3530 __instr_alu_and_i_exec(p, t, ip);
3537 instr_alu_or_exec(struct rte_swx_pipeline *p)
3539 struct thread *t = &p->threads[p->thread_id];
3540 struct instruction *ip = t->ip;
3543 __instr_alu_or_exec(p, t, ip);
3550 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3552 struct thread *t = &p->threads[p->thread_id];
3553 struct instruction *ip = t->ip;
3556 __instr_alu_or_mh_exec(p, t, ip);
3563 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3565 struct thread *t = &p->threads[p->thread_id];
3566 struct instruction *ip = t->ip;
3569 __instr_alu_or_hm_exec(p, t, ip);
3576 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3578 struct thread *t = &p->threads[p->thread_id];
3579 struct instruction *ip = t->ip;
3582 __instr_alu_or_hh_exec(p, t, ip);
3589 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3591 struct thread *t = &p->threads[p->thread_id];
3592 struct instruction *ip = t->ip;
3595 __instr_alu_or_i_exec(p, t, ip);
3602 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3604 struct thread *t = &p->threads[p->thread_id];
3605 struct instruction *ip = t->ip;
3608 __instr_alu_xor_exec(p, t, ip);
3615 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3617 struct thread *t = &p->threads[p->thread_id];
3618 struct instruction *ip = t->ip;
3621 __instr_alu_xor_mh_exec(p, t, ip);
3628 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3630 struct thread *t = &p->threads[p->thread_id];
3631 struct instruction *ip = t->ip;
3634 __instr_alu_xor_hm_exec(p, t, ip);
3641 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3643 struct thread *t = &p->threads[p->thread_id];
3644 struct instruction *ip = t->ip;
3647 __instr_alu_xor_hh_exec(p, t, ip);
3654 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3656 struct thread *t = &p->threads[p->thread_id];
3657 struct instruction *ip = t->ip;
3660 __instr_alu_xor_i_exec(p, t, ip);
3667 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3669 struct thread *t = &p->threads[p->thread_id];
3670 struct instruction *ip = t->ip;
3673 __instr_alu_ckadd_field_exec(p, t, ip);
3680 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3682 struct thread *t = &p->threads[p->thread_id];
3683 struct instruction *ip = t->ip;
3686 __instr_alu_cksub_field_exec(p, t, ip);
3693 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3695 struct thread *t = &p->threads[p->thread_id];
3696 struct instruction *ip = t->ip;
3699 __instr_alu_ckadd_struct20_exec(p, t, ip);
3706 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3708 struct thread *t = &p->threads[p->thread_id];
3709 struct instruction *ip = t->ip;
3712 __instr_alu_ckadd_struct_exec(p, t, ip);
3721 static struct regarray *
3722 regarray_find(struct rte_swx_pipeline *p, const char *name);
3725 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3726 struct action *action,
3729 struct instruction *instr,
3730 struct instruction_data *data __rte_unused)
3732 char *regarray = tokens[1], *idx = tokens[2];
3735 uint32_t idx_struct_id, idx_val;
3737 CHECK(n_tokens == 3, EINVAL);
3739 r = regarray_find(p, regarray);
3742 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3743 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3745 CHECK(!fidx->var_size, EINVAL);
3747 instr->type = INSTR_REGPREFETCH_RM;
3749 instr->type = INSTR_REGPREFETCH_RH;
3751 instr->regarray.regarray_id = r->id;
3752 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3753 instr->regarray.idx.n_bits = fidx->n_bits;
3754 instr->regarray.idx.offset = fidx->offset / 8;
3755 instr->regarray.dstsrc_val = 0; /* Unused. */
3759 /* REGPREFETCH_RI. */
3760 idx_val = strtoul(idx, &idx, 0);
3761 CHECK(!idx[0], EINVAL);
3763 instr->type = INSTR_REGPREFETCH_RI;
3764 instr->regarray.regarray_id = r->id;
3765 instr->regarray.idx_val = idx_val;
3766 instr->regarray.dstsrc_val = 0; /* Unused. */
3771 instr_regrd_translate(struct rte_swx_pipeline *p,
3772 struct action *action,
3775 struct instruction *instr,
3776 struct instruction_data *data __rte_unused)
3778 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3780 struct field *fdst, *fidx;
3781 uint32_t dst_struct_id, idx_struct_id, idx_val;
3783 CHECK(n_tokens == 4, EINVAL);
3785 r = regarray_find(p, regarray);
3788 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3789 CHECK(fdst, EINVAL);
3790 CHECK(!fdst->var_size, EINVAL);
3792 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3793 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3795 CHECK(!fidx->var_size, EINVAL);
3797 instr->type = INSTR_REGRD_MRM;
3798 if (dst[0] == 'h' && idx[0] != 'h')
3799 instr->type = INSTR_REGRD_HRM;
3800 if (dst[0] != 'h' && idx[0] == 'h')
3801 instr->type = INSTR_REGRD_MRH;
3802 if (dst[0] == 'h' && idx[0] == 'h')
3803 instr->type = INSTR_REGRD_HRH;
3805 instr->regarray.regarray_id = r->id;
3806 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3807 instr->regarray.idx.n_bits = fidx->n_bits;
3808 instr->regarray.idx.offset = fidx->offset / 8;
3809 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3810 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3811 instr->regarray.dstsrc.offset = fdst->offset / 8;
3815 /* REGRD_MRI, REGRD_HRI. */
3816 idx_val = strtoul(idx, &idx, 0);
3817 CHECK(!idx[0], EINVAL);
3819 instr->type = INSTR_REGRD_MRI;
3821 instr->type = INSTR_REGRD_HRI;
3823 instr->regarray.regarray_id = r->id;
3824 instr->regarray.idx_val = idx_val;
3825 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3826 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3827 instr->regarray.dstsrc.offset = fdst->offset / 8;
3832 instr_regwr_translate(struct rte_swx_pipeline *p,
3833 struct action *action,
3836 struct instruction *instr,
3837 struct instruction_data *data __rte_unused)
3839 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3841 struct field *fidx, *fsrc;
3843 uint32_t idx_struct_id, idx_val, src_struct_id;
3845 CHECK(n_tokens == 4, EINVAL);
3847 r = regarray_find(p, regarray);
3850 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3851 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3852 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3854 CHECK(!fidx->var_size, EINVAL);
3855 CHECK(!fsrc->var_size, EINVAL);
3857 instr->type = INSTR_REGWR_RMM;
3858 if (idx[0] == 'h' && src[0] != 'h')
3859 instr->type = INSTR_REGWR_RHM;
3860 if (idx[0] != 'h' && src[0] == 'h')
3861 instr->type = INSTR_REGWR_RMH;
3862 if (idx[0] == 'h' && src[0] == 'h')
3863 instr->type = INSTR_REGWR_RHH;
3865 instr->regarray.regarray_id = r->id;
3866 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3867 instr->regarray.idx.n_bits = fidx->n_bits;
3868 instr->regarray.idx.offset = fidx->offset / 8;
3869 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3870 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3871 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3875 /* REGWR_RHI, REGWR_RMI. */
3876 if (fidx && !fsrc) {
3877 CHECK(!fidx->var_size, EINVAL);
3879 src_val = strtoull(src, &src, 0);
3880 CHECK(!src[0], EINVAL);
3882 instr->type = INSTR_REGWR_RMI;
3884 instr->type = INSTR_REGWR_RHI;
3886 instr->regarray.regarray_id = r->id;
3887 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3888 instr->regarray.idx.n_bits = fidx->n_bits;
3889 instr->regarray.idx.offset = fidx->offset / 8;
3890 instr->regarray.dstsrc_val = src_val;
3894 /* REGWR_RIH, REGWR_RIM. */
3895 if (!fidx && fsrc) {
3896 idx_val = strtoul(idx, &idx, 0);
3897 CHECK(!idx[0], EINVAL);
3899 CHECK(!fsrc->var_size, EINVAL);
3901 instr->type = INSTR_REGWR_RIM;
3903 instr->type = INSTR_REGWR_RIH;
3905 instr->regarray.regarray_id = r->id;
3906 instr->regarray.idx_val = idx_val;
3907 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3908 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3909 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3914 src_val = strtoull(src, &src, 0);
3915 CHECK(!src[0], EINVAL);
3917 idx_val = strtoul(idx, &idx, 0);
3918 CHECK(!idx[0], EINVAL);
3920 instr->type = INSTR_REGWR_RII;
3921 instr->regarray.idx_val = idx_val;
3922 instr->regarray.dstsrc_val = src_val;
3928 instr_regadd_translate(struct rte_swx_pipeline *p,
3929 struct action *action,
3932 struct instruction *instr,
3933 struct instruction_data *data __rte_unused)
3935 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3937 struct field *fidx, *fsrc;
3939 uint32_t idx_struct_id, idx_val, src_struct_id;
3941 CHECK(n_tokens == 4, EINVAL);
3943 r = regarray_find(p, regarray);
3946 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3947 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3948 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3950 CHECK(!fidx->var_size, EINVAL);
3951 CHECK(!fsrc->var_size, EINVAL);
3953 instr->type = INSTR_REGADD_RMM;
3954 if (idx[0] == 'h' && src[0] != 'h')
3955 instr->type = INSTR_REGADD_RHM;
3956 if (idx[0] != 'h' && src[0] == 'h')
3957 instr->type = INSTR_REGADD_RMH;
3958 if (idx[0] == 'h' && src[0] == 'h')
3959 instr->type = INSTR_REGADD_RHH;
3961 instr->regarray.regarray_id = r->id;
3962 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3963 instr->regarray.idx.n_bits = fidx->n_bits;
3964 instr->regarray.idx.offset = fidx->offset / 8;
3965 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3966 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3967 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3971 /* REGADD_RHI, REGADD_RMI. */
3972 if (fidx && !fsrc) {
3973 CHECK(!fidx->var_size, EINVAL);
3975 src_val = strtoull(src, &src, 0);
3976 CHECK(!src[0], EINVAL);
3978 instr->type = INSTR_REGADD_RMI;
3980 instr->type = INSTR_REGADD_RHI;
3982 instr->regarray.regarray_id = r->id;
3983 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3984 instr->regarray.idx.n_bits = fidx->n_bits;
3985 instr->regarray.idx.offset = fidx->offset / 8;
3986 instr->regarray.dstsrc_val = src_val;
3990 /* REGADD_RIH, REGADD_RIM. */
3991 if (!fidx && fsrc) {
3992 idx_val = strtoul(idx, &idx, 0);
3993 CHECK(!idx[0], EINVAL);
3995 CHECK(!fsrc->var_size, EINVAL);
3997 instr->type = INSTR_REGADD_RIM;
3999 instr->type = INSTR_REGADD_RIH;
4001 instr->regarray.regarray_id = r->id;
4002 instr->regarray.idx_val = idx_val;
4003 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4004 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4005 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4010 src_val = strtoull(src, &src, 0);
4011 CHECK(!src[0], EINVAL);
4013 idx_val = strtoul(idx, &idx, 0);
4014 CHECK(!idx[0], EINVAL);
4016 instr->type = INSTR_REGADD_RII;
4017 instr->regarray.idx_val = idx_val;
4018 instr->regarray.dstsrc_val = src_val;
4023 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
4025 struct thread *t = &p->threads[p->thread_id];
4026 struct instruction *ip = t->ip;
4029 __instr_regprefetch_rh_exec(p, t, ip);
4036 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
4038 struct thread *t = &p->threads[p->thread_id];
4039 struct instruction *ip = t->ip;
4042 __instr_regprefetch_rm_exec(p, t, ip);
4049 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
4051 struct thread *t = &p->threads[p->thread_id];
4052 struct instruction *ip = t->ip;
4055 __instr_regprefetch_ri_exec(p, t, ip);
4062 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
4064 struct thread *t = &p->threads[p->thread_id];
4065 struct instruction *ip = t->ip;
4068 __instr_regrd_hrh_exec(p, t, ip);
4075 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
4077 struct thread *t = &p->threads[p->thread_id];
4078 struct instruction *ip = t->ip;
4081 __instr_regrd_hrm_exec(p, t, ip);
4088 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
4090 struct thread *t = &p->threads[p->thread_id];
4091 struct instruction *ip = t->ip;
4094 __instr_regrd_mrh_exec(p, t, ip);
4101 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
4103 struct thread *t = &p->threads[p->thread_id];
4104 struct instruction *ip = t->ip;
4107 __instr_regrd_mrm_exec(p, t, ip);
4114 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
4116 struct thread *t = &p->threads[p->thread_id];
4117 struct instruction *ip = t->ip;
4120 __instr_regrd_hri_exec(p, t, ip);
4127 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4129 struct thread *t = &p->threads[p->thread_id];
4130 struct instruction *ip = t->ip;
4133 __instr_regrd_mri_exec(p, t, ip);
4140 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4142 struct thread *t = &p->threads[p->thread_id];
4143 struct instruction *ip = t->ip;
4146 __instr_regwr_rhh_exec(p, t, ip);
4153 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4155 struct thread *t = &p->threads[p->thread_id];
4156 struct instruction *ip = t->ip;
4159 __instr_regwr_rhm_exec(p, t, ip);
4166 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4168 struct thread *t = &p->threads[p->thread_id];
4169 struct instruction *ip = t->ip;
4172 __instr_regwr_rmh_exec(p, t, ip);
4179 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4181 struct thread *t = &p->threads[p->thread_id];
4182 struct instruction *ip = t->ip;
4185 __instr_regwr_rmm_exec(p, t, ip);
4192 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4194 struct thread *t = &p->threads[p->thread_id];
4195 struct instruction *ip = t->ip;
4198 __instr_regwr_rhi_exec(p, t, ip);
4205 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4207 struct thread *t = &p->threads[p->thread_id];
4208 struct instruction *ip = t->ip;
4211 __instr_regwr_rmi_exec(p, t, ip);
4218 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4220 struct thread *t = &p->threads[p->thread_id];
4221 struct instruction *ip = t->ip;
4224 __instr_regwr_rih_exec(p, t, ip);
4231 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4233 struct thread *t = &p->threads[p->thread_id];
4234 struct instruction *ip = t->ip;
4237 __instr_regwr_rim_exec(p, t, ip);
4244 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4246 struct thread *t = &p->threads[p->thread_id];
4247 struct instruction *ip = t->ip;
4250 __instr_regwr_rii_exec(p, t, ip);
4257 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4259 struct thread *t = &p->threads[p->thread_id];
4260 struct instruction *ip = t->ip;
4263 __instr_regadd_rhh_exec(p, t, ip);
4270 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4272 struct thread *t = &p->threads[p->thread_id];
4273 struct instruction *ip = t->ip;
4276 __instr_regadd_rhm_exec(p, t, ip);
4283 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4285 struct thread *t = &p->threads[p->thread_id];
4286 struct instruction *ip = t->ip;
4289 __instr_regadd_rmh_exec(p, t, ip);
4296 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4298 struct thread *t = &p->threads[p->thread_id];
4299 struct instruction *ip = t->ip;
4302 __instr_regadd_rmm_exec(p, t, ip);
4309 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4311 struct thread *t = &p->threads[p->thread_id];
4312 struct instruction *ip = t->ip;
4315 __instr_regadd_rhi_exec(p, t, ip);
4322 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4324 struct thread *t = &p->threads[p->thread_id];
4325 struct instruction *ip = t->ip;
4328 __instr_regadd_rmi_exec(p, t, ip);
4335 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4337 struct thread *t = &p->threads[p->thread_id];
4338 struct instruction *ip = t->ip;
4341 __instr_regadd_rih_exec(p, t, ip);
4348 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4350 struct thread *t = &p->threads[p->thread_id];
4351 struct instruction *ip = t->ip;
4354 __instr_regadd_rim_exec(p, t, ip);
4361 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4363 struct thread *t = &p->threads[p->thread_id];
4364 struct instruction *ip = t->ip;
4367 __instr_regadd_rii_exec(p, t, ip);
4376 static struct metarray *
4377 metarray_find(struct rte_swx_pipeline *p, const char *name);
4380 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4381 struct action *action,
4384 struct instruction *instr,
4385 struct instruction_data *data __rte_unused)
4387 char *metarray = tokens[1], *idx = tokens[2];
4390 uint32_t idx_struct_id, idx_val;
4392 CHECK(n_tokens == 3, EINVAL);
4394 m = metarray_find(p, metarray);
4397 /* METPREFETCH_H, METPREFETCH_M. */
4398 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4400 CHECK(!fidx->var_size, EINVAL);
4402 instr->type = INSTR_METPREFETCH_M;
4404 instr->type = INSTR_METPREFETCH_H;
4406 instr->meter.metarray_id = m->id;
4407 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4408 instr->meter.idx.n_bits = fidx->n_bits;
4409 instr->meter.idx.offset = fidx->offset / 8;
4413 /* METPREFETCH_I. */
4414 idx_val = strtoul(idx, &idx, 0);
4415 CHECK(!idx[0], EINVAL);
4417 instr->type = INSTR_METPREFETCH_I;
4418 instr->meter.metarray_id = m->id;
4419 instr->meter.idx_val = idx_val;
4424 instr_meter_translate(struct rte_swx_pipeline *p,
4425 struct action *action,
4428 struct instruction *instr,
4429 struct instruction_data *data __rte_unused)
4431 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4432 char *color_in = tokens[4], *color_out = tokens[5];
4434 struct field *fidx, *flength, *fcin, *fcout;
4435 uint32_t idx_struct_id, length_struct_id;
4436 uint32_t color_in_struct_id, color_out_struct_id;
4438 CHECK(n_tokens == 6, EINVAL);
4440 m = metarray_find(p, metarray);
4443 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4445 flength = struct_field_parse(p, action, length, &length_struct_id);
4446 CHECK(flength, EINVAL);
4447 CHECK(!flength->var_size, EINVAL);
4449 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4451 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4452 CHECK(fcout, EINVAL);
4453 CHECK(!fcout->var_size, EINVAL);
4455 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4457 CHECK(!fidx->var_size, EINVAL);
4458 CHECK(!fcin->var_size, EINVAL);
4460 instr->type = INSTR_METER_MMM;
4461 if (idx[0] == 'h' && length[0] == 'h')
4462 instr->type = INSTR_METER_HHM;
4463 if (idx[0] == 'h' && length[0] != 'h')
4464 instr->type = INSTR_METER_HMM;
4465 if (idx[0] != 'h' && length[0] == 'h')
4466 instr->type = INSTR_METER_MHM;
4468 instr->meter.metarray_id = m->id;
4470 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4471 instr->meter.idx.n_bits = fidx->n_bits;
4472 instr->meter.idx.offset = fidx->offset / 8;
4474 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4475 instr->meter.length.n_bits = flength->n_bits;
4476 instr->meter.length.offset = flength->offset / 8;
4478 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4479 instr->meter.color_in.n_bits = fcin->n_bits;
4480 instr->meter.color_in.offset = fcin->offset / 8;
4482 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4483 instr->meter.color_out.n_bits = fcout->n_bits;
4484 instr->meter.color_out.offset = fcout->offset / 8;
4489 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4490 if (fidx && !fcin) {
4491 uint32_t color_in_val;
4493 CHECK(!fidx->var_size, EINVAL);
4495 color_in_val = strtoul(color_in, &color_in, 0);
4496 CHECK(!color_in[0], EINVAL);
4498 instr->type = INSTR_METER_MMI;
4499 if (idx[0] == 'h' && length[0] == 'h')
4500 instr->type = INSTR_METER_HHI;
4501 if (idx[0] == 'h' && length[0] != 'h')
4502 instr->type = INSTR_METER_HMI;
4503 if (idx[0] != 'h' && length[0] == 'h')
4504 instr->type = INSTR_METER_MHI;
4506 instr->meter.metarray_id = m->id;
4508 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4509 instr->meter.idx.n_bits = fidx->n_bits;
4510 instr->meter.idx.offset = fidx->offset / 8;
4512 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4513 instr->meter.length.n_bits = flength->n_bits;
4514 instr->meter.length.offset = flength->offset / 8;
4516 instr->meter.color_in_val = color_in_val;
4518 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4519 instr->meter.color_out.n_bits = fcout->n_bits;
4520 instr->meter.color_out.offset = fcout->offset / 8;
4525 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4526 if (!fidx && fcin) {
4529 idx_val = strtoul(idx, &idx, 0);
4530 CHECK(!idx[0], EINVAL);
4532 CHECK(!fcin->var_size, EINVAL);
4534 instr->type = INSTR_METER_IMM;
4535 if (length[0] == 'h')
4536 instr->type = INSTR_METER_IHM;
4538 instr->meter.metarray_id = m->id;
4540 instr->meter.idx_val = idx_val;
4542 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4543 instr->meter.length.n_bits = flength->n_bits;
4544 instr->meter.length.offset = flength->offset / 8;
4546 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4547 instr->meter.color_in.n_bits = fcin->n_bits;
4548 instr->meter.color_in.offset = fcin->offset / 8;
4550 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4551 instr->meter.color_out.n_bits = fcout->n_bits;
4552 instr->meter.color_out.offset = fcout->offset / 8;
4557 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4558 if (!fidx && !fcin) {
4559 uint32_t idx_val, color_in_val;
4561 idx_val = strtoul(idx, &idx, 0);
4562 CHECK(!idx[0], EINVAL);
4564 color_in_val = strtoul(color_in, &color_in, 0);
4565 CHECK(!color_in[0], EINVAL);
4567 instr->type = INSTR_METER_IMI;
4568 if (length[0] == 'h')
4569 instr->type = INSTR_METER_IHI;
4571 instr->meter.metarray_id = m->id;
4573 instr->meter.idx_val = idx_val;
4575 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4576 instr->meter.length.n_bits = flength->n_bits;
4577 instr->meter.length.offset = flength->offset / 8;
4579 instr->meter.color_in_val = color_in_val;
4581 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4582 instr->meter.color_out.n_bits = fcout->n_bits;
4583 instr->meter.color_out.offset = fcout->offset / 8;
4592 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4594 struct thread *t = &p->threads[p->thread_id];
4595 struct instruction *ip = t->ip;
4598 __instr_metprefetch_h_exec(p, t, ip);
4605 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4607 struct thread *t = &p->threads[p->thread_id];
4608 struct instruction *ip = t->ip;
4611 __instr_metprefetch_m_exec(p, t, ip);
4618 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4620 struct thread *t = &p->threads[p->thread_id];
4621 struct instruction *ip = t->ip;
4624 __instr_metprefetch_i_exec(p, t, ip);
4631 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4633 struct thread *t = &p->threads[p->thread_id];
4634 struct instruction *ip = t->ip;
4637 __instr_meter_hhm_exec(p, t, ip);
4644 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4646 struct thread *t = &p->threads[p->thread_id];
4647 struct instruction *ip = t->ip;
4650 __instr_meter_hhi_exec(p, t, ip);
4657 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4659 struct thread *t = &p->threads[p->thread_id];
4660 struct instruction *ip = t->ip;
4663 __instr_meter_hmm_exec(p, t, ip);
4670 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4672 struct thread *t = &p->threads[p->thread_id];
4673 struct instruction *ip = t->ip;
4676 __instr_meter_hmi_exec(p, t, ip);
4683 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4685 struct thread *t = &p->threads[p->thread_id];
4686 struct instruction *ip = t->ip;
4689 __instr_meter_mhm_exec(p, t, ip);
4696 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4698 struct thread *t = &p->threads[p->thread_id];
4699 struct instruction *ip = t->ip;
4702 __instr_meter_mhi_exec(p, t, ip);
4709 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4711 struct thread *t = &p->threads[p->thread_id];
4712 struct instruction *ip = t->ip;
4715 __instr_meter_mmm_exec(p, t, ip);
4722 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4724 struct thread *t = &p->threads[p->thread_id];
4725 struct instruction *ip = t->ip;
4728 __instr_meter_mmi_exec(p, t, ip);
4735 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4737 struct thread *t = &p->threads[p->thread_id];
4738 struct instruction *ip = t->ip;
4741 __instr_meter_ihm_exec(p, t, ip);
4748 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4750 struct thread *t = &p->threads[p->thread_id];
4751 struct instruction *ip = t->ip;
4754 __instr_meter_ihi_exec(p, t, ip);
4761 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4763 struct thread *t = &p->threads[p->thread_id];
4764 struct instruction *ip = t->ip;
4767 __instr_meter_imm_exec(p, t, ip);
4774 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4776 struct thread *t = &p->threads[p->thread_id];
4777 struct instruction *ip = t->ip;
4780 __instr_meter_imi_exec(p, t, ip);
4790 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4791 struct action *action __rte_unused,
4794 struct instruction *instr,
4795 struct instruction_data *data)
4797 CHECK(n_tokens == 2, EINVAL);
4799 strcpy(data->jmp_label, tokens[1]);
4801 instr->type = INSTR_JMP;
4802 instr->jmp.ip = NULL; /* Resolved later. */
4807 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4808 struct action *action __rte_unused,
4811 struct instruction *instr,
4812 struct instruction_data *data)
4816 CHECK(n_tokens == 3, EINVAL);
4818 strcpy(data->jmp_label, tokens[1]);
4820 h = header_parse(p, tokens[2]);
4823 instr->type = INSTR_JMP_VALID;
4824 instr->jmp.ip = NULL; /* Resolved later. */
4825 instr->jmp.header_id = h->id;
4830 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4831 struct action *action __rte_unused,
4834 struct instruction *instr,
4835 struct instruction_data *data)
4839 CHECK(n_tokens == 3, EINVAL);
4841 strcpy(data->jmp_label, tokens[1]);
4843 h = header_parse(p, tokens[2]);
4846 instr->type = INSTR_JMP_INVALID;
4847 instr->jmp.ip = NULL; /* Resolved later. */
4848 instr->jmp.header_id = h->id;
4853 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4854 struct action *action,
4857 struct instruction *instr,
4858 struct instruction_data *data)
4860 CHECK(!action, EINVAL);
4861 CHECK(n_tokens == 2, EINVAL);
4863 strcpy(data->jmp_label, tokens[1]);
4865 instr->type = INSTR_JMP_HIT;
4866 instr->jmp.ip = NULL; /* Resolved later. */
4871 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4872 struct action *action,
4875 struct instruction *instr,
4876 struct instruction_data *data)
4878 CHECK(!action, EINVAL);
4879 CHECK(n_tokens == 2, EINVAL);
4881 strcpy(data->jmp_label, tokens[1]);
4883 instr->type = INSTR_JMP_MISS;
4884 instr->jmp.ip = NULL; /* Resolved later. */
4889 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4890 struct action *action,
4893 struct instruction *instr,
4894 struct instruction_data *data)
4898 CHECK(!action, EINVAL);
4899 CHECK(n_tokens == 3, EINVAL);
4901 strcpy(data->jmp_label, tokens[1]);
4903 a = action_find(p, tokens[2]);
4906 instr->type = INSTR_JMP_ACTION_HIT;
4907 instr->jmp.ip = NULL; /* Resolved later. */
4908 instr->jmp.action_id = a->id;
4913 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4914 struct action *action,
4917 struct instruction *instr,
4918 struct instruction_data *data)
4922 CHECK(!action, EINVAL);
4923 CHECK(n_tokens == 3, EINVAL);
4925 strcpy(data->jmp_label, tokens[1]);
4927 a = action_find(p, tokens[2]);
4930 instr->type = INSTR_JMP_ACTION_MISS;
4931 instr->jmp.ip = NULL; /* Resolved later. */
4932 instr->jmp.action_id = a->id;
4937 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4938 struct action *action,
4941 struct instruction *instr,
4942 struct instruction_data *data)
4944 char *a = tokens[2], *b = tokens[3];
4945 struct field *fa, *fb;
4947 uint32_t a_struct_id, b_struct_id;
4949 CHECK(n_tokens == 4, EINVAL);
4951 strcpy(data->jmp_label, tokens[1]);
4953 fa = struct_field_parse(p, action, a, &a_struct_id);
4955 CHECK(!fa->var_size, EINVAL);
4957 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4958 fb = struct_field_parse(p, action, b, &b_struct_id);
4960 CHECK(!fb->var_size, EINVAL);
4962 instr->type = INSTR_JMP_EQ;
4963 if (a[0] != 'h' && b[0] == 'h')
4964 instr->type = INSTR_JMP_EQ_MH;
4965 if (a[0] == 'h' && b[0] != 'h')
4966 instr->type = INSTR_JMP_EQ_HM;
4967 if (a[0] == 'h' && b[0] == 'h')
4968 instr->type = INSTR_JMP_EQ_HH;
4969 instr->jmp.ip = NULL; /* Resolved later. */
4971 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4972 instr->jmp.a.n_bits = fa->n_bits;
4973 instr->jmp.a.offset = fa->offset / 8;
4974 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4975 instr->jmp.b.n_bits = fb->n_bits;
4976 instr->jmp.b.offset = fb->offset / 8;
4981 b_val = strtoull(b, &b, 0);
4982 CHECK(!b[0], EINVAL);
4985 b_val = hton64(b_val) >> (64 - fa->n_bits);
4987 instr->type = INSTR_JMP_EQ_I;
4988 instr->jmp.ip = NULL; /* Resolved later. */
4989 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4990 instr->jmp.a.n_bits = fa->n_bits;
4991 instr->jmp.a.offset = fa->offset / 8;
4992 instr->jmp.b_val = b_val;
4997 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
4998 struct action *action,
5001 struct instruction *instr,
5002 struct instruction_data *data)
5004 char *a = tokens[2], *b = tokens[3];
5005 struct field *fa, *fb;
5007 uint32_t a_struct_id, b_struct_id;
5009 CHECK(n_tokens == 4, EINVAL);
5011 strcpy(data->jmp_label, tokens[1]);
5013 fa = struct_field_parse(p, action, a, &a_struct_id);
5015 CHECK(!fa->var_size, EINVAL);
5017 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
5018 fb = struct_field_parse(p, action, b, &b_struct_id);
5020 CHECK(!fb->var_size, EINVAL);
5022 instr->type = INSTR_JMP_NEQ;
5023 if (a[0] != 'h' && b[0] == 'h')
5024 instr->type = INSTR_JMP_NEQ_MH;
5025 if (a[0] == 'h' && b[0] != 'h')
5026 instr->type = INSTR_JMP_NEQ_HM;
5027 if (a[0] == 'h' && b[0] == 'h')
5028 instr->type = INSTR_JMP_NEQ_HH;
5029 instr->jmp.ip = NULL; /* Resolved later. */
5031 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5032 instr->jmp.a.n_bits = fa->n_bits;
5033 instr->jmp.a.offset = fa->offset / 8;
5034 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5035 instr->jmp.b.n_bits = fb->n_bits;
5036 instr->jmp.b.offset = fb->offset / 8;
5041 b_val = strtoull(b, &b, 0);
5042 CHECK(!b[0], EINVAL);
5045 b_val = hton64(b_val) >> (64 - fa->n_bits);
5047 instr->type = INSTR_JMP_NEQ_I;
5048 instr->jmp.ip = NULL; /* Resolved later. */
5049 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5050 instr->jmp.a.n_bits = fa->n_bits;
5051 instr->jmp.a.offset = fa->offset / 8;
5052 instr->jmp.b_val = b_val;
5057 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
5058 struct action *action,
5061 struct instruction *instr,
5062 struct instruction_data *data)
5064 char *a = tokens[2], *b = tokens[3];
5065 struct field *fa, *fb;
5067 uint32_t a_struct_id, b_struct_id;
5069 CHECK(n_tokens == 4, EINVAL);
5071 strcpy(data->jmp_label, tokens[1]);
5073 fa = struct_field_parse(p, action, a, &a_struct_id);
5075 CHECK(!fa->var_size, EINVAL);
5077 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
5078 fb = struct_field_parse(p, action, b, &b_struct_id);
5080 CHECK(!fb->var_size, EINVAL);
5082 instr->type = INSTR_JMP_LT;
5083 if (a[0] == 'h' && b[0] != 'h')
5084 instr->type = INSTR_JMP_LT_HM;
5085 if (a[0] != 'h' && b[0] == 'h')
5086 instr->type = INSTR_JMP_LT_MH;
5087 if (a[0] == 'h' && b[0] == 'h')
5088 instr->type = INSTR_JMP_LT_HH;
5089 instr->jmp.ip = NULL; /* Resolved later. */
5091 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5092 instr->jmp.a.n_bits = fa->n_bits;
5093 instr->jmp.a.offset = fa->offset / 8;
5094 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5095 instr->jmp.b.n_bits = fb->n_bits;
5096 instr->jmp.b.offset = fb->offset / 8;
5100 /* JMP_LT_MI, JMP_LT_HI. */
5101 b_val = strtoull(b, &b, 0);
5102 CHECK(!b[0], EINVAL);
5104 instr->type = INSTR_JMP_LT_MI;
5106 instr->type = INSTR_JMP_LT_HI;
5107 instr->jmp.ip = NULL; /* Resolved later. */
5109 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5110 instr->jmp.a.n_bits = fa->n_bits;
5111 instr->jmp.a.offset = fa->offset / 8;
5112 instr->jmp.b_val = b_val;
5117 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5118 struct action *action,
5121 struct instruction *instr,
5122 struct instruction_data *data)
5124 char *a = tokens[2], *b = tokens[3];
5125 struct field *fa, *fb;
5127 uint32_t a_struct_id, b_struct_id;
5129 CHECK(n_tokens == 4, EINVAL);
5131 strcpy(data->jmp_label, tokens[1]);
5133 fa = struct_field_parse(p, action, a, &a_struct_id);
5135 CHECK(!fa->var_size, EINVAL);
5137 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5138 fb = struct_field_parse(p, action, b, &b_struct_id);
5140 CHECK(!fb->var_size, EINVAL);
5142 instr->type = INSTR_JMP_GT;
5143 if (a[0] == 'h' && b[0] != 'h')
5144 instr->type = INSTR_JMP_GT_HM;
5145 if (a[0] != 'h' && b[0] == 'h')
5146 instr->type = INSTR_JMP_GT_MH;
5147 if (a[0] == 'h' && b[0] == 'h')
5148 instr->type = INSTR_JMP_GT_HH;
5149 instr->jmp.ip = NULL; /* Resolved later. */
5151 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5152 instr->jmp.a.n_bits = fa->n_bits;
5153 instr->jmp.a.offset = fa->offset / 8;
5154 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5155 instr->jmp.b.n_bits = fb->n_bits;
5156 instr->jmp.b.offset = fb->offset / 8;
5160 /* JMP_GT_MI, JMP_GT_HI. */
5161 b_val = strtoull(b, &b, 0);
5162 CHECK(!b[0], EINVAL);
5164 instr->type = INSTR_JMP_GT_MI;
5166 instr->type = INSTR_JMP_GT_HI;
5167 instr->jmp.ip = NULL; /* Resolved later. */
5169 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5170 instr->jmp.a.n_bits = fa->n_bits;
5171 instr->jmp.a.offset = fa->offset / 8;
5172 instr->jmp.b_val = b_val;
5177 instr_jmp_exec(struct rte_swx_pipeline *p)
5179 struct thread *t = &p->threads[p->thread_id];
5180 struct instruction *ip = t->ip;
5182 TRACE("[Thread %2u] jmp\n", p->thread_id);
5184 thread_ip_set(t, ip->jmp.ip);
5188 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5190 struct thread *t = &p->threads[p->thread_id];
5191 struct instruction *ip = t->ip;
5192 uint32_t header_id = ip->jmp.header_id;
5194 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5196 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5200 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5202 struct thread *t = &p->threads[p->thread_id];
5203 struct instruction *ip = t->ip;
5204 uint32_t header_id = ip->jmp.header_id;
5206 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5208 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5212 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5214 struct thread *t = &p->threads[p->thread_id];
5215 struct instruction *ip = t->ip;
5216 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5218 TRACE("[Thread %2u] jmph\n", p->thread_id);
5220 t->ip = ip_next[t->hit];
5224 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5226 struct thread *t = &p->threads[p->thread_id];
5227 struct instruction *ip = t->ip;
5228 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5230 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5232 t->ip = ip_next[t->hit];
5236 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5238 struct thread *t = &p->threads[p->thread_id];
5239 struct instruction *ip = t->ip;
5241 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5243 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5247 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5249 struct thread *t = &p->threads[p->thread_id];
5250 struct instruction *ip = t->ip;
5252 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5254 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5258 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5260 struct thread *t = &p->threads[p->thread_id];
5261 struct instruction *ip = t->ip;
5263 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5269 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5271 struct thread *t = &p->threads[p->thread_id];
5272 struct instruction *ip = t->ip;
5274 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5276 JMP_CMP_MH(t, ip, ==);
5280 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5282 struct thread *t = &p->threads[p->thread_id];
5283 struct instruction *ip = t->ip;
5285 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5287 JMP_CMP_HM(t, ip, ==);
5291 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5293 struct thread *t = &p->threads[p->thread_id];
5294 struct instruction *ip = t->ip;
5296 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5298 JMP_CMP_HH_FAST(t, ip, ==);
5302 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5304 struct thread *t = &p->threads[p->thread_id];
5305 struct instruction *ip = t->ip;
5307 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5309 JMP_CMP_I(t, ip, ==);
5313 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5315 struct thread *t = &p->threads[p->thread_id];
5316 struct instruction *ip = t->ip;
5318 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5324 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5326 struct thread *t = &p->threads[p->thread_id];
5327 struct instruction *ip = t->ip;
5329 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5331 JMP_CMP_MH(t, ip, !=);
5335 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5337 struct thread *t = &p->threads[p->thread_id];
5338 struct instruction *ip = t->ip;
5340 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5342 JMP_CMP_HM(t, ip, !=);
5346 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5348 struct thread *t = &p->threads[p->thread_id];
5349 struct instruction *ip = t->ip;
5351 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5353 JMP_CMP_HH_FAST(t, ip, !=);
5357 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5359 struct thread *t = &p->threads[p->thread_id];
5360 struct instruction *ip = t->ip;
5362 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5364 JMP_CMP_I(t, ip, !=);
5368 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5370 struct thread *t = &p->threads[p->thread_id];
5371 struct instruction *ip = t->ip;
5373 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5379 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5381 struct thread *t = &p->threads[p->thread_id];
5382 struct instruction *ip = t->ip;
5384 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5386 JMP_CMP_MH(t, ip, <);
5390 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5392 struct thread *t = &p->threads[p->thread_id];
5393 struct instruction *ip = t->ip;
5395 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5397 JMP_CMP_HM(t, ip, <);
5401 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5403 struct thread *t = &p->threads[p->thread_id];
5404 struct instruction *ip = t->ip;
5406 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5408 JMP_CMP_HH(t, ip, <);
5412 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5414 struct thread *t = &p->threads[p->thread_id];
5415 struct instruction *ip = t->ip;
5417 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5419 JMP_CMP_MI(t, ip, <);
5423 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5425 struct thread *t = &p->threads[p->thread_id];
5426 struct instruction *ip = t->ip;
5428 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5430 JMP_CMP_HI(t, ip, <);
5434 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5436 struct thread *t = &p->threads[p->thread_id];
5437 struct instruction *ip = t->ip;
5439 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5445 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5447 struct thread *t = &p->threads[p->thread_id];
5448 struct instruction *ip = t->ip;
5450 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5452 JMP_CMP_MH(t, ip, >);
5456 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5458 struct thread *t = &p->threads[p->thread_id];
5459 struct instruction *ip = t->ip;
5461 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5463 JMP_CMP_HM(t, ip, >);
5467 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5469 struct thread *t = &p->threads[p->thread_id];
5470 struct instruction *ip = t->ip;
5472 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5474 JMP_CMP_HH(t, ip, >);
5478 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5480 struct thread *t = &p->threads[p->thread_id];
5481 struct instruction *ip = t->ip;
5483 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5485 JMP_CMP_MI(t, ip, >);
5489 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5491 struct thread *t = &p->threads[p->thread_id];
5492 struct instruction *ip = t->ip;
5494 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5496 JMP_CMP_HI(t, ip, >);
5503 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5504 struct action *action,
5505 char **tokens __rte_unused,
5507 struct instruction *instr,
5508 struct instruction_data *data __rte_unused)
5510 CHECK(action, EINVAL);
5511 CHECK(n_tokens == 1, EINVAL);
5513 instr->type = INSTR_RETURN;
5518 instr_return_exec(struct rte_swx_pipeline *p)
5520 struct thread *t = &p->threads[p->thread_id];
5522 TRACE("[Thread %2u] return\n", p->thread_id);
5528 instr_translate(struct rte_swx_pipeline *p,
5529 struct action *action,
5531 struct instruction *instr,
5532 struct instruction_data *data)
5534 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5535 int n_tokens = 0, tpos = 0;
5537 /* Parse the instruction string into tokens. */
5541 token = strtok_r(string, " \t\v", &string);
5545 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5546 CHECK_NAME(token, EINVAL);
5548 tokens[n_tokens] = token;
5552 CHECK(n_tokens, EINVAL);
5554 /* Handle the optional instruction label. */
5555 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5556 strcpy(data->label, tokens[0]);
5559 CHECK(n_tokens - tpos, EINVAL);
5562 /* Identify the instruction type. */
5563 if (!strcmp(tokens[tpos], "rx"))
5564 return instr_rx_translate(p,
5571 if (!strcmp(tokens[tpos], "tx"))
5572 return instr_tx_translate(p,
5579 if (!strcmp(tokens[tpos], "drop"))
5580 return instr_drop_translate(p,
5587 if (!strcmp(tokens[tpos], "extract"))
5588 return instr_hdr_extract_translate(p,
5595 if (!strcmp(tokens[tpos], "lookahead"))
5596 return instr_hdr_lookahead_translate(p,
5603 if (!strcmp(tokens[tpos], "emit"))
5604 return instr_hdr_emit_translate(p,
5611 if (!strcmp(tokens[tpos], "validate"))
5612 return instr_hdr_validate_translate(p,
5619 if (!strcmp(tokens[tpos], "invalidate"))
5620 return instr_hdr_invalidate_translate(p,
5627 if (!strcmp(tokens[tpos], "mov"))
5628 return instr_mov_translate(p,
5635 if (!strcmp(tokens[tpos], "add"))
5636 return instr_alu_add_translate(p,
5643 if (!strcmp(tokens[tpos], "sub"))
5644 return instr_alu_sub_translate(p,
5651 if (!strcmp(tokens[tpos], "ckadd"))
5652 return instr_alu_ckadd_translate(p,
5659 if (!strcmp(tokens[tpos], "cksub"))
5660 return instr_alu_cksub_translate(p,
5667 if (!strcmp(tokens[tpos], "and"))
5668 return instr_alu_and_translate(p,
5675 if (!strcmp(tokens[tpos], "or"))
5676 return instr_alu_or_translate(p,
5683 if (!strcmp(tokens[tpos], "xor"))
5684 return instr_alu_xor_translate(p,
5691 if (!strcmp(tokens[tpos], "shl"))
5692 return instr_alu_shl_translate(p,
5699 if (!strcmp(tokens[tpos], "shr"))
5700 return instr_alu_shr_translate(p,
5707 if (!strcmp(tokens[tpos], "regprefetch"))
5708 return instr_regprefetch_translate(p,
5715 if (!strcmp(tokens[tpos], "regrd"))
5716 return instr_regrd_translate(p,
5723 if (!strcmp(tokens[tpos], "regwr"))
5724 return instr_regwr_translate(p,
5731 if (!strcmp(tokens[tpos], "regadd"))
5732 return instr_regadd_translate(p,
5739 if (!strcmp(tokens[tpos], "metprefetch"))
5740 return instr_metprefetch_translate(p,
5747 if (!strcmp(tokens[tpos], "meter"))
5748 return instr_meter_translate(p,
5755 if (!strcmp(tokens[tpos], "table"))
5756 return instr_table_translate(p,
5763 if (!strcmp(tokens[tpos], "learn"))
5764 return instr_learn_translate(p,
5771 if (!strcmp(tokens[tpos], "forget"))
5772 return instr_forget_translate(p,
5779 if (!strcmp(tokens[tpos], "extern"))
5780 return instr_extern_translate(p,
5787 if (!strcmp(tokens[tpos], "jmp"))
5788 return instr_jmp_translate(p,
5795 if (!strcmp(tokens[tpos], "jmpv"))
5796 return instr_jmp_valid_translate(p,
5803 if (!strcmp(tokens[tpos], "jmpnv"))
5804 return instr_jmp_invalid_translate(p,
5811 if (!strcmp(tokens[tpos], "jmph"))
5812 return instr_jmp_hit_translate(p,
5819 if (!strcmp(tokens[tpos], "jmpnh"))
5820 return instr_jmp_miss_translate(p,
5827 if (!strcmp(tokens[tpos], "jmpa"))
5828 return instr_jmp_action_hit_translate(p,
5835 if (!strcmp(tokens[tpos], "jmpna"))
5836 return instr_jmp_action_miss_translate(p,
5843 if (!strcmp(tokens[tpos], "jmpeq"))
5844 return instr_jmp_eq_translate(p,
5851 if (!strcmp(tokens[tpos], "jmpneq"))
5852 return instr_jmp_neq_translate(p,
5859 if (!strcmp(tokens[tpos], "jmplt"))
5860 return instr_jmp_lt_translate(p,
5867 if (!strcmp(tokens[tpos], "jmpgt"))
5868 return instr_jmp_gt_translate(p,
5875 if (!strcmp(tokens[tpos], "return"))
5876 return instr_return_translate(p,
5886 static struct instruction_data *
5887 label_find(struct instruction_data *data, uint32_t n, const char *label)
5891 for (i = 0; i < n; i++)
5892 if (!strcmp(label, data[i].label))
5899 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5901 uint32_t count = 0, i;
5906 for (i = 0; i < n; i++)
5907 if (!strcmp(label, data[i].jmp_label))
5914 instr_label_check(struct instruction_data *instruction_data,
5915 uint32_t n_instructions)
5919 /* Check that all instruction labels are unique. */
5920 for (i = 0; i < n_instructions; i++) {
5921 struct instruction_data *data = &instruction_data[i];
5922 char *label = data->label;
5928 for (j = i + 1; j < n_instructions; j++)
5929 CHECK(strcmp(label, data[j].label), EINVAL);
5932 /* Get users for each instruction label. */
5933 for (i = 0; i < n_instructions; i++) {
5934 struct instruction_data *data = &instruction_data[i];
5935 char *label = data->label;
5937 data->n_users = label_is_used(instruction_data,
5946 instr_jmp_resolve(struct instruction *instructions,
5947 struct instruction_data *instruction_data,
5948 uint32_t n_instructions)
5952 for (i = 0; i < n_instructions; i++) {
5953 struct instruction *instr = &instructions[i];
5954 struct instruction_data *data = &instruction_data[i];
5955 struct instruction_data *found;
5957 if (!instruction_is_jmp(instr))
5960 found = label_find(instruction_data,
5963 CHECK(found, EINVAL);
5965 instr->jmp.ip = &instructions[found - instruction_data];
5972 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5974 struct instruction *instr,
5975 struct instruction_data *data __rte_unused,
5976 uint32_t n_instructions)
5979 enum instruction_type type;
5982 /* Check that the first instruction is rx. */
5983 CHECK(instr[0].type == INSTR_RX, EINVAL);
5985 /* Check that there is at least one tx instruction. */
5986 for (i = 0; i < n_instructions; i++) {
5987 type = instr[i].type;
5989 if (instruction_is_tx(type))
5992 CHECK(i < n_instructions, EINVAL);
5994 /* Check that the last instruction is either tx or unconditional
5997 type = instr[n_instructions - 1].type;
5998 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
6002 enum instruction_type type;
6005 /* Check that there is at least one return or tx instruction. */
6006 for (i = 0; i < n_instructions; i++) {
6007 type = instr[i].type;
6009 if ((type == INSTR_RETURN) || instruction_is_tx(type))
6012 CHECK(i < n_instructions, EINVAL);
6019 instr_compact(struct instruction *instructions,
6020 struct instruction_data *instruction_data,
6021 uint32_t n_instructions)
6023 uint32_t i, pos = 0;
6025 /* Eliminate the invalid instructions that have been optimized out. */
6026 for (i = 0; i < n_instructions; i++) {
6027 struct instruction *instr = &instructions[i];
6028 struct instruction_data *data = &instruction_data[i];
6034 memcpy(&instructions[pos], instr, sizeof(*instr));
6035 memcpy(&instruction_data[pos], data, sizeof(*data));
6045 instr_pattern_extract_many_search(struct instruction *instr,
6046 struct instruction_data *data,
6048 uint32_t *n_pattern_instr)
6052 for (i = 0; i < n_instr; i++) {
6053 if (data[i].invalid)
6056 if (instr[i].type != INSTR_HDR_EXTRACT)
6059 if (i == RTE_DIM(instr->io.hdr.header_id))
6062 if (i && data[i].n_users)
6069 *n_pattern_instr = i;
6074 instr_pattern_extract_many_replace(struct instruction *instr,
6075 struct instruction_data *data,
6080 for (i = 1; i < n_instr; i++) {
6082 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6083 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6084 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6086 data[i].invalid = 1;
6091 instr_pattern_extract_many_optimize(struct instruction *instructions,
6092 struct instruction_data *instruction_data,
6093 uint32_t n_instructions)
6097 for (i = 0; i < n_instructions; ) {
6098 struct instruction *instr = &instructions[i];
6099 struct instruction_data *data = &instruction_data[i];
6100 uint32_t n_instr = 0;
6104 detected = instr_pattern_extract_many_search(instr,
6109 instr_pattern_extract_many_replace(instr,
6116 /* No pattern starting at the current instruction. */
6120 /* Eliminate the invalid instructions that have been optimized out. */
6121 n_instructions = instr_compact(instructions,
6125 return n_instructions;
6129 instr_pattern_emit_many_tx_search(struct instruction *instr,
6130 struct instruction_data *data,
6132 uint32_t *n_pattern_instr)
6136 for (i = 0; i < n_instr; i++) {
6137 if (data[i].invalid)
6140 if (instr[i].type != INSTR_HDR_EMIT)
6143 if (i == RTE_DIM(instr->io.hdr.header_id))
6146 if (i && data[i].n_users)
6153 if (!instruction_is_tx(instr[i].type))
6156 if (data[i].n_users)
6161 *n_pattern_instr = i;
6166 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6167 struct instruction_data *data,
6172 /* Any emit instruction in addition to the first one. */
6173 for (i = 1; i < n_instr - 1; i++) {
6175 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6176 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6177 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6179 data[i].invalid = 1;
6182 /* The TX instruction is the last one in the pattern. */
6184 instr[0].io.io.offset = instr[i].io.io.offset;
6185 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6186 data[i].invalid = 1;
6190 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6191 struct instruction_data *instruction_data,
6192 uint32_t n_instructions)
6196 for (i = 0; i < n_instructions; ) {
6197 struct instruction *instr = &instructions[i];
6198 struct instruction_data *data = &instruction_data[i];
6199 uint32_t n_instr = 0;
6202 /* Emit many + TX. */
6203 detected = instr_pattern_emit_many_tx_search(instr,
6208 instr_pattern_emit_many_tx_replace(instr,
6215 /* No pattern starting at the current instruction. */
6219 /* Eliminate the invalid instructions that have been optimized out. */
6220 n_instructions = instr_compact(instructions,
6224 return n_instructions;
6228 action_arg_src_mov_count(struct action *a,
6230 struct instruction *instructions,
6231 struct instruction_data *instruction_data,
6232 uint32_t n_instructions);
6235 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6237 struct instruction *instr,
6238 struct instruction_data *data,
6240 struct instruction *instructions,
6241 struct instruction_data *instruction_data,
6242 uint32_t n_instructions,
6243 uint32_t *n_pattern_instr)
6246 uint32_t src_field_id, i, j;
6248 /* Prerequisites. */
6252 /* First instruction: MOV_HM. */
6253 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6256 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6257 if (!h || h->st->var_size)
6260 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6261 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6264 if (src_field_id == a->st->n_fields)
6267 if (instr[0].mov.dst.offset ||
6268 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6269 instr[0].mov.src.struct_id ||
6270 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6271 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6274 if ((n_instr < h->st->n_fields + 1) ||
6275 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6278 /* Subsequent instructions: MOV_HM. */
6279 for (i = 1; i < h->st->n_fields; i++)
6280 if (data[i].invalid ||
6282 (instr[i].type != INSTR_MOV_HM) ||
6283 (instr[i].mov.dst.struct_id != h->struct_id) ||
6284 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6285 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6286 instr[i].mov.src.struct_id ||
6287 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6288 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6289 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6292 /* Last instruction: HDR_VALIDATE. */
6293 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6294 (instr[i].valid.header_id != h->id))
6297 /* Check that none of the action args that are used as source for this
6298 * DMA transfer are not used as source in any other mov instruction.
6300 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6303 n_users = action_arg_src_mov_count(a,
6312 *n_pattern_instr = 1 + i;
6317 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6319 struct instruction *instr,
6320 struct instruction_data *data,
6324 uint32_t src_field_id, src_offset, i;
6326 /* Read from the instructions before they are modified. */
6327 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6331 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6332 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6335 if (src_field_id == a->st->n_fields)
6338 src_offset = instr[0].mov.src.offset;
6340 /* Modify the instructions. */
6341 instr[0].type = INSTR_DMA_HT;
6342 instr[0].dma.dst.header_id[0] = h->id;
6343 instr[0].dma.dst.struct_id[0] = h->struct_id;
6344 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6345 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6347 for (i = 1; i < n_instr; i++)
6348 data[i].invalid = 1;
6350 /* Update the endianness of the action arguments to header endianness. */
6351 for (i = 0; i < h->st->n_fields; i++)
6352 a->args_endianness[src_field_id + i] = 1;
6356 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6358 struct instruction *instructions,
6359 struct instruction_data *instruction_data,
6360 uint32_t n_instructions)
6365 return n_instructions;
6367 for (i = 0; i < n_instructions; ) {
6368 struct instruction *instr = &instructions[i];
6369 struct instruction_data *data = &instruction_data[i];
6370 uint32_t n_instr = 0;
6373 /* Mov all + validate. */
6374 detected = instr_pattern_mov_all_validate_search(p,
6384 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6389 /* No pattern starting at the current instruction. */
6393 /* Eliminate the invalid instructions that have been optimized out. */
6394 n_instructions = instr_compact(instructions,
6398 return n_instructions;
6402 instr_pattern_dma_many_search(struct instruction *instr,
6403 struct instruction_data *data,
6405 uint32_t *n_pattern_instr)
6409 for (i = 0; i < n_instr; i++) {
6410 if (data[i].invalid)
6413 if (instr[i].type != INSTR_DMA_HT)
6416 if (i == RTE_DIM(instr->dma.dst.header_id))
6419 if (i && data[i].n_users)
6426 *n_pattern_instr = i;
6431 instr_pattern_dma_many_replace(struct instruction *instr,
6432 struct instruction_data *data,
6437 for (i = 1; i < n_instr; i++) {
6439 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6440 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6441 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6442 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6444 data[i].invalid = 1;
6449 instr_pattern_dma_many_optimize(struct instruction *instructions,
6450 struct instruction_data *instruction_data,
6451 uint32_t n_instructions)
6455 for (i = 0; i < n_instructions; ) {
6456 struct instruction *instr = &instructions[i];
6457 struct instruction_data *data = &instruction_data[i];
6458 uint32_t n_instr = 0;
6462 detected = instr_pattern_dma_many_search(instr,
6467 instr_pattern_dma_many_replace(instr, data, n_instr);
6472 /* No pattern starting at the current instruction. */
6476 /* Eliminate the invalid instructions that have been optimized out. */
6477 n_instructions = instr_compact(instructions,
6481 return n_instructions;
6485 instr_optimize(struct rte_swx_pipeline *p,
6487 struct instruction *instructions,
6488 struct instruction_data *instruction_data,
6489 uint32_t n_instructions)
6492 n_instructions = instr_pattern_extract_many_optimize(instructions,
6496 /* Emit many + TX. */
6497 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6501 /* Mov all + validate. */
6502 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6509 n_instructions = instr_pattern_dma_many_optimize(instructions,
6513 return n_instructions;
6517 instruction_config(struct rte_swx_pipeline *p,
6519 const char **instructions,
6520 uint32_t n_instructions)
6522 struct instruction *instr = NULL;
6523 struct instruction_data *data = NULL;
6527 CHECK(n_instructions, EINVAL);
6528 CHECK(instructions, EINVAL);
6529 for (i = 0; i < n_instructions; i++)
6530 CHECK_INSTRUCTION(instructions[i], EINVAL);
6532 /* Memory allocation. */
6533 instr = calloc(n_instructions, sizeof(struct instruction));
6539 data = calloc(n_instructions, sizeof(struct instruction_data));
6545 for (i = 0; i < n_instructions; i++) {
6546 char *string = strdup(instructions[i]);
6552 err = instr_translate(p, a, string, &instr[i], &data[i]);
6561 err = instr_label_check(data, n_instructions);
6565 err = instr_verify(p, a, instr, data, n_instructions);
6569 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6571 err = instr_jmp_resolve(instr, data, n_instructions);
6576 a->instructions = instr;
6577 a->instruction_data = data;
6578 a->n_instructions = n_instructions;
6580 p->instructions = instr;
6581 p->instruction_data = data;
6582 p->n_instructions = n_instructions;
6593 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
6595 static instr_exec_t instruction_table[] = {
6596 [INSTR_RX] = instr_rx_exec,
6597 [INSTR_TX] = instr_tx_exec,
6598 [INSTR_TX_I] = instr_tx_i_exec,
6600 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6601 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6602 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6603 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6604 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6605 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6606 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6607 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6608 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6609 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6611 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6612 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6613 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6614 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6615 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6616 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6617 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6618 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6619 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6621 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6622 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6624 [INSTR_MOV] = instr_mov_exec,
6625 [INSTR_MOV_MH] = instr_mov_mh_exec,
6626 [INSTR_MOV_HM] = instr_mov_hm_exec,
6627 [INSTR_MOV_HH] = instr_mov_hh_exec,
6628 [INSTR_MOV_I] = instr_mov_i_exec,
6630 [INSTR_DMA_HT] = instr_dma_ht_exec,
6631 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6632 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6633 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6634 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6635 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6636 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6637 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6639 [INSTR_ALU_ADD] = instr_alu_add_exec,
6640 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6641 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6642 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6643 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6644 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6646 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6647 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6648 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6649 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6650 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6651 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6653 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6654 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6655 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6656 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6658 [INSTR_ALU_AND] = instr_alu_and_exec,
6659 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6660 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6661 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6662 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6664 [INSTR_ALU_OR] = instr_alu_or_exec,
6665 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6666 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6667 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6668 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6670 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6671 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6672 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6673 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6674 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6676 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6677 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6678 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6679 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6680 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6681 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6683 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6684 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6685 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6686 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6687 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6688 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6690 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6691 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6692 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6694 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6695 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6696 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6697 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6698 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6699 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6701 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6702 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6703 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6704 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6705 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6706 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6707 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6708 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6709 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6711 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6712 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6713 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6714 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6715 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6716 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6717 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6718 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6719 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6721 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6722 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6723 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6725 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6726 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6727 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6728 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6729 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6730 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6731 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6732 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6733 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6734 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6735 [INSTR_METER_IMM] = instr_meter_imm_exec,
6736 [INSTR_METER_IMI] = instr_meter_imi_exec,
6738 [INSTR_TABLE] = instr_table_exec,
6739 [INSTR_TABLE_AF] = instr_table_af_exec,
6740 [INSTR_SELECTOR] = instr_selector_exec,
6741 [INSTR_LEARNER] = instr_learner_exec,
6742 [INSTR_LEARNER_AF] = instr_learner_af_exec,
6743 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6744 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6745 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6746 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6748 [INSTR_JMP] = instr_jmp_exec,
6749 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6750 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6751 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6752 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6753 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6754 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6756 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6757 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6758 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6759 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6760 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6762 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6763 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6764 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6765 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6766 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6768 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6769 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6770 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6771 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6772 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6773 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6775 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6776 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6777 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6778 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6779 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6780 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6782 [INSTR_RETURN] = instr_return_exec,
6786 instr_exec(struct rte_swx_pipeline *p)
6788 struct thread *t = &p->threads[p->thread_id];
6789 struct instruction *ip = t->ip;
6790 instr_exec_t instr = instruction_table[ip->type];
6798 static struct action *
6799 action_find(struct rte_swx_pipeline *p, const char *name)
6801 struct action *elem;
6806 TAILQ_FOREACH(elem, &p->actions, node)
6807 if (strcmp(elem->name, name) == 0)
6813 static struct action *
6814 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6816 struct action *action = NULL;
6818 TAILQ_FOREACH(action, &p->actions, node)
6819 if (action->id == id)
6825 static struct field *
6826 action_field_find(struct action *a, const char *name)
6828 return a->st ? struct_type_field_find(a->st, name) : NULL;
6831 static struct field *
6832 action_field_parse(struct action *action, const char *name)
6834 if (name[0] != 't' || name[1] != '.')
6837 return action_field_find(action, &name[2]);
6841 action_has_nbo_args(struct action *a)
6845 /* Return if the action does not have any args. */
6847 return 0; /* FALSE */
6849 for (i = 0; i < a->st->n_fields; i++)
6850 if (a->args_endianness[i])
6851 return 1; /* TRUE */
6853 return 0; /* FALSE */
6857 action_does_learning(struct action *a)
6861 for (i = 0; i < a->n_instructions; i++)
6862 switch (a->instructions[i].type) {
6863 case INSTR_LEARNER_LEARN:
6864 return 1; /* TRUE */
6866 case INSTR_LEARNER_FORGET:
6867 return 1; /* TRUE */
6873 return 0; /* FALSE */
6877 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6879 const char *args_struct_type_name,
6880 const char **instructions,
6881 uint32_t n_instructions)
6883 struct struct_type *args_struct_type = NULL;
6889 CHECK_NAME(name, EINVAL);
6890 CHECK(!action_find(p, name), EEXIST);
6892 if (args_struct_type_name) {
6893 CHECK_NAME(args_struct_type_name, EINVAL);
6894 args_struct_type = struct_type_find(p, args_struct_type_name);
6895 CHECK(args_struct_type, EINVAL);
6896 CHECK(!args_struct_type->var_size, EINVAL);
6899 /* Node allocation. */
6900 a = calloc(1, sizeof(struct action));
6902 if (args_struct_type) {
6903 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6904 if (!a->args_endianness) {
6910 /* Node initialization. */
6911 strcpy(a->name, name);
6912 a->st = args_struct_type;
6913 a->id = p->n_actions;
6915 /* Instruction translation. */
6916 err = instruction_config(p, a, instructions, n_instructions);
6918 free(a->args_endianness);
6923 /* Node add to tailq. */
6924 TAILQ_INSERT_TAIL(&p->actions, a, node);
6931 action_build(struct rte_swx_pipeline *p)
6933 struct action *action;
6935 /* p->action_instructions. */
6936 p->action_instructions = calloc(p->n_actions, sizeof(struct instruction *));
6937 CHECK(p->action_instructions, ENOMEM);
6939 TAILQ_FOREACH(action, &p->actions, node)
6940 p->action_instructions[action->id] = action->instructions;
6942 /* p->action_funcs. */
6943 p->action_funcs = calloc(p->n_actions, sizeof(action_func_t));
6944 CHECK(p->action_funcs, ENOMEM);
6950 action_build_free(struct rte_swx_pipeline *p)
6952 free(p->action_funcs);
6953 p->action_funcs = NULL;
6955 free(p->action_instructions);
6956 p->action_instructions = NULL;
6960 action_free(struct rte_swx_pipeline *p)
6962 action_build_free(p);
6965 struct action *action;
6967 action = TAILQ_FIRST(&p->actions);
6971 TAILQ_REMOVE(&p->actions, action, node);
6972 free(action->instruction_data);
6973 free(action->instructions);
6979 action_arg_src_mov_count(struct action *a,
6981 struct instruction *instructions,
6982 struct instruction_data *instruction_data,
6983 uint32_t n_instructions)
6985 uint32_t offset, n_users = 0, i;
6988 (arg_id >= a->st->n_fields) ||
6990 !instruction_data ||
6994 offset = a->st->fields[arg_id].offset / 8;
6996 for (i = 0; i < n_instructions; i++) {
6997 struct instruction *instr = &instructions[i];
6998 struct instruction_data *data = &instruction_data[i];
7000 if (data->invalid ||
7001 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
7002 instr->mov.src.struct_id ||
7003 (instr->mov.src.offset != offset))
7015 static struct table_type *
7016 table_type_find(struct rte_swx_pipeline *p, const char *name)
7018 struct table_type *elem;
7020 TAILQ_FOREACH(elem, &p->table_types, node)
7021 if (strcmp(elem->name, name) == 0)
7027 static struct table_type *
7028 table_type_resolve(struct rte_swx_pipeline *p,
7029 const char *recommended_type_name,
7030 enum rte_swx_table_match_type match_type)
7032 struct table_type *elem;
7034 /* Only consider the recommended type if the match type is correct. */
7035 if (recommended_type_name)
7036 TAILQ_FOREACH(elem, &p->table_types, node)
7037 if (!strcmp(elem->name, recommended_type_name) &&
7038 (elem->match_type == match_type))
7041 /* Ignore the recommended type and get the first element with this match
7044 TAILQ_FOREACH(elem, &p->table_types, node)
7045 if (elem->match_type == match_type)
7051 static struct table *
7052 table_find(struct rte_swx_pipeline *p, const char *name)
7056 TAILQ_FOREACH(elem, &p->tables, node)
7057 if (strcmp(elem->name, name) == 0)
7063 static struct table *
7064 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7066 struct table *table = NULL;
7068 TAILQ_FOREACH(table, &p->tables, node)
7069 if (table->id == id)
7076 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
7078 enum rte_swx_table_match_type match_type,
7079 struct rte_swx_table_ops *ops)
7081 struct table_type *elem;
7085 CHECK_NAME(name, EINVAL);
7086 CHECK(!table_type_find(p, name), EEXIST);
7089 CHECK(ops->create, EINVAL);
7090 CHECK(ops->lkp, EINVAL);
7091 CHECK(ops->free, EINVAL);
7093 /* Node allocation. */
7094 elem = calloc(1, sizeof(struct table_type));
7095 CHECK(elem, ENOMEM);
7097 /* Node initialization. */
7098 strcpy(elem->name, name);
7099 elem->match_type = match_type;
7100 memcpy(&elem->ops, ops, sizeof(*ops));
7102 /* Node add to tailq. */
7103 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
7109 table_match_type_resolve(struct rte_swx_match_field_params *fields,
7111 enum rte_swx_table_match_type *match_type)
7113 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
7115 for (i = 0; i < n_fields; i++) {
7116 struct rte_swx_match_field_params *f = &fields[i];
7118 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
7121 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
7125 if ((n_fields_lpm > 1) ||
7126 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7129 *match_type = (n_fields_em == n_fields) ?
7130 RTE_SWX_TABLE_MATCH_EXACT :
7131 RTE_SWX_TABLE_MATCH_WILDCARD;
7137 table_match_fields_check(struct rte_swx_pipeline *p,
7138 struct rte_swx_pipeline_table_params *params,
7139 struct header **header)
7141 struct header *h0 = NULL;
7142 struct field *hf, *mf;
7143 uint32_t *offset = NULL, i;
7146 /* Return if no match fields. */
7147 if (!params->n_fields) {
7148 if (params->fields) {
7159 /* Memory allocation. */
7160 offset = calloc(params->n_fields, sizeof(uint32_t));
7166 /* Check that all the match fields belong to either the same header or
7169 hf = header_field_parse(p, params->fields[0].name, &h0);
7170 mf = metadata_field_parse(p, params->fields[0].name);
7171 if ((!hf && !mf) || (hf && hf->var_size)) {
7176 offset[0] = h0 ? hf->offset : mf->offset;
7178 for (i = 1; i < params->n_fields; i++)
7182 hf = header_field_parse(p, params->fields[i].name, &h);
7183 if (!hf || (h->id != h0->id) || hf->var_size) {
7188 offset[i] = hf->offset;
7190 mf = metadata_field_parse(p, params->fields[i].name);
7196 offset[i] = mf->offset;
7199 /* Check that there are no duplicated match fields. */
7200 for (i = 0; i < params->n_fields; i++) {
7203 for (j = 0; j < i; j++)
7204 if (offset[j] == offset[i]) {
7220 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7222 struct rte_swx_pipeline_table_params *params,
7223 const char *recommended_table_type_name,
7227 struct table_type *type;
7229 struct action *default_action;
7230 struct header *header = NULL;
7231 uint32_t action_data_size_max = 0, i;
7236 CHECK_NAME(name, EINVAL);
7237 CHECK(!table_find(p, name), EEXIST);
7238 CHECK(!selector_find(p, name), EEXIST);
7239 CHECK(!learner_find(p, name), EEXIST);
7241 CHECK(params, EINVAL);
7244 status = table_match_fields_check(p, params, &header);
7248 /* Action checks. */
7249 CHECK(params->n_actions, EINVAL);
7250 CHECK(params->action_names, EINVAL);
7251 for (i = 0; i < params->n_actions; i++) {
7252 const char *action_name = params->action_names[i];
7254 uint32_t action_data_size;
7256 CHECK_NAME(action_name, EINVAL);
7258 a = action_find(p, action_name);
7260 CHECK(!action_does_learning(a), EINVAL);
7262 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7263 if (action_data_size > action_data_size_max)
7264 action_data_size_max = action_data_size;
7267 CHECK_NAME(params->default_action_name, EINVAL);
7268 for (i = 0; i < p->n_actions; i++)
7269 if (!strcmp(params->action_names[i],
7270 params->default_action_name))
7272 CHECK(i < params->n_actions, EINVAL);
7273 default_action = action_find(p, params->default_action_name);
7274 CHECK((default_action->st && params->default_action_data) ||
7275 !params->default_action_data, EINVAL);
7277 /* Table type checks. */
7278 if (recommended_table_type_name)
7279 CHECK_NAME(recommended_table_type_name, EINVAL);
7281 if (params->n_fields) {
7282 enum rte_swx_table_match_type match_type;
7284 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7288 type = table_type_resolve(p, recommended_table_type_name, match_type);
7289 CHECK(type, EINVAL);
7294 /* Memory allocation. */
7295 t = calloc(1, sizeof(struct table));
7298 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7304 t->actions = calloc(params->n_actions, sizeof(struct action *));
7311 if (action_data_size_max) {
7312 t->default_action_data = calloc(1, action_data_size_max);
7313 if (!t->default_action_data) {
7321 /* Node initialization. */
7322 strcpy(t->name, name);
7323 if (args && args[0])
7324 strcpy(t->args, args);
7327 for (i = 0; i < params->n_fields; i++) {
7328 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7329 struct match_field *f = &t->fields[i];
7331 f->match_type = field->match_type;
7333 header_field_parse(p, field->name, NULL) :
7334 metadata_field_parse(p, field->name);
7336 t->n_fields = params->n_fields;
7339 for (i = 0; i < params->n_actions; i++)
7340 t->actions[i] = action_find(p, params->action_names[i]);
7341 t->default_action = default_action;
7342 if (default_action->st)
7343 memcpy(t->default_action_data,
7344 params->default_action_data,
7345 default_action->st->n_bits / 8);
7346 t->n_actions = params->n_actions;
7347 t->default_action_is_const = params->default_action_is_const;
7348 t->action_data_size_max = action_data_size_max;
7351 t->id = p->n_tables;
7353 /* Node add to tailq. */
7354 TAILQ_INSERT_TAIL(&p->tables, t, node);
7360 static struct rte_swx_table_params *
7361 table_params_get(struct table *table)
7363 struct rte_swx_table_params *params;
7364 struct field *first, *last;
7366 uint32_t key_size, key_offset, action_data_size, i;
7368 /* Memory allocation. */
7369 params = calloc(1, sizeof(struct rte_swx_table_params));
7373 /* Find first (smallest offset) and last (biggest offset) match fields. */
7374 first = table->fields[0].field;
7375 last = table->fields[0].field;
7377 for (i = 0; i < table->n_fields; i++) {
7378 struct field *f = table->fields[i].field;
7380 if (f->offset < first->offset)
7383 if (f->offset > last->offset)
7387 /* Key offset and size. */
7388 key_offset = first->offset / 8;
7389 key_size = (last->offset + last->n_bits - first->offset) / 8;
7391 /* Memory allocation. */
7392 key_mask = calloc(1, key_size);
7399 for (i = 0; i < table->n_fields; i++) {
7400 struct field *f = table->fields[i].field;
7401 uint32_t start = (f->offset - first->offset) / 8;
7402 size_t size = f->n_bits / 8;
7404 memset(&key_mask[start], 0xFF, size);
7407 /* Action data size. */
7408 action_data_size = 0;
7409 for (i = 0; i < table->n_actions; i++) {
7410 struct action *action = table->actions[i];
7411 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7413 if (ads > action_data_size)
7414 action_data_size = ads;
7418 params->match_type = table->type->match_type;
7419 params->key_size = key_size;
7420 params->key_offset = key_offset;
7421 params->key_mask0 = key_mask;
7422 params->action_data_size = action_data_size;
7423 params->n_keys_max = table->size;
7429 table_params_free(struct rte_swx_table_params *params)
7434 free(params->key_mask0);
7439 table_stub_lkp(void *table __rte_unused,
7440 void *mailbox __rte_unused,
7441 uint8_t **key __rte_unused,
7442 uint64_t *action_id __rte_unused,
7443 uint8_t **action_data __rte_unused,
7447 return 1; /* DONE. */
7451 table_build(struct rte_swx_pipeline *p)
7455 /* Per pipeline: table statistics. */
7456 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7457 CHECK(p->table_stats, ENOMEM);
7459 for (i = 0; i < p->n_tables; i++) {
7460 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7461 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7464 /* Per thread: table runt-time. */
7465 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7466 struct thread *t = &p->threads[i];
7467 struct table *table;
7469 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7470 CHECK(t->tables, ENOMEM);
7472 TAILQ_FOREACH(table, &p->tables, node) {
7473 struct table_runtime *r = &t->tables[table->id];
7478 size = table->type->ops.mailbox_size_get();
7481 r->func = table->type->ops.lkp;
7485 r->mailbox = calloc(1, size);
7486 CHECK(r->mailbox, ENOMEM);
7490 r->key = table->header ?
7491 &t->structs[table->header->struct_id] :
7492 &t->structs[p->metadata_struct_id];
7494 r->func = table_stub_lkp;
7503 table_build_free(struct rte_swx_pipeline *p)
7507 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7508 struct thread *t = &p->threads[i];
7514 for (j = 0; j < p->n_tables; j++) {
7515 struct table_runtime *r = &t->tables[j];
7524 if (p->table_stats) {
7525 for (i = 0; i < p->n_tables; i++)
7526 free(p->table_stats[i].n_pkts_action);
7528 free(p->table_stats);
7533 table_free(struct rte_swx_pipeline *p)
7535 table_build_free(p);
7541 elem = TAILQ_FIRST(&p->tables);
7545 TAILQ_REMOVE(&p->tables, elem, node);
7547 free(elem->actions);
7548 free(elem->default_action_data);
7554 struct table_type *elem;
7556 elem = TAILQ_FIRST(&p->table_types);
7560 TAILQ_REMOVE(&p->table_types, elem, node);
7568 static struct selector *
7569 selector_find(struct rte_swx_pipeline *p, const char *name)
7573 TAILQ_FOREACH(s, &p->selectors, node)
7574 if (strcmp(s->name, name) == 0)
7580 static struct selector *
7581 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7583 struct selector *s = NULL;
7585 TAILQ_FOREACH(s, &p->selectors, node)
7593 selector_fields_check(struct rte_swx_pipeline *p,
7594 struct rte_swx_pipeline_selector_params *params,
7595 struct header **header)
7597 struct header *h0 = NULL;
7598 struct field *hf, *mf;
7601 /* Return if no selector fields. */
7602 if (!params->n_selector_fields || !params->selector_field_names)
7605 /* Check that all the selector fields either belong to the same header
7606 * or are all meta-data fields.
7608 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7609 mf = metadata_field_parse(p, params->selector_field_names[0]);
7613 for (i = 1; i < params->n_selector_fields; i++)
7617 hf = header_field_parse(p, params->selector_field_names[i], &h);
7618 if (!hf || (h->id != h0->id))
7621 mf = metadata_field_parse(p, params->selector_field_names[i]);
7626 /* Check that there are no duplicated match fields. */
7627 for (i = 0; i < params->n_selector_fields; i++) {
7628 const char *field_name = params->selector_field_names[i];
7631 for (j = i + 1; j < params->n_selector_fields; j++)
7632 if (!strcmp(params->selector_field_names[j], field_name))
7644 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7646 struct rte_swx_pipeline_selector_params *params)
7649 struct header *selector_header = NULL;
7650 struct field *group_id_field, *member_id_field;
7656 CHECK_NAME(name, EINVAL);
7657 CHECK(!table_find(p, name), EEXIST);
7658 CHECK(!selector_find(p, name), EEXIST);
7659 CHECK(!learner_find(p, name), EEXIST);
7661 CHECK(params, EINVAL);
7663 CHECK_NAME(params->group_id_field_name, EINVAL);
7664 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7665 CHECK(group_id_field, EINVAL);
7667 for (i = 0; i < params->n_selector_fields; i++) {
7668 const char *field_name = params->selector_field_names[i];
7670 CHECK_NAME(field_name, EINVAL);
7672 status = selector_fields_check(p, params, &selector_header);
7676 CHECK_NAME(params->member_id_field_name, EINVAL);
7677 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7678 CHECK(member_id_field, EINVAL);
7680 CHECK(params->n_groups_max, EINVAL);
7682 CHECK(params->n_members_per_group_max, EINVAL);
7684 /* Memory allocation. */
7685 s = calloc(1, sizeof(struct selector));
7691 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7692 if (!s->selector_fields) {
7697 /* Node initialization. */
7698 strcpy(s->name, name);
7700 s->group_id_field = group_id_field;
7702 for (i = 0; i < params->n_selector_fields; i++) {
7703 const char *field_name = params->selector_field_names[i];
7705 s->selector_fields[i] = selector_header ?
7706 header_field_parse(p, field_name, NULL) :
7707 metadata_field_parse(p, field_name);
7710 s->n_selector_fields = params->n_selector_fields;
7712 s->selector_header = selector_header;
7714 s->member_id_field = member_id_field;
7716 s->n_groups_max = params->n_groups_max;
7718 s->n_members_per_group_max = params->n_members_per_group_max;
7720 s->id = p->n_selectors;
7722 /* Node add to tailq. */
7723 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7732 free(s->selector_fields);
7740 selector_params_free(struct rte_swx_table_selector_params *params)
7745 free(params->selector_mask);
7750 static struct rte_swx_table_selector_params *
7751 selector_table_params_get(struct selector *s)
7753 struct rte_swx_table_selector_params *params = NULL;
7754 struct field *first, *last;
7757 /* Memory allocation. */
7758 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7763 params->group_id_offset = s->group_id_field->offset / 8;
7765 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7766 first = s->selector_fields[0];
7767 last = s->selector_fields[0];
7769 for (i = 0; i < s->n_selector_fields; i++) {
7770 struct field *f = s->selector_fields[i];
7772 if (f->offset < first->offset)
7775 if (f->offset > last->offset)
7779 /* Selector offset and size. */
7780 params->selector_offset = first->offset / 8;
7781 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7783 /* Memory allocation. */
7784 params->selector_mask = calloc(1, params->selector_size);
7785 if (!params->selector_mask)
7788 /* Selector mask. */
7789 for (i = 0; i < s->n_selector_fields; i++) {
7790 struct field *f = s->selector_fields[i];
7791 uint32_t start = (f->offset - first->offset) / 8;
7792 size_t size = f->n_bits / 8;
7794 memset(¶ms->selector_mask[start], 0xFF, size);
7798 params->member_id_offset = s->member_id_field->offset / 8;
7800 /* Maximum number of groups. */
7801 params->n_groups_max = s->n_groups_max;
7803 /* Maximum number of members per group. */
7804 params->n_members_per_group_max = s->n_members_per_group_max;
7809 selector_params_free(params);
7814 selector_build_free(struct rte_swx_pipeline *p)
7818 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7819 struct thread *t = &p->threads[i];
7825 for (j = 0; j < p->n_selectors; j++) {
7826 struct selector_runtime *r = &t->selectors[j];
7832 t->selectors = NULL;
7835 free(p->selector_stats);
7836 p->selector_stats = NULL;
7840 selector_build(struct rte_swx_pipeline *p)
7845 /* Per pipeline: selector statistics. */
7846 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7847 if (!p->selector_stats) {
7852 /* Per thread: selector run-time. */
7853 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7854 struct thread *t = &p->threads[i];
7857 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7858 if (!t->selectors) {
7863 TAILQ_FOREACH(s, &p->selectors, node) {
7864 struct selector_runtime *r = &t->selectors[s->id];
7868 size = rte_swx_table_selector_mailbox_size_get();
7870 r->mailbox = calloc(1, size);
7877 /* r->group_id_buffer. */
7878 r->group_id_buffer = &t->structs[p->metadata_struct_id];
7880 /* r->selector_buffer. */
7881 r->selector_buffer = s->selector_header ?
7882 &t->structs[s->selector_header->struct_id] :
7883 &t->structs[p->metadata_struct_id];
7885 /* r->member_id_buffer. */
7886 r->member_id_buffer = &t->structs[p->metadata_struct_id];
7893 selector_build_free(p);
7898 selector_free(struct rte_swx_pipeline *p)
7900 selector_build_free(p);
7902 /* Selector tables. */
7904 struct selector *elem;
7906 elem = TAILQ_FIRST(&p->selectors);
7910 TAILQ_REMOVE(&p->selectors, elem, node);
7911 free(elem->selector_fields);
7919 static struct learner *
7920 learner_find(struct rte_swx_pipeline *p, const char *name)
7924 TAILQ_FOREACH(l, &p->learners, node)
7925 if (!strcmp(l->name, name))
7931 static struct learner *
7932 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7934 struct learner *l = NULL;
7936 TAILQ_FOREACH(l, &p->learners, node)
7944 learner_match_fields_check(struct rte_swx_pipeline *p,
7945 struct rte_swx_pipeline_learner_params *params,
7946 struct header **header)
7948 struct header *h0 = NULL;
7949 struct field *hf, *mf;
7952 /* Return if no match fields. */
7953 if (!params->n_fields || !params->field_names)
7956 /* Check that all the match fields either belong to the same header
7957 * or are all meta-data fields.
7959 hf = header_field_parse(p, params->field_names[0], &h0);
7960 mf = metadata_field_parse(p, params->field_names[0]);
7964 for (i = 1; i < params->n_fields; i++)
7968 hf = header_field_parse(p, params->field_names[i], &h);
7969 if (!hf || (h->id != h0->id))
7972 mf = metadata_field_parse(p, params->field_names[i]);
7977 /* Check that there are no duplicated match fields. */
7978 for (i = 0; i < params->n_fields; i++) {
7979 const char *field_name = params->field_names[i];
7982 for (j = i + 1; j < params->n_fields; j++)
7983 if (!strcmp(params->field_names[j], field_name))
7995 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
7997 struct struct_type *mst = p->metadata_st, *ast = a->st;
7998 struct field *mf, *af;
8008 /* Check that mf_name is the name of a valid meta-data field. */
8009 CHECK_NAME(mf_name, EINVAL);
8010 mf = metadata_field_parse(p, mf_name);
8013 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
8014 * all the action arguments.
8016 mf_pos = mf - mst->fields;
8017 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
8019 /* Check that the size of each of the identified meta-data fields matches exactly the size
8020 * of the corresponding action argument.
8022 for (i = 0; i < ast->n_fields; i++) {
8023 mf = &mst->fields[mf_pos + i];
8024 af = &ast->fields[i];
8026 CHECK(mf->n_bits == af->n_bits, EINVAL);
8033 learner_action_learning_check(struct rte_swx_pipeline *p,
8034 struct action *action,
8035 const char **action_names,
8040 /* For each "learn" instruction of the current action, check that the learned action (i.e.
8041 * the action passed as argument to the "learn" instruction) is also enabled for the
8042 * current learner table.
8044 for (i = 0; i < action->n_instructions; i++) {
8045 struct instruction *instr = &action->instructions[i];
8046 uint32_t found = 0, j;
8048 if (instr->type != INSTR_LEARNER_LEARN)
8051 for (j = 0; j < n_actions; j++) {
8054 a = action_find(p, action_names[j]);
8058 if (a->id == instr->learn.action_id)
8070 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
8072 struct rte_swx_pipeline_learner_params *params,
8076 struct learner *l = NULL;
8077 struct action *default_action;
8078 struct header *header = NULL;
8079 uint32_t action_data_size_max = 0, i;
8084 CHECK_NAME(name, EINVAL);
8085 CHECK(!table_find(p, name), EEXIST);
8086 CHECK(!selector_find(p, name), EEXIST);
8087 CHECK(!learner_find(p, name), EEXIST);
8089 CHECK(params, EINVAL);
8092 status = learner_match_fields_check(p, params, &header);
8096 /* Action checks. */
8097 CHECK(params->n_actions, EINVAL);
8099 CHECK(params->action_names, EINVAL);
8100 for (i = 0; i < params->n_actions; i++) {
8101 const char *action_name = params->action_names[i];
8102 const char *action_field_name = params->action_field_names[i];
8104 uint32_t action_data_size;
8106 CHECK_NAME(action_name, EINVAL);
8108 a = action_find(p, action_name);
8111 status = learner_action_args_check(p, a, action_field_name);
8115 status = learner_action_learning_check(p,
8117 params->action_names,
8122 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8123 if (action_data_size > action_data_size_max)
8124 action_data_size_max = action_data_size;
8127 CHECK_NAME(params->default_action_name, EINVAL);
8128 for (i = 0; i < p->n_actions; i++)
8129 if (!strcmp(params->action_names[i],
8130 params->default_action_name))
8132 CHECK(i < params->n_actions, EINVAL);
8134 default_action = action_find(p, params->default_action_name);
8135 CHECK((default_action->st && params->default_action_data) ||
8136 !params->default_action_data, EINVAL);
8138 /* Any other checks. */
8139 CHECK(size, EINVAL);
8140 CHECK(timeout, EINVAL);
8142 /* Memory allocation. */
8143 l = calloc(1, sizeof(struct learner));
8147 l->fields = calloc(params->n_fields, sizeof(struct field *));
8151 l->actions = calloc(params->n_actions, sizeof(struct action *));
8155 l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8159 if (action_data_size_max) {
8160 l->default_action_data = calloc(1, action_data_size_max);
8161 if (!l->default_action_data)
8165 /* Node initialization. */
8166 strcpy(l->name, name);
8168 for (i = 0; i < params->n_fields; i++) {
8169 const char *field_name = params->field_names[i];
8171 l->fields[i] = header ?
8172 header_field_parse(p, field_name, NULL) :
8173 metadata_field_parse(p, field_name);
8176 l->n_fields = params->n_fields;
8180 for (i = 0; i < params->n_actions; i++) {
8181 const char *mf_name = params->action_field_names[i];
8183 l->actions[i] = action_find(p, params->action_names[i]);
8185 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8188 l->default_action = default_action;
8190 if (default_action->st)
8191 memcpy(l->default_action_data,
8192 params->default_action_data,
8193 default_action->st->n_bits / 8);
8195 l->n_actions = params->n_actions;
8197 l->default_action_is_const = params->default_action_is_const;
8199 l->action_data_size_max = action_data_size_max;
8203 l->timeout = timeout;
8205 l->id = p->n_learners;
8207 /* Node add to tailq. */
8208 TAILQ_INSERT_TAIL(&p->learners, l, node);
8217 free(l->action_arg);
8226 learner_params_free(struct rte_swx_table_learner_params *params)
8231 free(params->key_mask0);
8236 static struct rte_swx_table_learner_params *
8237 learner_params_get(struct learner *l)
8239 struct rte_swx_table_learner_params *params = NULL;
8240 struct field *first, *last;
8243 /* Memory allocation. */
8244 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8248 /* Find first (smallest offset) and last (biggest offset) match fields. */
8249 first = l->fields[0];
8250 last = l->fields[0];
8252 for (i = 0; i < l->n_fields; i++) {
8253 struct field *f = l->fields[i];
8255 if (f->offset < first->offset)
8258 if (f->offset > last->offset)
8262 /* Key offset and size. */
8263 params->key_offset = first->offset / 8;
8264 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8266 /* Memory allocation. */
8267 params->key_mask0 = calloc(1, params->key_size);
8268 if (!params->key_mask0)
8272 for (i = 0; i < l->n_fields; i++) {
8273 struct field *f = l->fields[i];
8274 uint32_t start = (f->offset - first->offset) / 8;
8275 size_t size = f->n_bits / 8;
8277 memset(¶ms->key_mask0[start], 0xFF, size);
8280 /* Action data size. */
8281 params->action_data_size = l->action_data_size_max;
8283 /* Maximum number of keys. */
8284 params->n_keys_max = l->size;
8287 params->key_timeout = l->timeout;
8292 learner_params_free(params);
8297 learner_build_free(struct rte_swx_pipeline *p)
8301 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8302 struct thread *t = &p->threads[i];
8308 for (j = 0; j < p->n_learners; j++) {
8309 struct learner_runtime *r = &t->learners[j];
8312 free(r->action_data);
8319 if (p->learner_stats) {
8320 for (i = 0; i < p->n_learners; i++)
8321 free(p->learner_stats[i].n_pkts_action);
8323 free(p->learner_stats);
8328 learner_build(struct rte_swx_pipeline *p)
8333 /* Per pipeline: learner statistics. */
8334 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8335 CHECK(p->learner_stats, ENOMEM);
8337 for (i = 0; i < p->n_learners; i++) {
8338 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8339 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8342 /* Per thread: learner run-time. */
8343 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8344 struct thread *t = &p->threads[i];
8347 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8353 TAILQ_FOREACH(l, &p->learners, node) {
8354 struct learner_runtime *r = &t->learners[l->id];
8359 size = rte_swx_table_learner_mailbox_size_get();
8361 r->mailbox = calloc(1, size);
8369 r->key = l->header ?
8370 &t->structs[l->header->struct_id] :
8371 &t->structs[p->metadata_struct_id];
8373 /* r->action_data. */
8374 r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8375 if (!r->action_data) {
8380 for (j = 0; j < l->n_actions; j++) {
8381 struct action *a = l->actions[j];
8382 struct field *mf = l->action_arg[j];
8383 uint8_t *m = t->structs[p->metadata_struct_id];
8385 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8393 learner_build_free(p);
8398 learner_free(struct rte_swx_pipeline *p)
8400 learner_build_free(p);
8402 /* Learner tables. */
8406 l = TAILQ_FIRST(&p->learners);
8410 TAILQ_REMOVE(&p->learners, l, node);
8413 free(l->action_arg);
8414 free(l->default_action_data);
8423 table_state_build(struct rte_swx_pipeline *p)
8425 struct table *table;
8429 p->table_state = calloc(p->n_tables + p->n_selectors,
8430 sizeof(struct rte_swx_table_state));
8431 CHECK(p->table_state, ENOMEM);
8433 TAILQ_FOREACH(table, &p->tables, node) {
8434 struct rte_swx_table_state *ts = &p->table_state[table->id];
8437 struct rte_swx_table_params *params;
8440 params = table_params_get(table);
8441 CHECK(params, ENOMEM);
8443 ts->obj = table->type->ops.create(params,
8448 table_params_free(params);
8449 CHECK(ts->obj, ENODEV);
8452 /* ts->default_action_data. */
8453 if (table->action_data_size_max) {
8454 ts->default_action_data =
8455 malloc(table->action_data_size_max);
8456 CHECK(ts->default_action_data, ENOMEM);
8458 memcpy(ts->default_action_data,
8459 table->default_action_data,
8460 table->action_data_size_max);
8463 /* ts->default_action_id. */
8464 ts->default_action_id = table->default_action->id;
8467 TAILQ_FOREACH(s, &p->selectors, node) {
8468 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8469 struct rte_swx_table_selector_params *params;
8472 params = selector_table_params_get(s);
8473 CHECK(params, ENOMEM);
8475 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8477 selector_params_free(params);
8478 CHECK(ts->obj, ENODEV);
8481 TAILQ_FOREACH(l, &p->learners, node) {
8482 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8483 p->n_selectors + l->id];
8484 struct rte_swx_table_learner_params *params;
8487 params = learner_params_get(l);
8488 CHECK(params, ENOMEM);
8490 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8491 learner_params_free(params);
8492 CHECK(ts->obj, ENODEV);
8494 /* ts->default_action_data. */
8495 if (l->action_data_size_max) {
8496 ts->default_action_data = malloc(l->action_data_size_max);
8497 CHECK(ts->default_action_data, ENOMEM);
8499 memcpy(ts->default_action_data,
8500 l->default_action_data,
8501 l->action_data_size_max);
8504 /* ts->default_action_id. */
8505 ts->default_action_id = l->default_action->id;
8512 table_state_build_free(struct rte_swx_pipeline *p)
8516 if (!p->table_state)
8519 for (i = 0; i < p->n_tables; i++) {
8520 struct rte_swx_table_state *ts = &p->table_state[i];
8521 struct table *table = table_find_by_id(p, i);
8524 if (table->type && ts->obj)
8525 table->type->ops.free(ts->obj);
8527 /* ts->default_action_data. */
8528 free(ts->default_action_data);
8531 for (i = 0; i < p->n_selectors; i++) {
8532 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8536 rte_swx_table_selector_free(ts->obj);
8539 for (i = 0; i < p->n_learners; i++) {
8540 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8544 rte_swx_table_learner_free(ts->obj);
8546 /* ts->default_action_data. */
8547 free(ts->default_action_data);
8550 free(p->table_state);
8551 p->table_state = NULL;
8555 table_state_free(struct rte_swx_pipeline *p)
8557 table_state_build_free(p);
8563 static struct regarray *
8564 regarray_find(struct rte_swx_pipeline *p, const char *name)
8566 struct regarray *elem;
8568 TAILQ_FOREACH(elem, &p->regarrays, node)
8569 if (!strcmp(elem->name, name))
8575 static struct regarray *
8576 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8578 struct regarray *elem = NULL;
8580 TAILQ_FOREACH(elem, &p->regarrays, node)
8588 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8597 CHECK_NAME(name, EINVAL);
8598 CHECK(!regarray_find(p, name), EEXIST);
8600 CHECK(size, EINVAL);
8601 size = rte_align32pow2(size);
8603 /* Memory allocation. */
8604 r = calloc(1, sizeof(struct regarray));
8607 /* Node initialization. */
8608 strcpy(r->name, name);
8609 r->init_val = init_val;
8611 r->id = p->n_regarrays;
8613 /* Node add to tailq. */
8614 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8621 regarray_build(struct rte_swx_pipeline *p)
8623 struct regarray *regarray;
8625 if (!p->n_regarrays)
8628 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8629 CHECK(p->regarray_runtime, ENOMEM);
8631 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8632 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8635 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8636 RTE_CACHE_LINE_SIZE,
8638 CHECK(r->regarray, ENOMEM);
8640 if (regarray->init_val)
8641 for (i = 0; i < regarray->size; i++)
8642 r->regarray[i] = regarray->init_val;
8644 r->size_mask = regarray->size - 1;
8651 regarray_build_free(struct rte_swx_pipeline *p)
8655 if (!p->regarray_runtime)
8658 for (i = 0; i < p->n_regarrays; i++) {
8659 struct regarray *regarray = regarray_find_by_id(p, i);
8660 struct regarray_runtime *r = &p->regarray_runtime[i];
8662 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8665 free(p->regarray_runtime);
8666 p->regarray_runtime = NULL;
8670 regarray_free(struct rte_swx_pipeline *p)
8672 regarray_build_free(p);
8675 struct regarray *elem;
8677 elem = TAILQ_FIRST(&p->regarrays);
8681 TAILQ_REMOVE(&p->regarrays, elem, node);
8689 static struct meter_profile *
8690 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8692 struct meter_profile *elem;
8694 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8695 if (!strcmp(elem->name, name))
8701 static struct metarray *
8702 metarray_find(struct rte_swx_pipeline *p, const char *name)
8704 struct metarray *elem;
8706 TAILQ_FOREACH(elem, &p->metarrays, node)
8707 if (!strcmp(elem->name, name))
8713 static struct metarray *
8714 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8716 struct metarray *elem = NULL;
8718 TAILQ_FOREACH(elem, &p->metarrays, node)
8726 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8734 CHECK_NAME(name, EINVAL);
8735 CHECK(!metarray_find(p, name), EEXIST);
8737 CHECK(size, EINVAL);
8738 size = rte_align32pow2(size);
8740 /* Memory allocation. */
8741 m = calloc(1, sizeof(struct metarray));
8744 /* Node initialization. */
8745 strcpy(m->name, name);
8747 m->id = p->n_metarrays;
8749 /* Node add to tailq. */
8750 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8756 struct meter_profile meter_profile_default = {
8765 .cir_bytes_per_period = 1,
8767 .pir_bytes_per_period = 1,
8774 meter_init(struct meter *m)
8776 memset(m, 0, sizeof(struct meter));
8777 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8778 m->profile = &meter_profile_default;
8779 m->color_mask = RTE_COLOR_GREEN;
8781 meter_profile_default.n_users++;
8785 metarray_build(struct rte_swx_pipeline *p)
8789 if (!p->n_metarrays)
8792 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8793 CHECK(p->metarray_runtime, ENOMEM);
8795 TAILQ_FOREACH(m, &p->metarrays, node) {
8796 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8799 r->metarray = env_malloc(m->size * sizeof(struct meter),
8800 RTE_CACHE_LINE_SIZE,
8802 CHECK(r->metarray, ENOMEM);
8804 for (i = 0; i < m->size; i++)
8805 meter_init(&r->metarray[i]);
8807 r->size_mask = m->size - 1;
8814 metarray_build_free(struct rte_swx_pipeline *p)
8818 if (!p->metarray_runtime)
8821 for (i = 0; i < p->n_metarrays; i++) {
8822 struct metarray *m = metarray_find_by_id(p, i);
8823 struct metarray_runtime *r = &p->metarray_runtime[i];
8825 env_free(r->metarray, m->size * sizeof(struct meter));
8828 free(p->metarray_runtime);
8829 p->metarray_runtime = NULL;
8833 metarray_free(struct rte_swx_pipeline *p)
8835 metarray_build_free(p);
8839 struct metarray *elem;
8841 elem = TAILQ_FIRST(&p->metarrays);
8845 TAILQ_REMOVE(&p->metarrays, elem, node);
8849 /* Meter profiles. */
8851 struct meter_profile *elem;
8853 elem = TAILQ_FIRST(&p->meter_profiles);
8857 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8866 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8868 struct rte_swx_pipeline *pipeline;
8870 /* Check input parameters. */
8873 /* Memory allocation. */
8874 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8875 CHECK(pipeline, ENOMEM);
8877 /* Initialization. */
8878 TAILQ_INIT(&pipeline->struct_types);
8879 TAILQ_INIT(&pipeline->port_in_types);
8880 TAILQ_INIT(&pipeline->ports_in);
8881 TAILQ_INIT(&pipeline->port_out_types);
8882 TAILQ_INIT(&pipeline->ports_out);
8883 TAILQ_INIT(&pipeline->extern_types);
8884 TAILQ_INIT(&pipeline->extern_objs);
8885 TAILQ_INIT(&pipeline->extern_funcs);
8886 TAILQ_INIT(&pipeline->headers);
8887 TAILQ_INIT(&pipeline->actions);
8888 TAILQ_INIT(&pipeline->table_types);
8889 TAILQ_INIT(&pipeline->tables);
8890 TAILQ_INIT(&pipeline->selectors);
8891 TAILQ_INIT(&pipeline->learners);
8892 TAILQ_INIT(&pipeline->regarrays);
8893 TAILQ_INIT(&pipeline->meter_profiles);
8894 TAILQ_INIT(&pipeline->metarrays);
8896 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8897 pipeline->numa_node = numa_node;
8904 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8909 free(p->instruction_data);
8910 free(p->instructions);
8914 table_state_free(p);
8921 extern_func_free(p);
8931 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8932 const char **instructions,
8933 uint32_t n_instructions)
8938 err = instruction_config(p, NULL, instructions, n_instructions);
8942 /* Thread instruction pointer reset. */
8943 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8944 struct thread *t = &p->threads[i];
8946 thread_ip_reset(p, t);
8953 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
8958 CHECK(p->build_done == 0, EEXIST);
8960 status = port_in_build(p);
8964 status = port_out_build(p);
8968 status = struct_build(p);
8972 status = extern_obj_build(p);
8976 status = extern_func_build(p);
8980 status = header_build(p);
8984 status = metadata_build(p);
8988 status = action_build(p);
8992 status = table_build(p);
8996 status = selector_build(p);
9000 status = learner_build(p);
9004 status = table_state_build(p);
9008 status = regarray_build(p);
9012 status = metarray_build(p);
9020 metarray_build_free(p);
9021 regarray_build_free(p);
9022 table_state_build_free(p);
9023 learner_build_free(p);
9024 selector_build_free(p);
9025 table_build_free(p);
9026 action_build_free(p);
9027 metadata_build_free(p);
9028 header_build_free(p);
9029 extern_func_build_free(p);
9030 extern_obj_build_free(p);
9031 port_out_build_free(p);
9032 port_in_build_free(p);
9033 struct_build_free(p);
9039 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9043 for (i = 0; i < n_instructions; i++)
9048 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9052 for (i = 0; i < p->n_ports_out; i++) {
9053 struct port_out_runtime *port = &p->out[i];
9056 port->flush(port->obj);
9064 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9065 struct rte_swx_ctl_pipeline_info *pipeline)
9067 struct action *action;
9068 struct table *table;
9069 uint32_t n_actions = 0, n_tables = 0;
9071 if (!p || !pipeline)
9074 TAILQ_FOREACH(action, &p->actions, node)
9077 TAILQ_FOREACH(table, &p->tables, node)
9080 pipeline->n_ports_in = p->n_ports_in;
9081 pipeline->n_ports_out = p->n_ports_out;
9082 pipeline->n_actions = n_actions;
9083 pipeline->n_tables = n_tables;
9084 pipeline->n_selectors = p->n_selectors;
9085 pipeline->n_learners = p->n_learners;
9086 pipeline->n_regarrays = p->n_regarrays;
9087 pipeline->n_metarrays = p->n_metarrays;
9093 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9095 if (!p || !numa_node)
9098 *numa_node = p->numa_node;
9103 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9105 struct rte_swx_ctl_action_info *action)
9107 struct action *a = NULL;
9109 if (!p || (action_id >= p->n_actions) || !action)
9112 a = action_find_by_id(p, action_id);
9116 strcpy(action->name, a->name);
9117 action->n_args = a->st ? a->st->n_fields : 0;
9122 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9124 uint32_t action_arg_id,
9125 struct rte_swx_ctl_action_arg_info *action_arg)
9127 struct action *a = NULL;
9128 struct field *arg = NULL;
9130 if (!p || (action_id >= p->n_actions) || !action_arg)
9133 a = action_find_by_id(p, action_id);
9134 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9137 arg = &a->st->fields[action_arg_id];
9138 strcpy(action_arg->name, arg->name);
9139 action_arg->n_bits = arg->n_bits;
9140 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9146 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9148 struct rte_swx_ctl_table_info *table)
9150 struct table *t = NULL;
9155 t = table_find_by_id(p, table_id);
9159 strcpy(table->name, t->name);
9160 strcpy(table->args, t->args);
9161 table->n_match_fields = t->n_fields;
9162 table->n_actions = t->n_actions;
9163 table->default_action_is_const = t->default_action_is_const;
9164 table->size = t->size;
9169 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9171 uint32_t match_field_id,
9172 struct rte_swx_ctl_table_match_field_info *match_field)
9175 struct match_field *f;
9177 if (!p || (table_id >= p->n_tables) || !match_field)
9180 t = table_find_by_id(p, table_id);
9181 if (!t || (match_field_id >= t->n_fields))
9184 f = &t->fields[match_field_id];
9185 match_field->match_type = f->match_type;
9186 match_field->is_header = t->header ? 1 : 0;
9187 match_field->n_bits = f->field->n_bits;
9188 match_field->offset = f->field->offset;
9194 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9196 uint32_t table_action_id,
9197 struct rte_swx_ctl_table_action_info *table_action)
9201 if (!p || (table_id >= p->n_tables) || !table_action)
9204 t = table_find_by_id(p, table_id);
9205 if (!t || (table_action_id >= t->n_actions))
9208 table_action->action_id = t->actions[table_action_id]->id;
9214 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9216 struct rte_swx_table_ops *table_ops,
9221 if (!p || (table_id >= p->n_tables))
9224 t = table_find_by_id(p, table_id);
9230 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9240 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9241 uint32_t selector_id,
9242 struct rte_swx_ctl_selector_info *selector)
9244 struct selector *s = NULL;
9246 if (!p || !selector)
9249 s = selector_find_by_id(p, selector_id);
9253 strcpy(selector->name, s->name);
9255 selector->n_selector_fields = s->n_selector_fields;
9256 selector->n_groups_max = s->n_groups_max;
9257 selector->n_members_per_group_max = s->n_members_per_group_max;
9263 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9264 uint32_t selector_id,
9265 struct rte_swx_ctl_table_match_field_info *field)
9269 if (!p || (selector_id >= p->n_selectors) || !field)
9272 s = selector_find_by_id(p, selector_id);
9276 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9277 field->is_header = 0;
9278 field->n_bits = s->group_id_field->n_bits;
9279 field->offset = s->group_id_field->offset;
9285 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9286 uint32_t selector_id,
9287 uint32_t selector_field_id,
9288 struct rte_swx_ctl_table_match_field_info *field)
9293 if (!p || (selector_id >= p->n_selectors) || !field)
9296 s = selector_find_by_id(p, selector_id);
9297 if (!s || (selector_field_id >= s->n_selector_fields))
9300 f = s->selector_fields[selector_field_id];
9301 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9302 field->is_header = s->selector_header ? 1 : 0;
9303 field->n_bits = f->n_bits;
9304 field->offset = f->offset;
9310 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9311 uint32_t selector_id,
9312 struct rte_swx_ctl_table_match_field_info *field)
9316 if (!p || (selector_id >= p->n_selectors) || !field)
9319 s = selector_find_by_id(p, selector_id);
9323 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9324 field->is_header = 0;
9325 field->n_bits = s->member_id_field->n_bits;
9326 field->offset = s->member_id_field->offset;
9332 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9333 uint32_t learner_id,
9334 struct rte_swx_ctl_learner_info *learner)
9336 struct learner *l = NULL;
9341 l = learner_find_by_id(p, learner_id);
9345 strcpy(learner->name, l->name);
9347 learner->n_match_fields = l->n_fields;
9348 learner->n_actions = l->n_actions;
9349 learner->default_action_is_const = l->default_action_is_const;
9350 learner->size = l->size;
9356 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9357 uint32_t learner_id,
9358 uint32_t match_field_id,
9359 struct rte_swx_ctl_table_match_field_info *match_field)
9364 if (!p || (learner_id >= p->n_learners) || !match_field)
9367 l = learner_find_by_id(p, learner_id);
9368 if (!l || (match_field_id >= l->n_fields))
9371 f = l->fields[match_field_id];
9372 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9373 match_field->is_header = l->header ? 1 : 0;
9374 match_field->n_bits = f->n_bits;
9375 match_field->offset = f->offset;
9381 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9382 uint32_t learner_id,
9383 uint32_t learner_action_id,
9384 struct rte_swx_ctl_table_action_info *learner_action)
9388 if (!p || (learner_id >= p->n_learners) || !learner_action)
9391 l = learner_find_by_id(p, learner_id);
9392 if (!l || (learner_action_id >= l->n_actions))
9395 learner_action->action_id = l->actions[learner_action_id]->id;
9401 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9402 struct rte_swx_table_state **table_state)
9404 if (!p || !table_state || !p->build_done)
9407 *table_state = p->table_state;
9412 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9413 struct rte_swx_table_state *table_state)
9415 if (!p || !table_state || !p->build_done)
9418 p->table_state = table_state;
9423 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9425 struct rte_swx_port_in_stats *stats)
9427 struct port_in *port;
9432 port = port_in_find(p, port_id);
9436 port->type->ops.stats_read(port->obj, stats);
9441 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9443 struct rte_swx_port_out_stats *stats)
9445 struct port_out *port;
9450 port = port_out_find(p, port_id);
9454 port->type->ops.stats_read(port->obj, stats);
9459 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9460 const char *table_name,
9461 struct rte_swx_table_stats *stats)
9463 struct table *table;
9464 struct table_statistics *table_stats;
9466 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9469 table = table_find(p, table_name);
9473 table_stats = &p->table_stats[table->id];
9475 memcpy(stats->n_pkts_action,
9476 table_stats->n_pkts_action,
9477 p->n_actions * sizeof(uint64_t));
9479 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9480 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9486 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9487 const char *selector_name,
9488 struct rte_swx_pipeline_selector_stats *stats)
9492 if (!p || !selector_name || !selector_name[0] || !stats)
9495 s = selector_find(p, selector_name);
9499 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9505 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9506 const char *learner_name,
9507 struct rte_swx_learner_stats *stats)
9510 struct learner_statistics *learner_stats;
9512 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9515 l = learner_find(p, learner_name);
9519 learner_stats = &p->learner_stats[l->id];
9521 memcpy(stats->n_pkts_action,
9522 learner_stats->n_pkts_action,
9523 p->n_actions * sizeof(uint64_t));
9525 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9526 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9528 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9529 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9531 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9537 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9538 uint32_t regarray_id,
9539 struct rte_swx_ctl_regarray_info *regarray)
9543 if (!p || !regarray)
9546 r = regarray_find_by_id(p, regarray_id);
9550 strcpy(regarray->name, r->name);
9551 regarray->size = r->size;
9556 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9557 const char *regarray_name,
9558 uint32_t regarray_index,
9561 struct regarray *regarray;
9562 struct regarray_runtime *r;
9564 if (!p || !regarray_name || !value)
9567 regarray = regarray_find(p, regarray_name);
9568 if (!regarray || (regarray_index >= regarray->size))
9571 r = &p->regarray_runtime[regarray->id];
9572 *value = r->regarray[regarray_index];
9577 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9578 const char *regarray_name,
9579 uint32_t regarray_index,
9582 struct regarray *regarray;
9583 struct regarray_runtime *r;
9585 if (!p || !regarray_name)
9588 regarray = regarray_find(p, regarray_name);
9589 if (!regarray || (regarray_index >= regarray->size))
9592 r = &p->regarray_runtime[regarray->id];
9593 r->regarray[regarray_index] = value;
9598 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9599 uint32_t metarray_id,
9600 struct rte_swx_ctl_metarray_info *metarray)
9604 if (!p || !metarray)
9607 m = metarray_find_by_id(p, metarray_id);
9611 strcpy(metarray->name, m->name);
9612 metarray->size = m->size;
9617 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9619 struct rte_meter_trtcm_params *params)
9621 struct meter_profile *mp;
9625 CHECK_NAME(name, EINVAL);
9626 CHECK(params, EINVAL);
9627 CHECK(!meter_profile_find(p, name), EEXIST);
9629 /* Node allocation. */
9630 mp = calloc(1, sizeof(struct meter_profile));
9633 /* Node initialization. */
9634 strcpy(mp->name, name);
9635 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9636 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9642 /* Node add to tailq. */
9643 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9649 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9652 struct meter_profile *mp;
9655 CHECK_NAME(name, EINVAL);
9657 mp = meter_profile_find(p, name);
9659 CHECK(!mp->n_users, EBUSY);
9661 /* Remove node from tailq. */
9662 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9669 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9670 const char *metarray_name,
9671 uint32_t metarray_index)
9673 struct meter_profile *mp_old;
9674 struct metarray *metarray;
9675 struct metarray_runtime *metarray_runtime;
9679 CHECK_NAME(metarray_name, EINVAL);
9681 metarray = metarray_find(p, metarray_name);
9682 CHECK(metarray, EINVAL);
9683 CHECK(metarray_index < metarray->size, EINVAL);
9685 metarray_runtime = &p->metarray_runtime[metarray->id];
9686 m = &metarray_runtime->metarray[metarray_index];
9687 mp_old = m->profile;
9697 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9698 const char *metarray_name,
9699 uint32_t metarray_index,
9700 const char *profile_name)
9702 struct meter_profile *mp, *mp_old;
9703 struct metarray *metarray;
9704 struct metarray_runtime *metarray_runtime;
9708 CHECK_NAME(metarray_name, EINVAL);
9710 metarray = metarray_find(p, metarray_name);
9711 CHECK(metarray, EINVAL);
9712 CHECK(metarray_index < metarray->size, EINVAL);
9714 mp = meter_profile_find(p, profile_name);
9717 metarray_runtime = &p->metarray_runtime[metarray->id];
9718 m = &metarray_runtime->metarray[metarray_index];
9719 mp_old = m->profile;
9721 memset(m, 0, sizeof(struct meter));
9722 rte_meter_trtcm_config(&m->m, &mp->profile);
9724 m->color_mask = RTE_COLORS;
9733 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9734 const char *metarray_name,
9735 uint32_t metarray_index,
9736 struct rte_swx_ctl_meter_stats *stats)
9738 struct metarray *metarray;
9739 struct metarray_runtime *metarray_runtime;
9743 CHECK_NAME(metarray_name, EINVAL);
9745 metarray = metarray_find(p, metarray_name);
9746 CHECK(metarray, EINVAL);
9747 CHECK(metarray_index < metarray->size, EINVAL);
9749 CHECK(stats, EINVAL);
9751 metarray_runtime = &p->metarray_runtime[metarray->id];
9752 m = &metarray_runtime->metarray[metarray_index];
9754 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9755 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));