1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
9 #include "rte_swx_pipeline_internal.h"
11 #define CHECK(condition, err_code) \
17 #define CHECK_NAME(name, err_code) \
20 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
23 #define CHECK_INSTRUCTION(instr, err_code) \
26 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
27 RTE_SWX_INSTRUCTION_SIZE), \
33 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
35 #include <rte_malloc.h>
38 env_malloc(size_t size, size_t alignment, int numa_node)
40 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
44 env_free(void *start, size_t size __rte_unused)
54 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
58 if (numa_available() == -1)
61 start = numa_alloc_onnode(size, numa_node);
65 memset(start, 0, size);
70 env_free(void *start, size_t size)
72 if (numa_available() == -1)
75 numa_free(start, size);
83 static struct struct_type *
84 struct_type_find(struct rte_swx_pipeline *p, const char *name)
86 struct struct_type *elem;
88 TAILQ_FOREACH(elem, &p->struct_types, node)
89 if (strcmp(elem->name, name) == 0)
96 struct_type_field_find(struct struct_type *st, const char *name)
100 for (i = 0; i < st->n_fields; i++) {
101 struct field *f = &st->fields[i];
103 if (strcmp(f->name, name) == 0)
111 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
113 struct rte_swx_field_params *fields,
115 int last_field_has_variable_size)
117 struct struct_type *st;
121 CHECK_NAME(name, EINVAL);
122 CHECK(fields, EINVAL);
123 CHECK(n_fields, EINVAL);
125 for (i = 0; i < n_fields; i++) {
126 struct rte_swx_field_params *f = &fields[i];
127 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
130 CHECK_NAME(f->name, EINVAL);
131 CHECK(f->n_bits, EINVAL);
132 CHECK((f->n_bits <= 64) || var_size, EINVAL);
133 CHECK((f->n_bits & 7) == 0, EINVAL);
135 for (j = 0; j < i; j++) {
136 struct rte_swx_field_params *f_prev = &fields[j];
138 CHECK(strcmp(f->name, f_prev->name), EINVAL);
142 CHECK(!struct_type_find(p, name), EEXIST);
144 /* Node allocation. */
145 st = calloc(1, sizeof(struct struct_type));
148 st->fields = calloc(n_fields, sizeof(struct field));
154 /* Node initialization. */
155 strcpy(st->name, name);
156 for (i = 0; i < n_fields; i++) {
157 struct field *dst = &st->fields[i];
158 struct rte_swx_field_params *src = &fields[i];
159 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
161 strcpy(dst->name, src->name);
162 dst->n_bits = src->n_bits;
163 dst->offset = st->n_bits;
164 dst->var_size = var_size;
166 st->n_bits += src->n_bits;
167 st->n_bits_min += var_size ? 0 : src->n_bits;
169 st->n_fields = n_fields;
170 st->var_size = last_field_has_variable_size;
172 /* Node add to tailq. */
173 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
179 struct_build(struct rte_swx_pipeline *p)
183 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
184 struct thread *t = &p->threads[i];
186 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
187 CHECK(t->structs, ENOMEM);
194 struct_build_free(struct rte_swx_pipeline *p)
198 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
199 struct thread *t = &p->threads[i];
207 struct_free(struct rte_swx_pipeline *p)
209 struct_build_free(p);
213 struct struct_type *elem;
215 elem = TAILQ_FIRST(&p->struct_types);
219 TAILQ_REMOVE(&p->struct_types, elem, node);
228 static struct port_in_type *
229 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
231 struct port_in_type *elem;
236 TAILQ_FOREACH(elem, &p->port_in_types, node)
237 if (strcmp(elem->name, name) == 0)
244 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
246 struct rte_swx_port_in_ops *ops)
248 struct port_in_type *elem;
251 CHECK_NAME(name, EINVAL);
253 CHECK(ops->create, EINVAL);
254 CHECK(ops->free, EINVAL);
255 CHECK(ops->pkt_rx, EINVAL);
256 CHECK(ops->stats_read, EINVAL);
258 CHECK(!port_in_type_find(p, name), EEXIST);
260 /* Node allocation. */
261 elem = calloc(1, sizeof(struct port_in_type));
264 /* Node initialization. */
265 strcpy(elem->name, name);
266 memcpy(&elem->ops, ops, sizeof(*ops));
268 /* Node add to tailq. */
269 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
274 static struct port_in *
275 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
277 struct port_in *port;
279 TAILQ_FOREACH(port, &p->ports_in, node)
280 if (port->id == port_id)
287 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
289 const char *port_type_name,
292 struct port_in_type *type = NULL;
293 struct port_in *port = NULL;
298 CHECK(!port_in_find(p, port_id), EINVAL);
300 CHECK_NAME(port_type_name, EINVAL);
301 type = port_in_type_find(p, port_type_name);
304 obj = type->ops.create(args);
307 /* Node allocation. */
308 port = calloc(1, sizeof(struct port_in));
311 /* Node initialization. */
316 /* Node add to tailq. */
317 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
318 if (p->n_ports_in < port_id + 1)
319 p->n_ports_in = port_id + 1;
325 port_in_build(struct rte_swx_pipeline *p)
327 struct port_in *port;
330 CHECK(p->n_ports_in, EINVAL);
331 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
333 for (i = 0; i < p->n_ports_in; i++)
334 CHECK(port_in_find(p, i), EINVAL);
336 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
337 CHECK(p->in, ENOMEM);
339 TAILQ_FOREACH(port, &p->ports_in, node) {
340 struct port_in_runtime *in = &p->in[port->id];
342 in->pkt_rx = port->type->ops.pkt_rx;
350 port_in_build_free(struct rte_swx_pipeline *p)
357 port_in_free(struct rte_swx_pipeline *p)
359 port_in_build_free(p);
363 struct port_in *port;
365 port = TAILQ_FIRST(&p->ports_in);
369 TAILQ_REMOVE(&p->ports_in, port, node);
370 port->type->ops.free(port->obj);
374 /* Input port types. */
376 struct port_in_type *elem;
378 elem = TAILQ_FIRST(&p->port_in_types);
382 TAILQ_REMOVE(&p->port_in_types, elem, node);
390 static struct port_out_type *
391 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
393 struct port_out_type *elem;
398 TAILQ_FOREACH(elem, &p->port_out_types, node)
399 if (!strcmp(elem->name, name))
406 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
408 struct rte_swx_port_out_ops *ops)
410 struct port_out_type *elem;
413 CHECK_NAME(name, EINVAL);
415 CHECK(ops->create, EINVAL);
416 CHECK(ops->free, EINVAL);
417 CHECK(ops->pkt_tx, EINVAL);
418 CHECK(ops->stats_read, EINVAL);
420 CHECK(!port_out_type_find(p, name), EEXIST);
422 /* Node allocation. */
423 elem = calloc(1, sizeof(struct port_out_type));
426 /* Node initialization. */
427 strcpy(elem->name, name);
428 memcpy(&elem->ops, ops, sizeof(*ops));
430 /* Node add to tailq. */
431 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
436 static struct port_out *
437 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
439 struct port_out *port;
441 TAILQ_FOREACH(port, &p->ports_out, node)
442 if (port->id == port_id)
449 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
451 const char *port_type_name,
454 struct port_out_type *type = NULL;
455 struct port_out *port = NULL;
460 CHECK(!port_out_find(p, port_id), EINVAL);
462 CHECK_NAME(port_type_name, EINVAL);
463 type = port_out_type_find(p, port_type_name);
466 obj = type->ops.create(args);
469 /* Node allocation. */
470 port = calloc(1, sizeof(struct port_out));
473 /* Node initialization. */
478 /* Node add to tailq. */
479 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
480 if (p->n_ports_out < port_id + 1)
481 p->n_ports_out = port_id + 1;
487 port_out_build(struct rte_swx_pipeline *p)
489 struct port_out *port;
492 CHECK(p->n_ports_out, EINVAL);
494 for (i = 0; i < p->n_ports_out; i++)
495 CHECK(port_out_find(p, i), EINVAL);
497 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
498 CHECK(p->out, ENOMEM);
500 TAILQ_FOREACH(port, &p->ports_out, node) {
501 struct port_out_runtime *out = &p->out[port->id];
503 out->pkt_tx = port->type->ops.pkt_tx;
504 out->flush = port->type->ops.flush;
505 out->obj = port->obj;
512 port_out_build_free(struct rte_swx_pipeline *p)
519 port_out_free(struct rte_swx_pipeline *p)
521 port_out_build_free(p);
525 struct port_out *port;
527 port = TAILQ_FIRST(&p->ports_out);
531 TAILQ_REMOVE(&p->ports_out, port, node);
532 port->type->ops.free(port->obj);
536 /* Output port types. */
538 struct port_out_type *elem;
540 elem = TAILQ_FIRST(&p->port_out_types);
544 TAILQ_REMOVE(&p->port_out_types, elem, node);
552 static struct extern_type *
553 extern_type_find(struct rte_swx_pipeline *p, const char *name)
555 struct extern_type *elem;
557 TAILQ_FOREACH(elem, &p->extern_types, node)
558 if (strcmp(elem->name, name) == 0)
564 static struct extern_type_member_func *
565 extern_type_member_func_find(struct extern_type *type, const char *name)
567 struct extern_type_member_func *elem;
569 TAILQ_FOREACH(elem, &type->funcs, node)
570 if (strcmp(elem->name, name) == 0)
576 static struct extern_obj *
577 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
579 struct extern_obj *elem;
581 TAILQ_FOREACH(elem, &p->extern_objs, node)
582 if (strcmp(elem->name, name) == 0)
588 static struct extern_type_member_func *
589 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
591 struct extern_obj **obj)
593 struct extern_obj *object;
594 struct extern_type_member_func *func;
595 char *object_name, *func_name;
597 if (name[0] != 'e' || name[1] != '.')
600 object_name = strdup(&name[2]);
604 func_name = strchr(object_name, '.');
613 object = extern_obj_find(p, object_name);
619 func = extern_type_member_func_find(object->type, func_name);
632 static struct field *
633 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
635 struct extern_obj **object)
637 struct extern_obj *obj;
639 char *obj_name, *field_name;
641 if ((name[0] != 'e') || (name[1] != '.'))
644 obj_name = strdup(&name[2]);
648 field_name = strchr(obj_name, '.');
657 obj = extern_obj_find(p, obj_name);
663 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
677 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
679 const char *mailbox_struct_type_name,
680 rte_swx_extern_type_constructor_t constructor,
681 rte_swx_extern_type_destructor_t destructor)
683 struct extern_type *elem;
684 struct struct_type *mailbox_struct_type;
688 CHECK_NAME(name, EINVAL);
689 CHECK(!extern_type_find(p, name), EEXIST);
691 CHECK_NAME(mailbox_struct_type_name, EINVAL);
692 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
693 CHECK(mailbox_struct_type, EINVAL);
694 CHECK(!mailbox_struct_type->var_size, EINVAL);
696 CHECK(constructor, EINVAL);
697 CHECK(destructor, EINVAL);
699 /* Node allocation. */
700 elem = calloc(1, sizeof(struct extern_type));
703 /* Node initialization. */
704 strcpy(elem->name, name);
705 elem->mailbox_struct_type = mailbox_struct_type;
706 elem->constructor = constructor;
707 elem->destructor = destructor;
708 TAILQ_INIT(&elem->funcs);
710 /* Node add to tailq. */
711 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
717 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
718 const char *extern_type_name,
720 rte_swx_extern_type_member_func_t member_func)
722 struct extern_type *type;
723 struct extern_type_member_func *type_member;
727 CHECK_NAME(extern_type_name, EINVAL);
728 type = extern_type_find(p, extern_type_name);
730 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
732 CHECK_NAME(name, EINVAL);
733 CHECK(!extern_type_member_func_find(type, name), EEXIST);
735 CHECK(member_func, EINVAL);
737 /* Node allocation. */
738 type_member = calloc(1, sizeof(struct extern_type_member_func));
739 CHECK(type_member, ENOMEM);
741 /* Node initialization. */
742 strcpy(type_member->name, name);
743 type_member->func = member_func;
744 type_member->id = type->n_funcs;
746 /* Node add to tailq. */
747 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
754 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
755 const char *extern_type_name,
759 struct extern_type *type;
760 struct extern_obj *obj;
765 CHECK_NAME(extern_type_name, EINVAL);
766 type = extern_type_find(p, extern_type_name);
769 CHECK_NAME(name, EINVAL);
770 CHECK(!extern_obj_find(p, name), EEXIST);
772 /* Node allocation. */
773 obj = calloc(1, sizeof(struct extern_obj));
776 /* Object construction. */
777 obj_handle = type->constructor(args);
783 /* Node initialization. */
784 strcpy(obj->name, name);
786 obj->obj = obj_handle;
787 obj->struct_id = p->n_structs;
788 obj->id = p->n_extern_objs;
790 /* Node add to tailq. */
791 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
799 extern_obj_build(struct rte_swx_pipeline *p)
803 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
804 struct thread *t = &p->threads[i];
805 struct extern_obj *obj;
807 t->extern_objs = calloc(p->n_extern_objs,
808 sizeof(struct extern_obj_runtime));
809 CHECK(t->extern_objs, ENOMEM);
811 TAILQ_FOREACH(obj, &p->extern_objs, node) {
812 struct extern_obj_runtime *r =
813 &t->extern_objs[obj->id];
814 struct extern_type_member_func *func;
815 uint32_t mailbox_size =
816 obj->type->mailbox_struct_type->n_bits / 8;
820 r->mailbox = calloc(1, mailbox_size);
821 CHECK(r->mailbox, ENOMEM);
823 TAILQ_FOREACH(func, &obj->type->funcs, node)
824 r->funcs[func->id] = func->func;
826 t->structs[obj->struct_id] = r->mailbox;
834 extern_obj_build_free(struct rte_swx_pipeline *p)
838 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
839 struct thread *t = &p->threads[i];
845 for (j = 0; j < p->n_extern_objs; j++) {
846 struct extern_obj_runtime *r = &t->extern_objs[j];
851 free(t->extern_objs);
852 t->extern_objs = NULL;
857 extern_obj_free(struct rte_swx_pipeline *p)
859 extern_obj_build_free(p);
861 /* Extern objects. */
863 struct extern_obj *elem;
865 elem = TAILQ_FIRST(&p->extern_objs);
869 TAILQ_REMOVE(&p->extern_objs, elem, node);
871 elem->type->destructor(elem->obj);
877 struct extern_type *elem;
879 elem = TAILQ_FIRST(&p->extern_types);
883 TAILQ_REMOVE(&p->extern_types, elem, node);
886 struct extern_type_member_func *func;
888 func = TAILQ_FIRST(&elem->funcs);
892 TAILQ_REMOVE(&elem->funcs, func, node);
903 static struct extern_func *
904 extern_func_find(struct rte_swx_pipeline *p, const char *name)
906 struct extern_func *elem;
908 TAILQ_FOREACH(elem, &p->extern_funcs, node)
909 if (strcmp(elem->name, name) == 0)
915 static struct extern_func *
916 extern_func_parse(struct rte_swx_pipeline *p,
919 if (name[0] != 'f' || name[1] != '.')
922 return extern_func_find(p, &name[2]);
925 static struct field *
926 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
928 struct extern_func **function)
930 struct extern_func *func;
932 char *func_name, *field_name;
934 if ((name[0] != 'f') || (name[1] != '.'))
937 func_name = strdup(&name[2]);
941 field_name = strchr(func_name, '.');
950 func = extern_func_find(p, func_name);
956 f = struct_type_field_find(func->mailbox_struct_type, field_name);
970 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
972 const char *mailbox_struct_type_name,
973 rte_swx_extern_func_t func)
975 struct extern_func *f;
976 struct struct_type *mailbox_struct_type;
980 CHECK_NAME(name, EINVAL);
981 CHECK(!extern_func_find(p, name), EEXIST);
983 CHECK_NAME(mailbox_struct_type_name, EINVAL);
984 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
985 CHECK(mailbox_struct_type, EINVAL);
986 CHECK(!mailbox_struct_type->var_size, EINVAL);
990 /* Node allocation. */
991 f = calloc(1, sizeof(struct extern_func));
994 /* Node initialization. */
995 strcpy(f->name, name);
996 f->mailbox_struct_type = mailbox_struct_type;
998 f->struct_id = p->n_structs;
999 f->id = p->n_extern_funcs;
1001 /* Node add to tailq. */
1002 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
1003 p->n_extern_funcs++;
1010 extern_func_build(struct rte_swx_pipeline *p)
1014 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1015 struct thread *t = &p->threads[i];
1016 struct extern_func *func;
1018 /* Memory allocation. */
1019 t->extern_funcs = calloc(p->n_extern_funcs,
1020 sizeof(struct extern_func_runtime));
1021 CHECK(t->extern_funcs, ENOMEM);
1023 /* Extern function. */
1024 TAILQ_FOREACH(func, &p->extern_funcs, node) {
1025 struct extern_func_runtime *r =
1026 &t->extern_funcs[func->id];
1027 uint32_t mailbox_size =
1028 func->mailbox_struct_type->n_bits / 8;
1030 r->func = func->func;
1032 r->mailbox = calloc(1, mailbox_size);
1033 CHECK(r->mailbox, ENOMEM);
1035 t->structs[func->struct_id] = r->mailbox;
1043 extern_func_build_free(struct rte_swx_pipeline *p)
1047 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1048 struct thread *t = &p->threads[i];
1051 if (!t->extern_funcs)
1054 for (j = 0; j < p->n_extern_funcs; j++) {
1055 struct extern_func_runtime *r = &t->extern_funcs[j];
1060 free(t->extern_funcs);
1061 t->extern_funcs = NULL;
1066 extern_func_free(struct rte_swx_pipeline *p)
1068 extern_func_build_free(p);
1071 struct extern_func *elem;
1073 elem = TAILQ_FIRST(&p->extern_funcs);
1077 TAILQ_REMOVE(&p->extern_funcs, elem, node);
1085 static struct header *
1086 header_find(struct rte_swx_pipeline *p, const char *name)
1088 struct header *elem;
1090 TAILQ_FOREACH(elem, &p->headers, node)
1091 if (strcmp(elem->name, name) == 0)
1097 static struct header *
1098 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
1100 struct header *elem;
1102 TAILQ_FOREACH(elem, &p->headers, node)
1103 if (elem->struct_id == struct_id)
1109 static struct header *
1110 header_parse(struct rte_swx_pipeline *p,
1113 if (name[0] != 'h' || name[1] != '.')
1116 return header_find(p, &name[2]);
1119 static struct field *
1120 header_field_parse(struct rte_swx_pipeline *p,
1122 struct header **header)
1126 char *header_name, *field_name;
1128 if ((name[0] != 'h') || (name[1] != '.'))
1131 header_name = strdup(&name[2]);
1135 field_name = strchr(header_name, '.');
1144 h = header_find(p, header_name);
1150 f = struct_type_field_find(h->st, field_name);
1164 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
1166 const char *struct_type_name)
1168 struct struct_type *st;
1170 size_t n_headers_max;
1173 CHECK_NAME(name, EINVAL);
1174 CHECK_NAME(struct_type_name, EINVAL);
1176 CHECK(!header_find(p, name), EEXIST);
1178 st = struct_type_find(p, struct_type_name);
1181 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
1182 CHECK(p->n_headers < n_headers_max, ENOSPC);
1184 /* Node allocation. */
1185 h = calloc(1, sizeof(struct header));
1188 /* Node initialization. */
1189 strcpy(h->name, name);
1191 h->struct_id = p->n_structs;
1192 h->id = p->n_headers;
1194 /* Node add to tailq. */
1195 TAILQ_INSERT_TAIL(&p->headers, h, node);
1203 header_build(struct rte_swx_pipeline *p)
1206 uint32_t n_bytes = 0, i;
1208 TAILQ_FOREACH(h, &p->headers, node) {
1209 n_bytes += h->st->n_bits / 8;
1212 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1213 struct thread *t = &p->threads[i];
1214 uint32_t offset = 0;
1216 t->headers = calloc(p->n_headers,
1217 sizeof(struct header_runtime));
1218 CHECK(t->headers, ENOMEM);
1220 t->headers_out = calloc(p->n_headers,
1221 sizeof(struct header_out_runtime));
1222 CHECK(t->headers_out, ENOMEM);
1224 t->header_storage = calloc(1, n_bytes);
1225 CHECK(t->header_storage, ENOMEM);
1227 t->header_out_storage = calloc(1, n_bytes);
1228 CHECK(t->header_out_storage, ENOMEM);
1230 TAILQ_FOREACH(h, &p->headers, node) {
1231 uint8_t *header_storage;
1232 uint32_t n_bytes = h->st->n_bits / 8;
1234 header_storage = &t->header_storage[offset];
1237 t->headers[h->id].ptr0 = header_storage;
1238 t->headers[h->id].n_bytes = n_bytes;
1240 t->structs[h->struct_id] = header_storage;
1248 header_build_free(struct rte_swx_pipeline *p)
1252 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1253 struct thread *t = &p->threads[i];
1255 free(t->headers_out);
1256 t->headers_out = NULL;
1261 free(t->header_out_storage);
1262 t->header_out_storage = NULL;
1264 free(t->header_storage);
1265 t->header_storage = NULL;
1270 header_free(struct rte_swx_pipeline *p)
1272 header_build_free(p);
1275 struct header *elem;
1277 elem = TAILQ_FIRST(&p->headers);
1281 TAILQ_REMOVE(&p->headers, elem, node);
1289 static struct field *
1290 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
1292 if (!p->metadata_st)
1295 if (name[0] != 'm' || name[1] != '.')
1298 return struct_type_field_find(p->metadata_st, &name[2]);
1302 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
1303 const char *struct_type_name)
1305 struct struct_type *st = NULL;
1309 CHECK_NAME(struct_type_name, EINVAL);
1310 st = struct_type_find(p, struct_type_name);
1312 CHECK(!st->var_size, EINVAL);
1313 CHECK(!p->metadata_st, EINVAL);
1315 p->metadata_st = st;
1316 p->metadata_struct_id = p->n_structs;
1324 metadata_build(struct rte_swx_pipeline *p)
1326 uint32_t n_bytes = p->metadata_st->n_bits / 8;
1329 /* Thread-level initialization. */
1330 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1331 struct thread *t = &p->threads[i];
1334 metadata = calloc(1, n_bytes);
1335 CHECK(metadata, ENOMEM);
1337 t->metadata = metadata;
1338 t->structs[p->metadata_struct_id] = metadata;
1345 metadata_build_free(struct rte_swx_pipeline *p)
1349 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1350 struct thread *t = &p->threads[i];
1358 metadata_free(struct rte_swx_pipeline *p)
1360 metadata_build_free(p);
1367 instruction_is_tx(enum instruction_type type)
1380 instruction_is_jmp(struct instruction *instr)
1382 switch (instr->type) {
1384 case INSTR_JMP_VALID:
1385 case INSTR_JMP_INVALID:
1387 case INSTR_JMP_MISS:
1388 case INSTR_JMP_ACTION_HIT:
1389 case INSTR_JMP_ACTION_MISS:
1391 case INSTR_JMP_EQ_MH:
1392 case INSTR_JMP_EQ_HM:
1393 case INSTR_JMP_EQ_HH:
1394 case INSTR_JMP_EQ_I:
1396 case INSTR_JMP_NEQ_MH:
1397 case INSTR_JMP_NEQ_HM:
1398 case INSTR_JMP_NEQ_HH:
1399 case INSTR_JMP_NEQ_I:
1401 case INSTR_JMP_LT_MH:
1402 case INSTR_JMP_LT_HM:
1403 case INSTR_JMP_LT_HH:
1404 case INSTR_JMP_LT_MI:
1405 case INSTR_JMP_LT_HI:
1407 case INSTR_JMP_GT_MH:
1408 case INSTR_JMP_GT_HM:
1409 case INSTR_JMP_GT_HH:
1410 case INSTR_JMP_GT_MI:
1411 case INSTR_JMP_GT_HI:
1419 static struct field *
1420 action_field_parse(struct action *action, const char *name);
1422 static struct field *
1423 struct_field_parse(struct rte_swx_pipeline *p,
1424 struct action *action,
1426 uint32_t *struct_id)
1433 struct header *header;
1435 f = header_field_parse(p, name, &header);
1439 *struct_id = header->struct_id;
1445 f = metadata_field_parse(p, name);
1449 *struct_id = p->metadata_struct_id;
1458 f = action_field_parse(action, name);
1468 struct extern_obj *obj;
1470 f = extern_obj_mailbox_field_parse(p, name, &obj);
1474 *struct_id = obj->struct_id;
1480 struct extern_func *func;
1482 f = extern_func_mailbox_field_parse(p, name, &func);
1486 *struct_id = func->struct_id;
1499 instr_rx_translate(struct rte_swx_pipeline *p,
1500 struct action *action,
1503 struct instruction *instr,
1504 struct instruction_data *data __rte_unused)
1508 CHECK(!action, EINVAL);
1509 CHECK(n_tokens == 2, EINVAL);
1511 f = metadata_field_parse(p, tokens[1]);
1514 instr->type = INSTR_RX;
1515 instr->io.io.offset = f->offset / 8;
1516 instr->io.io.n_bits = f->n_bits;
1524 instr_tx_translate(struct rte_swx_pipeline *p,
1525 struct action *action __rte_unused,
1528 struct instruction *instr,
1529 struct instruction_data *data __rte_unused)
1531 char *port = tokens[1];
1535 CHECK(n_tokens == 2, EINVAL);
1537 f = metadata_field_parse(p, port);
1539 instr->type = INSTR_TX;
1540 instr->io.io.offset = f->offset / 8;
1541 instr->io.io.n_bits = f->n_bits;
1546 port_val = strtoul(port, &port, 0);
1547 CHECK(!port[0], EINVAL);
1549 instr->type = INSTR_TX_I;
1550 instr->io.io.val = port_val;
1555 instr_drop_translate(struct rte_swx_pipeline *p,
1556 struct action *action __rte_unused,
1557 char **tokens __rte_unused,
1559 struct instruction *instr,
1560 struct instruction_data *data __rte_unused)
1562 CHECK(n_tokens == 1, EINVAL);
1565 instr->type = INSTR_TX_I;
1566 instr->io.io.val = p->n_ports_out - 1;
1571 instr_tx_exec(struct rte_swx_pipeline *p)
1573 struct thread *t = &p->threads[p->thread_id];
1574 struct instruction *ip = t->ip;
1576 __instr_tx_exec(p, t, ip);
1579 thread_ip_reset(p, t);
1584 instr_tx_i_exec(struct rte_swx_pipeline *p)
1586 struct thread *t = &p->threads[p->thread_id];
1587 struct instruction *ip = t->ip;
1589 __instr_tx_i_exec(p, t, ip);
1592 thread_ip_reset(p, t);
1600 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
1601 struct action *action,
1604 struct instruction *instr,
1605 struct instruction_data *data __rte_unused)
1609 CHECK(!action, EINVAL);
1610 CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
1612 h = header_parse(p, tokens[1]);
1615 if (n_tokens == 2) {
1616 CHECK(!h->st->var_size, EINVAL);
1618 instr->type = INSTR_HDR_EXTRACT;
1619 instr->io.hdr.header_id[0] = h->id;
1620 instr->io.hdr.struct_id[0] = h->struct_id;
1621 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1625 CHECK(h->st->var_size, EINVAL);
1627 mf = metadata_field_parse(p, tokens[2]);
1629 CHECK(!mf->var_size, EINVAL);
1631 instr->type = INSTR_HDR_EXTRACT_M;
1632 instr->io.io.offset = mf->offset / 8;
1633 instr->io.io.n_bits = mf->n_bits;
1634 instr->io.hdr.header_id[0] = h->id;
1635 instr->io.hdr.struct_id[0] = h->struct_id;
1636 instr->io.hdr.n_bytes[0] = h->st->n_bits_min / 8;
1643 instr_hdr_lookahead_translate(struct rte_swx_pipeline *p,
1644 struct action *action,
1647 struct instruction *instr,
1648 struct instruction_data *data __rte_unused)
1652 CHECK(!action, EINVAL);
1653 CHECK(n_tokens == 2, EINVAL);
1655 h = header_parse(p, tokens[1]);
1657 CHECK(!h->st->var_size, EINVAL);
1659 instr->type = INSTR_HDR_LOOKAHEAD;
1660 instr->io.hdr.header_id[0] = h->id;
1661 instr->io.hdr.struct_id[0] = h->struct_id;
1662 instr->io.hdr.n_bytes[0] = 0; /* Unused. */
1668 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
1670 struct thread *t = &p->threads[p->thread_id];
1671 struct instruction *ip = t->ip;
1673 __instr_hdr_extract_exec(p, t, ip);
1680 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
1682 struct thread *t = &p->threads[p->thread_id];
1683 struct instruction *ip = t->ip;
1685 __instr_hdr_extract2_exec(p, t, ip);
1692 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
1694 struct thread *t = &p->threads[p->thread_id];
1695 struct instruction *ip = t->ip;
1697 __instr_hdr_extract3_exec(p, t, ip);
1704 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
1706 struct thread *t = &p->threads[p->thread_id];
1707 struct instruction *ip = t->ip;
1709 __instr_hdr_extract4_exec(p, t, ip);
1716 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
1718 struct thread *t = &p->threads[p->thread_id];
1719 struct instruction *ip = t->ip;
1721 __instr_hdr_extract5_exec(p, t, ip);
1728 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
1730 struct thread *t = &p->threads[p->thread_id];
1731 struct instruction *ip = t->ip;
1733 __instr_hdr_extract6_exec(p, t, ip);
1740 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
1742 struct thread *t = &p->threads[p->thread_id];
1743 struct instruction *ip = t->ip;
1745 __instr_hdr_extract7_exec(p, t, ip);
1752 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
1754 struct thread *t = &p->threads[p->thread_id];
1755 struct instruction *ip = t->ip;
1757 __instr_hdr_extract8_exec(p, t, ip);
1764 instr_hdr_extract_m_exec(struct rte_swx_pipeline *p)
1766 struct thread *t = &p->threads[p->thread_id];
1767 struct instruction *ip = t->ip;
1769 __instr_hdr_extract_m_exec(p, t, ip);
1776 instr_hdr_lookahead_exec(struct rte_swx_pipeline *p)
1778 struct thread *t = &p->threads[p->thread_id];
1779 struct instruction *ip = t->ip;
1781 __instr_hdr_lookahead_exec(p, t, ip);
1791 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
1792 struct action *action __rte_unused,
1795 struct instruction *instr,
1796 struct instruction_data *data __rte_unused)
1800 CHECK(n_tokens == 2, EINVAL);
1802 h = header_parse(p, tokens[1]);
1805 instr->type = INSTR_HDR_EMIT;
1806 instr->io.hdr.header_id[0] = h->id;
1807 instr->io.hdr.struct_id[0] = h->struct_id;
1808 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
1813 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
1815 struct thread *t = &p->threads[p->thread_id];
1816 struct instruction *ip = t->ip;
1818 __instr_hdr_emit_exec(p, t, ip);
1825 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
1827 struct thread *t = &p->threads[p->thread_id];
1828 struct instruction *ip = t->ip;
1830 __instr_hdr_emit_tx_exec(p, t, ip);
1833 thread_ip_reset(p, t);
1838 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
1840 struct thread *t = &p->threads[p->thread_id];
1841 struct instruction *ip = t->ip;
1843 __instr_hdr_emit2_tx_exec(p, t, ip);
1846 thread_ip_reset(p, t);
1851 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
1853 struct thread *t = &p->threads[p->thread_id];
1854 struct instruction *ip = t->ip;
1856 __instr_hdr_emit3_tx_exec(p, t, ip);
1859 thread_ip_reset(p, t);
1864 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
1866 struct thread *t = &p->threads[p->thread_id];
1867 struct instruction *ip = t->ip;
1869 __instr_hdr_emit4_tx_exec(p, t, ip);
1872 thread_ip_reset(p, t);
1877 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
1879 struct thread *t = &p->threads[p->thread_id];
1880 struct instruction *ip = t->ip;
1882 __instr_hdr_emit5_tx_exec(p, t, ip);
1885 thread_ip_reset(p, t);
1890 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
1892 struct thread *t = &p->threads[p->thread_id];
1893 struct instruction *ip = t->ip;
1895 __instr_hdr_emit6_tx_exec(p, t, ip);
1898 thread_ip_reset(p, t);
1903 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
1905 struct thread *t = &p->threads[p->thread_id];
1906 struct instruction *ip = t->ip;
1908 __instr_hdr_emit7_tx_exec(p, t, ip);
1911 thread_ip_reset(p, t);
1916 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
1918 struct thread *t = &p->threads[p->thread_id];
1919 struct instruction *ip = t->ip;
1921 __instr_hdr_emit8_tx_exec(p, t, ip);
1924 thread_ip_reset(p, t);
1932 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
1933 struct action *action __rte_unused,
1936 struct instruction *instr,
1937 struct instruction_data *data __rte_unused)
1941 CHECK(n_tokens == 2, EINVAL);
1943 h = header_parse(p, tokens[1]);
1946 instr->type = INSTR_HDR_VALIDATE;
1947 instr->valid.header_id = h->id;
1952 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
1954 struct thread *t = &p->threads[p->thread_id];
1955 struct instruction *ip = t->ip;
1957 __instr_hdr_validate_exec(p, t, ip);
1967 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
1968 struct action *action __rte_unused,
1971 struct instruction *instr,
1972 struct instruction_data *data __rte_unused)
1976 CHECK(n_tokens == 2, EINVAL);
1978 h = header_parse(p, tokens[1]);
1981 instr->type = INSTR_HDR_INVALIDATE;
1982 instr->valid.header_id = h->id;
1987 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
1989 struct thread *t = &p->threads[p->thread_id];
1990 struct instruction *ip = t->ip;
1992 __instr_hdr_invalidate_exec(p, t, ip);
2001 static struct table *
2002 table_find(struct rte_swx_pipeline *p, const char *name);
2004 static struct selector *
2005 selector_find(struct rte_swx_pipeline *p, const char *name);
2007 static struct learner *
2008 learner_find(struct rte_swx_pipeline *p, const char *name);
2011 instr_table_translate(struct rte_swx_pipeline *p,
2012 struct action *action,
2015 struct instruction *instr,
2016 struct instruction_data *data __rte_unused)
2022 CHECK(!action, EINVAL);
2023 CHECK(n_tokens == 2, EINVAL);
2025 t = table_find(p, tokens[1]);
2027 instr->type = INSTR_TABLE;
2028 instr->table.table_id = t->id;
2032 s = selector_find(p, tokens[1]);
2034 instr->type = INSTR_SELECTOR;
2035 instr->table.table_id = s->id;
2039 l = learner_find(p, tokens[1]);
2041 instr->type = INSTR_LEARNER;
2042 instr->table.table_id = l->id;
2050 instr_table_exec(struct rte_swx_pipeline *p)
2052 struct thread *t = &p->threads[p->thread_id];
2053 struct instruction *ip = t->ip;
2054 uint32_t table_id = ip->table.table_id;
2055 struct rte_swx_table_state *ts = &t->table_state[table_id];
2056 struct table_runtime *table = &t->tables[table_id];
2057 struct table_statistics *stats = &p->table_stats[table_id];
2058 uint64_t action_id, n_pkts_hit, n_pkts_action;
2059 uint8_t *action_data;
2063 done = table->func(ts->obj,
2071 TRACE("[Thread %2u] table %u (not finalized)\n",
2079 action_id = hit ? action_id : ts->default_action_id;
2080 action_data = hit ? action_data : ts->default_action_data;
2081 n_pkts_hit = stats->n_pkts_hit[hit];
2082 n_pkts_action = stats->n_pkts_action[action_id];
2084 TRACE("[Thread %2u] table %u (%s, action %u)\n",
2087 hit ? "hit" : "miss",
2088 (uint32_t)action_id);
2090 t->action_id = action_id;
2091 t->structs[0] = action_data;
2093 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2094 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2097 thread_ip_action_call(p, t, action_id);
2101 instr_selector_exec(struct rte_swx_pipeline *p)
2103 struct thread *t = &p->threads[p->thread_id];
2104 struct instruction *ip = t->ip;
2105 uint32_t selector_id = ip->table.table_id;
2106 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
2107 struct selector_runtime *selector = &t->selectors[selector_id];
2108 struct selector_statistics *stats = &p->selector_stats[selector_id];
2109 uint64_t n_pkts = stats->n_pkts;
2113 done = rte_swx_table_selector_select(ts->obj,
2115 selector->group_id_buffer,
2116 selector->selector_buffer,
2117 selector->member_id_buffer);
2120 TRACE("[Thread %2u] selector %u (not finalized)\n",
2129 TRACE("[Thread %2u] selector %u\n",
2133 stats->n_pkts = n_pkts + 1;
2140 instr_learner_exec(struct rte_swx_pipeline *p)
2142 struct thread *t = &p->threads[p->thread_id];
2143 struct instruction *ip = t->ip;
2144 uint32_t learner_id = ip->table.table_id;
2145 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2146 p->n_selectors + learner_id];
2147 struct learner_runtime *l = &t->learners[learner_id];
2148 struct learner_statistics *stats = &p->learner_stats[learner_id];
2149 uint64_t action_id, n_pkts_hit, n_pkts_action, time;
2150 uint8_t *action_data;
2154 time = rte_get_tsc_cycles();
2156 done = rte_swx_table_learner_lookup(ts->obj,
2165 TRACE("[Thread %2u] learner %u (not finalized)\n",
2173 action_id = hit ? action_id : ts->default_action_id;
2174 action_data = hit ? action_data : ts->default_action_data;
2175 n_pkts_hit = stats->n_pkts_hit[hit];
2176 n_pkts_action = stats->n_pkts_action[action_id];
2178 TRACE("[Thread %2u] learner %u (%s, action %u)\n",
2181 hit ? "hit" : "miss",
2182 (uint32_t)action_id);
2184 t->action_id = action_id;
2185 t->structs[0] = action_data;
2187 t->learner_id = learner_id;
2189 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
2190 stats->n_pkts_action[action_id] = n_pkts_action + 1;
2193 thread_ip_action_call(p, t, action_id);
2199 static struct action *
2200 action_find(struct rte_swx_pipeline *p, const char *name);
2203 action_has_nbo_args(struct action *a);
2206 instr_learn_translate(struct rte_swx_pipeline *p,
2207 struct action *action,
2210 struct instruction *instr,
2211 struct instruction_data *data __rte_unused)
2215 CHECK(action, EINVAL);
2216 CHECK(n_tokens == 2, EINVAL);
2218 a = action_find(p, tokens[1]);
2220 CHECK(!action_has_nbo_args(a), EINVAL);
2222 instr->type = INSTR_LEARNER_LEARN;
2223 instr->learn.action_id = a->id;
2229 instr_learn_exec(struct rte_swx_pipeline *p)
2231 struct thread *t = &p->threads[p->thread_id];
2232 struct instruction *ip = t->ip;
2234 __instr_learn_exec(p, t, ip);
2244 instr_forget_translate(struct rte_swx_pipeline *p __rte_unused,
2245 struct action *action,
2246 char **tokens __rte_unused,
2248 struct instruction *instr,
2249 struct instruction_data *data __rte_unused)
2251 CHECK(action, EINVAL);
2252 CHECK(n_tokens == 1, EINVAL);
2254 instr->type = INSTR_LEARNER_FORGET;
2260 instr_forget_exec(struct rte_swx_pipeline *p)
2262 struct thread *t = &p->threads[p->thread_id];
2263 struct instruction *ip = t->ip;
2265 __instr_forget_exec(p, t, ip);
2275 instr_extern_translate(struct rte_swx_pipeline *p,
2276 struct action *action __rte_unused,
2279 struct instruction *instr,
2280 struct instruction_data *data __rte_unused)
2282 char *token = tokens[1];
2284 CHECK(n_tokens == 2, EINVAL);
2286 if (token[0] == 'e') {
2287 struct extern_obj *obj;
2288 struct extern_type_member_func *func;
2290 func = extern_obj_member_func_parse(p, token, &obj);
2291 CHECK(func, EINVAL);
2293 instr->type = INSTR_EXTERN_OBJ;
2294 instr->ext_obj.ext_obj_id = obj->id;
2295 instr->ext_obj.func_id = func->id;
2300 if (token[0] == 'f') {
2301 struct extern_func *func;
2303 func = extern_func_parse(p, token);
2304 CHECK(func, EINVAL);
2306 instr->type = INSTR_EXTERN_FUNC;
2307 instr->ext_func.ext_func_id = func->id;
2316 instr_extern_obj_exec(struct rte_swx_pipeline *p)
2318 struct thread *t = &p->threads[p->thread_id];
2319 struct instruction *ip = t->ip;
2322 /* Extern object member function execute. */
2323 done = __instr_extern_obj_exec(p, t, ip);
2326 thread_ip_inc_cond(t, done);
2327 thread_yield_cond(p, done ^ 1);
2331 instr_extern_func_exec(struct rte_swx_pipeline *p)
2333 struct thread *t = &p->threads[p->thread_id];
2334 struct instruction *ip = t->ip;
2337 /* Extern function execute. */
2338 done = __instr_extern_func_exec(p, t, ip);
2341 thread_ip_inc_cond(t, done);
2342 thread_yield_cond(p, done ^ 1);
2349 instr_mov_translate(struct rte_swx_pipeline *p,
2350 struct action *action,
2353 struct instruction *instr,
2354 struct instruction_data *data __rte_unused)
2356 char *dst = tokens[1], *src = tokens[2];
2357 struct field *fdst, *fsrc;
2359 uint32_t dst_struct_id = 0, src_struct_id = 0;
2361 CHECK(n_tokens == 3, EINVAL);
2363 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2364 CHECK(fdst, EINVAL);
2365 CHECK(!fdst->var_size, EINVAL);
2367 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
2368 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2370 CHECK(!fsrc->var_size, EINVAL);
2372 instr->type = INSTR_MOV;
2373 if (dst[0] != 'h' && src[0] == 'h')
2374 instr->type = INSTR_MOV_MH;
2375 if (dst[0] == 'h' && src[0] != 'h')
2376 instr->type = INSTR_MOV_HM;
2377 if (dst[0] == 'h' && src[0] == 'h')
2378 instr->type = INSTR_MOV_HH;
2380 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2381 instr->mov.dst.n_bits = fdst->n_bits;
2382 instr->mov.dst.offset = fdst->offset / 8;
2383 instr->mov.src.struct_id = (uint8_t)src_struct_id;
2384 instr->mov.src.n_bits = fsrc->n_bits;
2385 instr->mov.src.offset = fsrc->offset / 8;
2390 src_val = strtoull(src, &src, 0);
2391 CHECK(!src[0], EINVAL);
2394 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2396 instr->type = INSTR_MOV_I;
2397 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
2398 instr->mov.dst.n_bits = fdst->n_bits;
2399 instr->mov.dst.offset = fdst->offset / 8;
2400 instr->mov.src_val = src_val;
2405 instr_mov_exec(struct rte_swx_pipeline *p)
2407 struct thread *t = &p->threads[p->thread_id];
2408 struct instruction *ip = t->ip;
2410 __instr_mov_exec(p, t, ip);
2417 instr_mov_mh_exec(struct rte_swx_pipeline *p)
2419 struct thread *t = &p->threads[p->thread_id];
2420 struct instruction *ip = t->ip;
2422 __instr_mov_mh_exec(p, t, ip);
2429 instr_mov_hm_exec(struct rte_swx_pipeline *p)
2431 struct thread *t = &p->threads[p->thread_id];
2432 struct instruction *ip = t->ip;
2434 __instr_mov_hm_exec(p, t, ip);
2441 instr_mov_hh_exec(struct rte_swx_pipeline *p)
2443 struct thread *t = &p->threads[p->thread_id];
2444 struct instruction *ip = t->ip;
2446 __instr_mov_hh_exec(p, t, ip);
2453 instr_mov_i_exec(struct rte_swx_pipeline *p)
2455 struct thread *t = &p->threads[p->thread_id];
2456 struct instruction *ip = t->ip;
2458 __instr_mov_i_exec(p, t, ip);
2468 instr_dma_ht_exec(struct rte_swx_pipeline *p)
2470 struct thread *t = &p->threads[p->thread_id];
2471 struct instruction *ip = t->ip;
2473 __instr_dma_ht_exec(p, t, ip);
2480 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
2482 struct thread *t = &p->threads[p->thread_id];
2483 struct instruction *ip = t->ip;
2485 __instr_dma_ht2_exec(p, t, ip);
2492 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
2494 struct thread *t = &p->threads[p->thread_id];
2495 struct instruction *ip = t->ip;
2497 __instr_dma_ht3_exec(p, t, ip);
2504 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
2506 struct thread *t = &p->threads[p->thread_id];
2507 struct instruction *ip = t->ip;
2509 __instr_dma_ht4_exec(p, t, ip);
2516 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
2518 struct thread *t = &p->threads[p->thread_id];
2519 struct instruction *ip = t->ip;
2521 __instr_dma_ht5_exec(p, t, ip);
2528 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
2530 struct thread *t = &p->threads[p->thread_id];
2531 struct instruction *ip = t->ip;
2533 __instr_dma_ht6_exec(p, t, ip);
2540 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
2542 struct thread *t = &p->threads[p->thread_id];
2543 struct instruction *ip = t->ip;
2545 __instr_dma_ht7_exec(p, t, ip);
2552 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
2554 struct thread *t = &p->threads[p->thread_id];
2555 struct instruction *ip = t->ip;
2557 __instr_dma_ht8_exec(p, t, ip);
2567 instr_alu_add_translate(struct rte_swx_pipeline *p,
2568 struct action *action,
2571 struct instruction *instr,
2572 struct instruction_data *data __rte_unused)
2574 char *dst = tokens[1], *src = tokens[2];
2575 struct field *fdst, *fsrc;
2577 uint32_t dst_struct_id = 0, src_struct_id = 0;
2579 CHECK(n_tokens == 3, EINVAL);
2581 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2582 CHECK(fdst, EINVAL);
2583 CHECK(!fdst->var_size, EINVAL);
2585 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
2586 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2588 CHECK(!fsrc->var_size, EINVAL);
2590 instr->type = INSTR_ALU_ADD;
2591 if (dst[0] == 'h' && src[0] != 'h')
2592 instr->type = INSTR_ALU_ADD_HM;
2593 if (dst[0] != 'h' && src[0] == 'h')
2594 instr->type = INSTR_ALU_ADD_MH;
2595 if (dst[0] == 'h' && src[0] == 'h')
2596 instr->type = INSTR_ALU_ADD_HH;
2598 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2599 instr->alu.dst.n_bits = fdst->n_bits;
2600 instr->alu.dst.offset = fdst->offset / 8;
2601 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2602 instr->alu.src.n_bits = fsrc->n_bits;
2603 instr->alu.src.offset = fsrc->offset / 8;
2607 /* ADD_MI, ADD_HI. */
2608 src_val = strtoull(src, &src, 0);
2609 CHECK(!src[0], EINVAL);
2611 instr->type = INSTR_ALU_ADD_MI;
2613 instr->type = INSTR_ALU_ADD_HI;
2615 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2616 instr->alu.dst.n_bits = fdst->n_bits;
2617 instr->alu.dst.offset = fdst->offset / 8;
2618 instr->alu.src_val = src_val;
2623 instr_alu_sub_translate(struct rte_swx_pipeline *p,
2624 struct action *action,
2627 struct instruction *instr,
2628 struct instruction_data *data __rte_unused)
2630 char *dst = tokens[1], *src = tokens[2];
2631 struct field *fdst, *fsrc;
2633 uint32_t dst_struct_id = 0, src_struct_id = 0;
2635 CHECK(n_tokens == 3, EINVAL);
2637 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2638 CHECK(fdst, EINVAL);
2639 CHECK(!fdst->var_size, EINVAL);
2641 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
2642 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2644 CHECK(!fsrc->var_size, EINVAL);
2646 instr->type = INSTR_ALU_SUB;
2647 if (dst[0] == 'h' && src[0] != 'h')
2648 instr->type = INSTR_ALU_SUB_HM;
2649 if (dst[0] != 'h' && src[0] == 'h')
2650 instr->type = INSTR_ALU_SUB_MH;
2651 if (dst[0] == 'h' && src[0] == 'h')
2652 instr->type = INSTR_ALU_SUB_HH;
2654 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2655 instr->alu.dst.n_bits = fdst->n_bits;
2656 instr->alu.dst.offset = fdst->offset / 8;
2657 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2658 instr->alu.src.n_bits = fsrc->n_bits;
2659 instr->alu.src.offset = fsrc->offset / 8;
2663 /* SUB_MI, SUB_HI. */
2664 src_val = strtoull(src, &src, 0);
2665 CHECK(!src[0], EINVAL);
2667 instr->type = INSTR_ALU_SUB_MI;
2669 instr->type = INSTR_ALU_SUB_HI;
2671 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2672 instr->alu.dst.n_bits = fdst->n_bits;
2673 instr->alu.dst.offset = fdst->offset / 8;
2674 instr->alu.src_val = src_val;
2679 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
2680 struct action *action __rte_unused,
2683 struct instruction *instr,
2684 struct instruction_data *data __rte_unused)
2686 char *dst = tokens[1], *src = tokens[2];
2687 struct header *hdst, *hsrc;
2688 struct field *fdst, *fsrc;
2690 CHECK(n_tokens == 3, EINVAL);
2692 fdst = header_field_parse(p, dst, &hdst);
2693 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2694 CHECK(!fdst->var_size, EINVAL);
2697 fsrc = header_field_parse(p, src, &hsrc);
2699 CHECK(!fsrc->var_size, EINVAL);
2701 instr->type = INSTR_ALU_CKADD_FIELD;
2702 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2703 instr->alu.dst.n_bits = fdst->n_bits;
2704 instr->alu.dst.offset = fdst->offset / 8;
2705 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2706 instr->alu.src.n_bits = fsrc->n_bits;
2707 instr->alu.src.offset = fsrc->offset / 8;
2711 /* CKADD_STRUCT, CKADD_STRUCT20. */
2712 hsrc = header_parse(p, src);
2713 CHECK(hsrc, EINVAL);
2714 CHECK(!hsrc->st->var_size, EINVAL);
2716 instr->type = INSTR_ALU_CKADD_STRUCT;
2717 if ((hsrc->st->n_bits / 8) == 20)
2718 instr->type = INSTR_ALU_CKADD_STRUCT20;
2720 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2721 instr->alu.dst.n_bits = fdst->n_bits;
2722 instr->alu.dst.offset = fdst->offset / 8;
2723 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2724 instr->alu.src.n_bits = hsrc->st->n_bits;
2725 instr->alu.src.offset = 0; /* Unused. */
2730 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
2731 struct action *action __rte_unused,
2734 struct instruction *instr,
2735 struct instruction_data *data __rte_unused)
2737 char *dst = tokens[1], *src = tokens[2];
2738 struct header *hdst, *hsrc;
2739 struct field *fdst, *fsrc;
2741 CHECK(n_tokens == 3, EINVAL);
2743 fdst = header_field_parse(p, dst, &hdst);
2744 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
2745 CHECK(!fdst->var_size, EINVAL);
2747 fsrc = header_field_parse(p, src, &hsrc);
2748 CHECK(fsrc, EINVAL);
2749 CHECK(!fsrc->var_size, EINVAL);
2751 instr->type = INSTR_ALU_CKSUB_FIELD;
2752 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
2753 instr->alu.dst.n_bits = fdst->n_bits;
2754 instr->alu.dst.offset = fdst->offset / 8;
2755 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
2756 instr->alu.src.n_bits = fsrc->n_bits;
2757 instr->alu.src.offset = fsrc->offset / 8;
2762 instr_alu_shl_translate(struct rte_swx_pipeline *p,
2763 struct action *action,
2766 struct instruction *instr,
2767 struct instruction_data *data __rte_unused)
2769 char *dst = tokens[1], *src = tokens[2];
2770 struct field *fdst, *fsrc;
2772 uint32_t dst_struct_id = 0, src_struct_id = 0;
2774 CHECK(n_tokens == 3, EINVAL);
2776 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2777 CHECK(fdst, EINVAL);
2778 CHECK(!fdst->var_size, EINVAL);
2780 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
2781 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2783 CHECK(!fsrc->var_size, EINVAL);
2785 instr->type = INSTR_ALU_SHL;
2786 if (dst[0] == 'h' && src[0] != 'h')
2787 instr->type = INSTR_ALU_SHL_HM;
2788 if (dst[0] != 'h' && src[0] == 'h')
2789 instr->type = INSTR_ALU_SHL_MH;
2790 if (dst[0] == 'h' && src[0] == 'h')
2791 instr->type = INSTR_ALU_SHL_HH;
2793 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2794 instr->alu.dst.n_bits = fdst->n_bits;
2795 instr->alu.dst.offset = fdst->offset / 8;
2796 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2797 instr->alu.src.n_bits = fsrc->n_bits;
2798 instr->alu.src.offset = fsrc->offset / 8;
2802 /* SHL_MI, SHL_HI. */
2803 src_val = strtoull(src, &src, 0);
2804 CHECK(!src[0], EINVAL);
2806 instr->type = INSTR_ALU_SHL_MI;
2808 instr->type = INSTR_ALU_SHL_HI;
2810 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2811 instr->alu.dst.n_bits = fdst->n_bits;
2812 instr->alu.dst.offset = fdst->offset / 8;
2813 instr->alu.src_val = src_val;
2818 instr_alu_shr_translate(struct rte_swx_pipeline *p,
2819 struct action *action,
2822 struct instruction *instr,
2823 struct instruction_data *data __rte_unused)
2825 char *dst = tokens[1], *src = tokens[2];
2826 struct field *fdst, *fsrc;
2828 uint32_t dst_struct_id = 0, src_struct_id = 0;
2830 CHECK(n_tokens == 3, EINVAL);
2832 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2833 CHECK(fdst, EINVAL);
2834 CHECK(!fdst->var_size, EINVAL);
2836 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
2837 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2839 CHECK(!fsrc->var_size, EINVAL);
2841 instr->type = INSTR_ALU_SHR;
2842 if (dst[0] == 'h' && src[0] != 'h')
2843 instr->type = INSTR_ALU_SHR_HM;
2844 if (dst[0] != 'h' && src[0] == 'h')
2845 instr->type = INSTR_ALU_SHR_MH;
2846 if (dst[0] == 'h' && src[0] == 'h')
2847 instr->type = INSTR_ALU_SHR_HH;
2849 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2850 instr->alu.dst.n_bits = fdst->n_bits;
2851 instr->alu.dst.offset = fdst->offset / 8;
2852 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2853 instr->alu.src.n_bits = fsrc->n_bits;
2854 instr->alu.src.offset = fsrc->offset / 8;
2858 /* SHR_MI, SHR_HI. */
2859 src_val = strtoull(src, &src, 0);
2860 CHECK(!src[0], EINVAL);
2862 instr->type = INSTR_ALU_SHR_MI;
2864 instr->type = INSTR_ALU_SHR_HI;
2866 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2867 instr->alu.dst.n_bits = fdst->n_bits;
2868 instr->alu.dst.offset = fdst->offset / 8;
2869 instr->alu.src_val = src_val;
2874 instr_alu_and_translate(struct rte_swx_pipeline *p,
2875 struct action *action,
2878 struct instruction *instr,
2879 struct instruction_data *data __rte_unused)
2881 char *dst = tokens[1], *src = tokens[2];
2882 struct field *fdst, *fsrc;
2884 uint32_t dst_struct_id = 0, src_struct_id = 0;
2886 CHECK(n_tokens == 3, EINVAL);
2888 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2889 CHECK(fdst, EINVAL);
2890 CHECK(!fdst->var_size, EINVAL);
2892 /* AND, AND_MH, AND_HM, AND_HH. */
2893 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2895 CHECK(!fsrc->var_size, EINVAL);
2897 instr->type = INSTR_ALU_AND;
2898 if (dst[0] != 'h' && src[0] == 'h')
2899 instr->type = INSTR_ALU_AND_MH;
2900 if (dst[0] == 'h' && src[0] != 'h')
2901 instr->type = INSTR_ALU_AND_HM;
2902 if (dst[0] == 'h' && src[0] == 'h')
2903 instr->type = INSTR_ALU_AND_HH;
2905 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2906 instr->alu.dst.n_bits = fdst->n_bits;
2907 instr->alu.dst.offset = fdst->offset / 8;
2908 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2909 instr->alu.src.n_bits = fsrc->n_bits;
2910 instr->alu.src.offset = fsrc->offset / 8;
2915 src_val = strtoull(src, &src, 0);
2916 CHECK(!src[0], EINVAL);
2919 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2921 instr->type = INSTR_ALU_AND_I;
2922 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2923 instr->alu.dst.n_bits = fdst->n_bits;
2924 instr->alu.dst.offset = fdst->offset / 8;
2925 instr->alu.src_val = src_val;
2930 instr_alu_or_translate(struct rte_swx_pipeline *p,
2931 struct action *action,
2934 struct instruction *instr,
2935 struct instruction_data *data __rte_unused)
2937 char *dst = tokens[1], *src = tokens[2];
2938 struct field *fdst, *fsrc;
2940 uint32_t dst_struct_id = 0, src_struct_id = 0;
2942 CHECK(n_tokens == 3, EINVAL);
2944 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
2945 CHECK(fdst, EINVAL);
2946 CHECK(!fdst->var_size, EINVAL);
2948 /* OR, OR_MH, OR_HM, OR_HH. */
2949 fsrc = struct_field_parse(p, action, src, &src_struct_id);
2951 CHECK(!fsrc->var_size, EINVAL);
2953 instr->type = INSTR_ALU_OR;
2954 if (dst[0] != 'h' && src[0] == 'h')
2955 instr->type = INSTR_ALU_OR_MH;
2956 if (dst[0] == 'h' && src[0] != 'h')
2957 instr->type = INSTR_ALU_OR_HM;
2958 if (dst[0] == 'h' && src[0] == 'h')
2959 instr->type = INSTR_ALU_OR_HH;
2961 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2962 instr->alu.dst.n_bits = fdst->n_bits;
2963 instr->alu.dst.offset = fdst->offset / 8;
2964 instr->alu.src.struct_id = (uint8_t)src_struct_id;
2965 instr->alu.src.n_bits = fsrc->n_bits;
2966 instr->alu.src.offset = fsrc->offset / 8;
2971 src_val = strtoull(src, &src, 0);
2972 CHECK(!src[0], EINVAL);
2975 src_val = hton64(src_val) >> (64 - fdst->n_bits);
2977 instr->type = INSTR_ALU_OR_I;
2978 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
2979 instr->alu.dst.n_bits = fdst->n_bits;
2980 instr->alu.dst.offset = fdst->offset / 8;
2981 instr->alu.src_val = src_val;
2986 instr_alu_xor_translate(struct rte_swx_pipeline *p,
2987 struct action *action,
2990 struct instruction *instr,
2991 struct instruction_data *data __rte_unused)
2993 char *dst = tokens[1], *src = tokens[2];
2994 struct field *fdst, *fsrc;
2996 uint32_t dst_struct_id = 0, src_struct_id = 0;
2998 CHECK(n_tokens == 3, EINVAL);
3000 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3001 CHECK(fdst, EINVAL);
3002 CHECK(!fdst->var_size, EINVAL);
3004 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
3005 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3007 CHECK(!fsrc->var_size, EINVAL);
3009 instr->type = INSTR_ALU_XOR;
3010 if (dst[0] != 'h' && src[0] == 'h')
3011 instr->type = INSTR_ALU_XOR_MH;
3012 if (dst[0] == 'h' && src[0] != 'h')
3013 instr->type = INSTR_ALU_XOR_HM;
3014 if (dst[0] == 'h' && src[0] == 'h')
3015 instr->type = INSTR_ALU_XOR_HH;
3017 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3018 instr->alu.dst.n_bits = fdst->n_bits;
3019 instr->alu.dst.offset = fdst->offset / 8;
3020 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3021 instr->alu.src.n_bits = fsrc->n_bits;
3022 instr->alu.src.offset = fsrc->offset / 8;
3027 src_val = strtoull(src, &src, 0);
3028 CHECK(!src[0], EINVAL);
3031 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3033 instr->type = INSTR_ALU_XOR_I;
3034 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3035 instr->alu.dst.n_bits = fdst->n_bits;
3036 instr->alu.dst.offset = fdst->offset / 8;
3037 instr->alu.src_val = src_val;
3042 instr_alu_add_exec(struct rte_swx_pipeline *p)
3044 struct thread *t = &p->threads[p->thread_id];
3045 struct instruction *ip = t->ip;
3048 __instr_alu_add_exec(p, t, ip);
3055 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
3057 struct thread *t = &p->threads[p->thread_id];
3058 struct instruction *ip = t->ip;
3061 __instr_alu_add_mh_exec(p, t, ip);
3068 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
3070 struct thread *t = &p->threads[p->thread_id];
3071 struct instruction *ip = t->ip;
3074 __instr_alu_add_hm_exec(p, t, ip);
3081 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
3083 struct thread *t = &p->threads[p->thread_id];
3084 struct instruction *ip = t->ip;
3087 __instr_alu_add_hh_exec(p, t, ip);
3094 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
3096 struct thread *t = &p->threads[p->thread_id];
3097 struct instruction *ip = t->ip;
3100 __instr_alu_add_mi_exec(p, t, ip);
3107 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
3109 struct thread *t = &p->threads[p->thread_id];
3110 struct instruction *ip = t->ip;
3113 __instr_alu_add_hi_exec(p, t, ip);
3120 instr_alu_sub_exec(struct rte_swx_pipeline *p)
3122 struct thread *t = &p->threads[p->thread_id];
3123 struct instruction *ip = t->ip;
3126 __instr_alu_sub_exec(p, t, ip);
3133 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
3135 struct thread *t = &p->threads[p->thread_id];
3136 struct instruction *ip = t->ip;
3139 __instr_alu_sub_mh_exec(p, t, ip);
3146 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
3148 struct thread *t = &p->threads[p->thread_id];
3149 struct instruction *ip = t->ip;
3152 __instr_alu_sub_hm_exec(p, t, ip);
3159 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
3161 struct thread *t = &p->threads[p->thread_id];
3162 struct instruction *ip = t->ip;
3165 __instr_alu_sub_hh_exec(p, t, ip);
3172 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
3174 struct thread *t = &p->threads[p->thread_id];
3175 struct instruction *ip = t->ip;
3178 __instr_alu_sub_mi_exec(p, t, ip);
3185 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
3187 struct thread *t = &p->threads[p->thread_id];
3188 struct instruction *ip = t->ip;
3191 __instr_alu_sub_hi_exec(p, t, ip);
3198 instr_alu_shl_exec(struct rte_swx_pipeline *p)
3200 struct thread *t = &p->threads[p->thread_id];
3201 struct instruction *ip = t->ip;
3204 __instr_alu_shl_exec(p, t, ip);
3211 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
3213 struct thread *t = &p->threads[p->thread_id];
3214 struct instruction *ip = t->ip;
3217 __instr_alu_shl_mh_exec(p, t, ip);
3224 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
3226 struct thread *t = &p->threads[p->thread_id];
3227 struct instruction *ip = t->ip;
3230 __instr_alu_shl_hm_exec(p, t, ip);
3237 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
3239 struct thread *t = &p->threads[p->thread_id];
3240 struct instruction *ip = t->ip;
3243 __instr_alu_shl_hh_exec(p, t, ip);
3250 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
3252 struct thread *t = &p->threads[p->thread_id];
3253 struct instruction *ip = t->ip;
3256 __instr_alu_shl_mi_exec(p, t, ip);
3263 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
3265 struct thread *t = &p->threads[p->thread_id];
3266 struct instruction *ip = t->ip;
3269 __instr_alu_shl_hi_exec(p, t, ip);
3276 instr_alu_shr_exec(struct rte_swx_pipeline *p)
3278 struct thread *t = &p->threads[p->thread_id];
3279 struct instruction *ip = t->ip;
3282 __instr_alu_shr_exec(p, t, ip);
3289 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
3291 struct thread *t = &p->threads[p->thread_id];
3292 struct instruction *ip = t->ip;
3295 __instr_alu_shr_mh_exec(p, t, ip);
3302 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
3304 struct thread *t = &p->threads[p->thread_id];
3305 struct instruction *ip = t->ip;
3308 __instr_alu_shr_hm_exec(p, t, ip);
3315 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
3317 struct thread *t = &p->threads[p->thread_id];
3318 struct instruction *ip = t->ip;
3321 __instr_alu_shr_hh_exec(p, t, ip);
3328 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
3330 struct thread *t = &p->threads[p->thread_id];
3331 struct instruction *ip = t->ip;
3334 __instr_alu_shr_mi_exec(p, t, ip);
3341 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
3343 struct thread *t = &p->threads[p->thread_id];
3344 struct instruction *ip = t->ip;
3347 __instr_alu_shr_hi_exec(p, t, ip);
3354 instr_alu_and_exec(struct rte_swx_pipeline *p)
3356 struct thread *t = &p->threads[p->thread_id];
3357 struct instruction *ip = t->ip;
3360 __instr_alu_and_exec(p, t, ip);
3367 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
3369 struct thread *t = &p->threads[p->thread_id];
3370 struct instruction *ip = t->ip;
3373 __instr_alu_and_mh_exec(p, t, ip);
3380 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
3382 struct thread *t = &p->threads[p->thread_id];
3383 struct instruction *ip = t->ip;
3386 __instr_alu_and_hm_exec(p, t, ip);
3393 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
3395 struct thread *t = &p->threads[p->thread_id];
3396 struct instruction *ip = t->ip;
3399 __instr_alu_and_hh_exec(p, t, ip);
3406 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
3408 struct thread *t = &p->threads[p->thread_id];
3409 struct instruction *ip = t->ip;
3412 __instr_alu_and_i_exec(p, t, ip);
3419 instr_alu_or_exec(struct rte_swx_pipeline *p)
3421 struct thread *t = &p->threads[p->thread_id];
3422 struct instruction *ip = t->ip;
3425 __instr_alu_or_exec(p, t, ip);
3432 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
3434 struct thread *t = &p->threads[p->thread_id];
3435 struct instruction *ip = t->ip;
3438 __instr_alu_or_mh_exec(p, t, ip);
3445 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
3447 struct thread *t = &p->threads[p->thread_id];
3448 struct instruction *ip = t->ip;
3451 __instr_alu_or_hm_exec(p, t, ip);
3458 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
3460 struct thread *t = &p->threads[p->thread_id];
3461 struct instruction *ip = t->ip;
3464 __instr_alu_or_hh_exec(p, t, ip);
3471 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
3473 struct thread *t = &p->threads[p->thread_id];
3474 struct instruction *ip = t->ip;
3477 __instr_alu_or_i_exec(p, t, ip);
3484 instr_alu_xor_exec(struct rte_swx_pipeline *p)
3486 struct thread *t = &p->threads[p->thread_id];
3487 struct instruction *ip = t->ip;
3490 __instr_alu_xor_exec(p, t, ip);
3497 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
3499 struct thread *t = &p->threads[p->thread_id];
3500 struct instruction *ip = t->ip;
3503 __instr_alu_xor_mh_exec(p, t, ip);
3510 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
3512 struct thread *t = &p->threads[p->thread_id];
3513 struct instruction *ip = t->ip;
3516 __instr_alu_xor_hm_exec(p, t, ip);
3523 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
3525 struct thread *t = &p->threads[p->thread_id];
3526 struct instruction *ip = t->ip;
3529 __instr_alu_xor_hh_exec(p, t, ip);
3536 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
3538 struct thread *t = &p->threads[p->thread_id];
3539 struct instruction *ip = t->ip;
3542 __instr_alu_xor_i_exec(p, t, ip);
3549 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
3551 struct thread *t = &p->threads[p->thread_id];
3552 struct instruction *ip = t->ip;
3555 __instr_alu_ckadd_field_exec(p, t, ip);
3562 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
3564 struct thread *t = &p->threads[p->thread_id];
3565 struct instruction *ip = t->ip;
3568 __instr_alu_cksub_field_exec(p, t, ip);
3575 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
3577 struct thread *t = &p->threads[p->thread_id];
3578 struct instruction *ip = t->ip;
3581 __instr_alu_ckadd_struct20_exec(p, t, ip);
3588 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
3590 struct thread *t = &p->threads[p->thread_id];
3591 struct instruction *ip = t->ip;
3594 __instr_alu_ckadd_struct_exec(p, t, ip);
3603 static struct regarray *
3604 regarray_find(struct rte_swx_pipeline *p, const char *name);
3607 instr_regprefetch_translate(struct rte_swx_pipeline *p,
3608 struct action *action,
3611 struct instruction *instr,
3612 struct instruction_data *data __rte_unused)
3614 char *regarray = tokens[1], *idx = tokens[2];
3617 uint32_t idx_struct_id, idx_val;
3619 CHECK(n_tokens == 3, EINVAL);
3621 r = regarray_find(p, regarray);
3624 /* REGPREFETCH_RH, REGPREFETCH_RM. */
3625 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3627 CHECK(!fidx->var_size, EINVAL);
3629 instr->type = INSTR_REGPREFETCH_RM;
3631 instr->type = INSTR_REGPREFETCH_RH;
3633 instr->regarray.regarray_id = r->id;
3634 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3635 instr->regarray.idx.n_bits = fidx->n_bits;
3636 instr->regarray.idx.offset = fidx->offset / 8;
3637 instr->regarray.dstsrc_val = 0; /* Unused. */
3641 /* REGPREFETCH_RI. */
3642 idx_val = strtoul(idx, &idx, 0);
3643 CHECK(!idx[0], EINVAL);
3645 instr->type = INSTR_REGPREFETCH_RI;
3646 instr->regarray.regarray_id = r->id;
3647 instr->regarray.idx_val = idx_val;
3648 instr->regarray.dstsrc_val = 0; /* Unused. */
3653 instr_regrd_translate(struct rte_swx_pipeline *p,
3654 struct action *action,
3657 struct instruction *instr,
3658 struct instruction_data *data __rte_unused)
3660 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
3662 struct field *fdst, *fidx;
3663 uint32_t dst_struct_id, idx_struct_id, idx_val;
3665 CHECK(n_tokens == 4, EINVAL);
3667 r = regarray_find(p, regarray);
3670 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3671 CHECK(fdst, EINVAL);
3672 CHECK(!fdst->var_size, EINVAL);
3674 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
3675 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3677 CHECK(!fidx->var_size, EINVAL);
3679 instr->type = INSTR_REGRD_MRM;
3680 if (dst[0] == 'h' && idx[0] != 'h')
3681 instr->type = INSTR_REGRD_HRM;
3682 if (dst[0] != 'h' && idx[0] == 'h')
3683 instr->type = INSTR_REGRD_MRH;
3684 if (dst[0] == 'h' && idx[0] == 'h')
3685 instr->type = INSTR_REGRD_HRH;
3687 instr->regarray.regarray_id = r->id;
3688 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3689 instr->regarray.idx.n_bits = fidx->n_bits;
3690 instr->regarray.idx.offset = fidx->offset / 8;
3691 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3692 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3693 instr->regarray.dstsrc.offset = fdst->offset / 8;
3697 /* REGRD_MRI, REGRD_HRI. */
3698 idx_val = strtoul(idx, &idx, 0);
3699 CHECK(!idx[0], EINVAL);
3701 instr->type = INSTR_REGRD_MRI;
3703 instr->type = INSTR_REGRD_HRI;
3705 instr->regarray.regarray_id = r->id;
3706 instr->regarray.idx_val = idx_val;
3707 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
3708 instr->regarray.dstsrc.n_bits = fdst->n_bits;
3709 instr->regarray.dstsrc.offset = fdst->offset / 8;
3714 instr_regwr_translate(struct rte_swx_pipeline *p,
3715 struct action *action,
3718 struct instruction *instr,
3719 struct instruction_data *data __rte_unused)
3721 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3723 struct field *fidx, *fsrc;
3725 uint32_t idx_struct_id, idx_val, src_struct_id;
3727 CHECK(n_tokens == 4, EINVAL);
3729 r = regarray_find(p, regarray);
3732 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
3733 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3734 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3736 CHECK(!fidx->var_size, EINVAL);
3737 CHECK(!fsrc->var_size, EINVAL);
3739 instr->type = INSTR_REGWR_RMM;
3740 if (idx[0] == 'h' && src[0] != 'h')
3741 instr->type = INSTR_REGWR_RHM;
3742 if (idx[0] != 'h' && src[0] == 'h')
3743 instr->type = INSTR_REGWR_RMH;
3744 if (idx[0] == 'h' && src[0] == 'h')
3745 instr->type = INSTR_REGWR_RHH;
3747 instr->regarray.regarray_id = r->id;
3748 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3749 instr->regarray.idx.n_bits = fidx->n_bits;
3750 instr->regarray.idx.offset = fidx->offset / 8;
3751 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3752 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3753 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3757 /* REGWR_RHI, REGWR_RMI. */
3758 if (fidx && !fsrc) {
3759 CHECK(!fidx->var_size, EINVAL);
3761 src_val = strtoull(src, &src, 0);
3762 CHECK(!src[0], EINVAL);
3764 instr->type = INSTR_REGWR_RMI;
3766 instr->type = INSTR_REGWR_RHI;
3768 instr->regarray.regarray_id = r->id;
3769 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3770 instr->regarray.idx.n_bits = fidx->n_bits;
3771 instr->regarray.idx.offset = fidx->offset / 8;
3772 instr->regarray.dstsrc_val = src_val;
3776 /* REGWR_RIH, REGWR_RIM. */
3777 if (!fidx && fsrc) {
3778 idx_val = strtoul(idx, &idx, 0);
3779 CHECK(!idx[0], EINVAL);
3781 CHECK(!fsrc->var_size, EINVAL);
3783 instr->type = INSTR_REGWR_RIM;
3785 instr->type = INSTR_REGWR_RIH;
3787 instr->regarray.regarray_id = r->id;
3788 instr->regarray.idx_val = idx_val;
3789 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3790 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3791 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3796 src_val = strtoull(src, &src, 0);
3797 CHECK(!src[0], EINVAL);
3799 idx_val = strtoul(idx, &idx, 0);
3800 CHECK(!idx[0], EINVAL);
3802 instr->type = INSTR_REGWR_RII;
3803 instr->regarray.idx_val = idx_val;
3804 instr->regarray.dstsrc_val = src_val;
3810 instr_regadd_translate(struct rte_swx_pipeline *p,
3811 struct action *action,
3814 struct instruction *instr,
3815 struct instruction_data *data __rte_unused)
3817 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
3819 struct field *fidx, *fsrc;
3821 uint32_t idx_struct_id, idx_val, src_struct_id;
3823 CHECK(n_tokens == 4, EINVAL);
3825 r = regarray_find(p, regarray);
3828 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
3829 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
3830 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3832 CHECK(!fidx->var_size, EINVAL);
3833 CHECK(!fsrc->var_size, EINVAL);
3835 instr->type = INSTR_REGADD_RMM;
3836 if (idx[0] == 'h' && src[0] != 'h')
3837 instr->type = INSTR_REGADD_RHM;
3838 if (idx[0] != 'h' && src[0] == 'h')
3839 instr->type = INSTR_REGADD_RMH;
3840 if (idx[0] == 'h' && src[0] == 'h')
3841 instr->type = INSTR_REGADD_RHH;
3843 instr->regarray.regarray_id = r->id;
3844 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3845 instr->regarray.idx.n_bits = fidx->n_bits;
3846 instr->regarray.idx.offset = fidx->offset / 8;
3847 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3848 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3849 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3853 /* REGADD_RHI, REGADD_RMI. */
3854 if (fidx && !fsrc) {
3855 CHECK(!fidx->var_size, EINVAL);
3857 src_val = strtoull(src, &src, 0);
3858 CHECK(!src[0], EINVAL);
3860 instr->type = INSTR_REGADD_RMI;
3862 instr->type = INSTR_REGADD_RHI;
3864 instr->regarray.regarray_id = r->id;
3865 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
3866 instr->regarray.idx.n_bits = fidx->n_bits;
3867 instr->regarray.idx.offset = fidx->offset / 8;
3868 instr->regarray.dstsrc_val = src_val;
3872 /* REGADD_RIH, REGADD_RIM. */
3873 if (!fidx && fsrc) {
3874 idx_val = strtoul(idx, &idx, 0);
3875 CHECK(!idx[0], EINVAL);
3877 CHECK(!fsrc->var_size, EINVAL);
3879 instr->type = INSTR_REGADD_RIM;
3881 instr->type = INSTR_REGADD_RIH;
3883 instr->regarray.regarray_id = r->id;
3884 instr->regarray.idx_val = idx_val;
3885 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
3886 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
3887 instr->regarray.dstsrc.offset = fsrc->offset / 8;
3892 src_val = strtoull(src, &src, 0);
3893 CHECK(!src[0], EINVAL);
3895 idx_val = strtoul(idx, &idx, 0);
3896 CHECK(!idx[0], EINVAL);
3898 instr->type = INSTR_REGADD_RII;
3899 instr->regarray.idx_val = idx_val;
3900 instr->regarray.dstsrc_val = src_val;
3905 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
3907 struct thread *t = &p->threads[p->thread_id];
3908 struct instruction *ip = t->ip;
3911 __instr_regprefetch_rh_exec(p, t, ip);
3918 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
3920 struct thread *t = &p->threads[p->thread_id];
3921 struct instruction *ip = t->ip;
3924 __instr_regprefetch_rm_exec(p, t, ip);
3931 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
3933 struct thread *t = &p->threads[p->thread_id];
3934 struct instruction *ip = t->ip;
3937 __instr_regprefetch_ri_exec(p, t, ip);
3944 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
3946 struct thread *t = &p->threads[p->thread_id];
3947 struct instruction *ip = t->ip;
3950 __instr_regrd_hrh_exec(p, t, ip);
3957 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
3959 struct thread *t = &p->threads[p->thread_id];
3960 struct instruction *ip = t->ip;
3963 __instr_regrd_hrm_exec(p, t, ip);
3970 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
3972 struct thread *t = &p->threads[p->thread_id];
3973 struct instruction *ip = t->ip;
3976 __instr_regrd_mrh_exec(p, t, ip);
3983 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
3985 struct thread *t = &p->threads[p->thread_id];
3986 struct instruction *ip = t->ip;
3989 __instr_regrd_mrm_exec(p, t, ip);
3996 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
3998 struct thread *t = &p->threads[p->thread_id];
3999 struct instruction *ip = t->ip;
4002 __instr_regrd_hri_exec(p, t, ip);
4009 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
4011 struct thread *t = &p->threads[p->thread_id];
4012 struct instruction *ip = t->ip;
4015 __instr_regrd_mri_exec(p, t, ip);
4022 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
4024 struct thread *t = &p->threads[p->thread_id];
4025 struct instruction *ip = t->ip;
4028 __instr_regwr_rhh_exec(p, t, ip);
4035 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
4037 struct thread *t = &p->threads[p->thread_id];
4038 struct instruction *ip = t->ip;
4041 __instr_regwr_rhm_exec(p, t, ip);
4048 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
4050 struct thread *t = &p->threads[p->thread_id];
4051 struct instruction *ip = t->ip;
4054 __instr_regwr_rmh_exec(p, t, ip);
4061 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
4063 struct thread *t = &p->threads[p->thread_id];
4064 struct instruction *ip = t->ip;
4067 __instr_regwr_rmm_exec(p, t, ip);
4074 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
4076 struct thread *t = &p->threads[p->thread_id];
4077 struct instruction *ip = t->ip;
4080 __instr_regwr_rhi_exec(p, t, ip);
4087 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
4089 struct thread *t = &p->threads[p->thread_id];
4090 struct instruction *ip = t->ip;
4093 __instr_regwr_rmi_exec(p, t, ip);
4100 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
4102 struct thread *t = &p->threads[p->thread_id];
4103 struct instruction *ip = t->ip;
4106 __instr_regwr_rih_exec(p, t, ip);
4113 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
4115 struct thread *t = &p->threads[p->thread_id];
4116 struct instruction *ip = t->ip;
4119 __instr_regwr_rim_exec(p, t, ip);
4126 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
4128 struct thread *t = &p->threads[p->thread_id];
4129 struct instruction *ip = t->ip;
4132 __instr_regwr_rii_exec(p, t, ip);
4139 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
4141 struct thread *t = &p->threads[p->thread_id];
4142 struct instruction *ip = t->ip;
4145 __instr_regadd_rhh_exec(p, t, ip);
4152 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
4154 struct thread *t = &p->threads[p->thread_id];
4155 struct instruction *ip = t->ip;
4158 __instr_regadd_rhm_exec(p, t, ip);
4165 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
4167 struct thread *t = &p->threads[p->thread_id];
4168 struct instruction *ip = t->ip;
4171 __instr_regadd_rmh_exec(p, t, ip);
4178 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
4180 struct thread *t = &p->threads[p->thread_id];
4181 struct instruction *ip = t->ip;
4184 __instr_regadd_rmm_exec(p, t, ip);
4191 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
4193 struct thread *t = &p->threads[p->thread_id];
4194 struct instruction *ip = t->ip;
4197 __instr_regadd_rhi_exec(p, t, ip);
4204 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
4206 struct thread *t = &p->threads[p->thread_id];
4207 struct instruction *ip = t->ip;
4210 __instr_regadd_rmi_exec(p, t, ip);
4217 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
4219 struct thread *t = &p->threads[p->thread_id];
4220 struct instruction *ip = t->ip;
4223 __instr_regadd_rih_exec(p, t, ip);
4230 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
4232 struct thread *t = &p->threads[p->thread_id];
4233 struct instruction *ip = t->ip;
4236 __instr_regadd_rim_exec(p, t, ip);
4243 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
4245 struct thread *t = &p->threads[p->thread_id];
4246 struct instruction *ip = t->ip;
4249 __instr_regadd_rii_exec(p, t, ip);
4258 static struct metarray *
4259 metarray_find(struct rte_swx_pipeline *p, const char *name);
4262 instr_metprefetch_translate(struct rte_swx_pipeline *p,
4263 struct action *action,
4266 struct instruction *instr,
4267 struct instruction_data *data __rte_unused)
4269 char *metarray = tokens[1], *idx = tokens[2];
4272 uint32_t idx_struct_id, idx_val;
4274 CHECK(n_tokens == 3, EINVAL);
4276 m = metarray_find(p, metarray);
4279 /* METPREFETCH_H, METPREFETCH_M. */
4280 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4282 CHECK(!fidx->var_size, EINVAL);
4284 instr->type = INSTR_METPREFETCH_M;
4286 instr->type = INSTR_METPREFETCH_H;
4288 instr->meter.metarray_id = m->id;
4289 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4290 instr->meter.idx.n_bits = fidx->n_bits;
4291 instr->meter.idx.offset = fidx->offset / 8;
4295 /* METPREFETCH_I. */
4296 idx_val = strtoul(idx, &idx, 0);
4297 CHECK(!idx[0], EINVAL);
4299 instr->type = INSTR_METPREFETCH_I;
4300 instr->meter.metarray_id = m->id;
4301 instr->meter.idx_val = idx_val;
4306 instr_meter_translate(struct rte_swx_pipeline *p,
4307 struct action *action,
4310 struct instruction *instr,
4311 struct instruction_data *data __rte_unused)
4313 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
4314 char *color_in = tokens[4], *color_out = tokens[5];
4316 struct field *fidx, *flength, *fcin, *fcout;
4317 uint32_t idx_struct_id, length_struct_id;
4318 uint32_t color_in_struct_id, color_out_struct_id;
4320 CHECK(n_tokens == 6, EINVAL);
4322 m = metarray_find(p, metarray);
4325 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4327 flength = struct_field_parse(p, action, length, &length_struct_id);
4328 CHECK(flength, EINVAL);
4329 CHECK(!flength->var_size, EINVAL);
4331 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
4333 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
4334 CHECK(fcout, EINVAL);
4335 CHECK(!fcout->var_size, EINVAL);
4337 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
4339 CHECK(!fidx->var_size, EINVAL);
4340 CHECK(!fcin->var_size, EINVAL);
4342 instr->type = INSTR_METER_MMM;
4343 if (idx[0] == 'h' && length[0] == 'h')
4344 instr->type = INSTR_METER_HHM;
4345 if (idx[0] == 'h' && length[0] != 'h')
4346 instr->type = INSTR_METER_HMM;
4347 if (idx[0] != 'h' && length[0] == 'h')
4348 instr->type = INSTR_METER_MHM;
4350 instr->meter.metarray_id = m->id;
4352 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4353 instr->meter.idx.n_bits = fidx->n_bits;
4354 instr->meter.idx.offset = fidx->offset / 8;
4356 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4357 instr->meter.length.n_bits = flength->n_bits;
4358 instr->meter.length.offset = flength->offset / 8;
4360 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4361 instr->meter.color_in.n_bits = fcin->n_bits;
4362 instr->meter.color_in.offset = fcin->offset / 8;
4364 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4365 instr->meter.color_out.n_bits = fcout->n_bits;
4366 instr->meter.color_out.offset = fcout->offset / 8;
4371 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
4372 if (fidx && !fcin) {
4373 uint32_t color_in_val;
4375 CHECK(!fidx->var_size, EINVAL);
4377 color_in_val = strtoul(color_in, &color_in, 0);
4378 CHECK(!color_in[0], EINVAL);
4380 instr->type = INSTR_METER_MMI;
4381 if (idx[0] == 'h' && length[0] == 'h')
4382 instr->type = INSTR_METER_HHI;
4383 if (idx[0] == 'h' && length[0] != 'h')
4384 instr->type = INSTR_METER_HMI;
4385 if (idx[0] != 'h' && length[0] == 'h')
4386 instr->type = INSTR_METER_MHI;
4388 instr->meter.metarray_id = m->id;
4390 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
4391 instr->meter.idx.n_bits = fidx->n_bits;
4392 instr->meter.idx.offset = fidx->offset / 8;
4394 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4395 instr->meter.length.n_bits = flength->n_bits;
4396 instr->meter.length.offset = flength->offset / 8;
4398 instr->meter.color_in_val = color_in_val;
4400 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4401 instr->meter.color_out.n_bits = fcout->n_bits;
4402 instr->meter.color_out.offset = fcout->offset / 8;
4407 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
4408 if (!fidx && fcin) {
4411 idx_val = strtoul(idx, &idx, 0);
4412 CHECK(!idx[0], EINVAL);
4414 CHECK(!fcin->var_size, EINVAL);
4416 instr->type = INSTR_METER_IMM;
4417 if (length[0] == 'h')
4418 instr->type = INSTR_METER_IHM;
4420 instr->meter.metarray_id = m->id;
4422 instr->meter.idx_val = idx_val;
4424 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4425 instr->meter.length.n_bits = flength->n_bits;
4426 instr->meter.length.offset = flength->offset / 8;
4428 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
4429 instr->meter.color_in.n_bits = fcin->n_bits;
4430 instr->meter.color_in.offset = fcin->offset / 8;
4432 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4433 instr->meter.color_out.n_bits = fcout->n_bits;
4434 instr->meter.color_out.offset = fcout->offset / 8;
4439 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
4440 if (!fidx && !fcin) {
4441 uint32_t idx_val, color_in_val;
4443 idx_val = strtoul(idx, &idx, 0);
4444 CHECK(!idx[0], EINVAL);
4446 color_in_val = strtoul(color_in, &color_in, 0);
4447 CHECK(!color_in[0], EINVAL);
4449 instr->type = INSTR_METER_IMI;
4450 if (length[0] == 'h')
4451 instr->type = INSTR_METER_IHI;
4453 instr->meter.metarray_id = m->id;
4455 instr->meter.idx_val = idx_val;
4457 instr->meter.length.struct_id = (uint8_t)length_struct_id;
4458 instr->meter.length.n_bits = flength->n_bits;
4459 instr->meter.length.offset = flength->offset / 8;
4461 instr->meter.color_in_val = color_in_val;
4463 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
4464 instr->meter.color_out.n_bits = fcout->n_bits;
4465 instr->meter.color_out.offset = fcout->offset / 8;
4474 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
4476 struct thread *t = &p->threads[p->thread_id];
4477 struct instruction *ip = t->ip;
4480 __instr_metprefetch_h_exec(p, t, ip);
4487 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
4489 struct thread *t = &p->threads[p->thread_id];
4490 struct instruction *ip = t->ip;
4493 __instr_metprefetch_m_exec(p, t, ip);
4500 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
4502 struct thread *t = &p->threads[p->thread_id];
4503 struct instruction *ip = t->ip;
4506 __instr_metprefetch_i_exec(p, t, ip);
4513 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
4515 struct thread *t = &p->threads[p->thread_id];
4516 struct instruction *ip = t->ip;
4519 __instr_meter_hhm_exec(p, t, ip);
4526 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
4528 struct thread *t = &p->threads[p->thread_id];
4529 struct instruction *ip = t->ip;
4532 __instr_meter_hhi_exec(p, t, ip);
4539 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
4541 struct thread *t = &p->threads[p->thread_id];
4542 struct instruction *ip = t->ip;
4545 __instr_meter_hmm_exec(p, t, ip);
4552 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
4554 struct thread *t = &p->threads[p->thread_id];
4555 struct instruction *ip = t->ip;
4558 __instr_meter_hmi_exec(p, t, ip);
4565 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
4567 struct thread *t = &p->threads[p->thread_id];
4568 struct instruction *ip = t->ip;
4571 __instr_meter_mhm_exec(p, t, ip);
4578 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
4580 struct thread *t = &p->threads[p->thread_id];
4581 struct instruction *ip = t->ip;
4584 __instr_meter_mhi_exec(p, t, ip);
4591 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
4593 struct thread *t = &p->threads[p->thread_id];
4594 struct instruction *ip = t->ip;
4597 __instr_meter_mmm_exec(p, t, ip);
4604 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
4606 struct thread *t = &p->threads[p->thread_id];
4607 struct instruction *ip = t->ip;
4610 __instr_meter_mmi_exec(p, t, ip);
4617 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
4619 struct thread *t = &p->threads[p->thread_id];
4620 struct instruction *ip = t->ip;
4623 __instr_meter_ihm_exec(p, t, ip);
4630 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
4632 struct thread *t = &p->threads[p->thread_id];
4633 struct instruction *ip = t->ip;
4636 __instr_meter_ihi_exec(p, t, ip);
4643 instr_meter_imm_exec(struct rte_swx_pipeline *p)
4645 struct thread *t = &p->threads[p->thread_id];
4646 struct instruction *ip = t->ip;
4649 __instr_meter_imm_exec(p, t, ip);
4656 instr_meter_imi_exec(struct rte_swx_pipeline *p)
4658 struct thread *t = &p->threads[p->thread_id];
4659 struct instruction *ip = t->ip;
4662 __instr_meter_imi_exec(p, t, ip);
4672 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
4673 struct action *action __rte_unused,
4676 struct instruction *instr,
4677 struct instruction_data *data)
4679 CHECK(n_tokens == 2, EINVAL);
4681 strcpy(data->jmp_label, tokens[1]);
4683 instr->type = INSTR_JMP;
4684 instr->jmp.ip = NULL; /* Resolved later. */
4689 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
4690 struct action *action __rte_unused,
4693 struct instruction *instr,
4694 struct instruction_data *data)
4698 CHECK(n_tokens == 3, EINVAL);
4700 strcpy(data->jmp_label, tokens[1]);
4702 h = header_parse(p, tokens[2]);
4705 instr->type = INSTR_JMP_VALID;
4706 instr->jmp.ip = NULL; /* Resolved later. */
4707 instr->jmp.header_id = h->id;
4712 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
4713 struct action *action __rte_unused,
4716 struct instruction *instr,
4717 struct instruction_data *data)
4721 CHECK(n_tokens == 3, EINVAL);
4723 strcpy(data->jmp_label, tokens[1]);
4725 h = header_parse(p, tokens[2]);
4728 instr->type = INSTR_JMP_INVALID;
4729 instr->jmp.ip = NULL; /* Resolved later. */
4730 instr->jmp.header_id = h->id;
4735 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
4736 struct action *action,
4739 struct instruction *instr,
4740 struct instruction_data *data)
4742 CHECK(!action, EINVAL);
4743 CHECK(n_tokens == 2, EINVAL);
4745 strcpy(data->jmp_label, tokens[1]);
4747 instr->type = INSTR_JMP_HIT;
4748 instr->jmp.ip = NULL; /* Resolved later. */
4753 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
4754 struct action *action,
4757 struct instruction *instr,
4758 struct instruction_data *data)
4760 CHECK(!action, EINVAL);
4761 CHECK(n_tokens == 2, EINVAL);
4763 strcpy(data->jmp_label, tokens[1]);
4765 instr->type = INSTR_JMP_MISS;
4766 instr->jmp.ip = NULL; /* Resolved later. */
4771 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
4772 struct action *action,
4775 struct instruction *instr,
4776 struct instruction_data *data)
4780 CHECK(!action, EINVAL);
4781 CHECK(n_tokens == 3, EINVAL);
4783 strcpy(data->jmp_label, tokens[1]);
4785 a = action_find(p, tokens[2]);
4788 instr->type = INSTR_JMP_ACTION_HIT;
4789 instr->jmp.ip = NULL; /* Resolved later. */
4790 instr->jmp.action_id = a->id;
4795 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
4796 struct action *action,
4799 struct instruction *instr,
4800 struct instruction_data *data)
4804 CHECK(!action, EINVAL);
4805 CHECK(n_tokens == 3, EINVAL);
4807 strcpy(data->jmp_label, tokens[1]);
4809 a = action_find(p, tokens[2]);
4812 instr->type = INSTR_JMP_ACTION_MISS;
4813 instr->jmp.ip = NULL; /* Resolved later. */
4814 instr->jmp.action_id = a->id;
4819 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
4820 struct action *action,
4823 struct instruction *instr,
4824 struct instruction_data *data)
4826 char *a = tokens[2], *b = tokens[3];
4827 struct field *fa, *fb;
4829 uint32_t a_struct_id, b_struct_id;
4831 CHECK(n_tokens == 4, EINVAL);
4833 strcpy(data->jmp_label, tokens[1]);
4835 fa = struct_field_parse(p, action, a, &a_struct_id);
4837 CHECK(!fa->var_size, EINVAL);
4839 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
4840 fb = struct_field_parse(p, action, b, &b_struct_id);
4842 CHECK(!fb->var_size, EINVAL);
4844 instr->type = INSTR_JMP_EQ;
4845 if (a[0] != 'h' && b[0] == 'h')
4846 instr->type = INSTR_JMP_EQ_MH;
4847 if (a[0] == 'h' && b[0] != 'h')
4848 instr->type = INSTR_JMP_EQ_HM;
4849 if (a[0] == 'h' && b[0] == 'h')
4850 instr->type = INSTR_JMP_EQ_HH;
4851 instr->jmp.ip = NULL; /* Resolved later. */
4853 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4854 instr->jmp.a.n_bits = fa->n_bits;
4855 instr->jmp.a.offset = fa->offset / 8;
4856 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4857 instr->jmp.b.n_bits = fb->n_bits;
4858 instr->jmp.b.offset = fb->offset / 8;
4863 b_val = strtoull(b, &b, 0);
4864 CHECK(!b[0], EINVAL);
4867 b_val = hton64(b_val) >> (64 - fa->n_bits);
4869 instr->type = INSTR_JMP_EQ_I;
4870 instr->jmp.ip = NULL; /* Resolved later. */
4871 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4872 instr->jmp.a.n_bits = fa->n_bits;
4873 instr->jmp.a.offset = fa->offset / 8;
4874 instr->jmp.b_val = b_val;
4879 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
4880 struct action *action,
4883 struct instruction *instr,
4884 struct instruction_data *data)
4886 char *a = tokens[2], *b = tokens[3];
4887 struct field *fa, *fb;
4889 uint32_t a_struct_id, b_struct_id;
4891 CHECK(n_tokens == 4, EINVAL);
4893 strcpy(data->jmp_label, tokens[1]);
4895 fa = struct_field_parse(p, action, a, &a_struct_id);
4897 CHECK(!fa->var_size, EINVAL);
4899 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
4900 fb = struct_field_parse(p, action, b, &b_struct_id);
4902 CHECK(!fb->var_size, EINVAL);
4904 instr->type = INSTR_JMP_NEQ;
4905 if (a[0] != 'h' && b[0] == 'h')
4906 instr->type = INSTR_JMP_NEQ_MH;
4907 if (a[0] == 'h' && b[0] != 'h')
4908 instr->type = INSTR_JMP_NEQ_HM;
4909 if (a[0] == 'h' && b[0] == 'h')
4910 instr->type = INSTR_JMP_NEQ_HH;
4911 instr->jmp.ip = NULL; /* Resolved later. */
4913 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4914 instr->jmp.a.n_bits = fa->n_bits;
4915 instr->jmp.a.offset = fa->offset / 8;
4916 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4917 instr->jmp.b.n_bits = fb->n_bits;
4918 instr->jmp.b.offset = fb->offset / 8;
4923 b_val = strtoull(b, &b, 0);
4924 CHECK(!b[0], EINVAL);
4927 b_val = hton64(b_val) >> (64 - fa->n_bits);
4929 instr->type = INSTR_JMP_NEQ_I;
4930 instr->jmp.ip = NULL; /* Resolved later. */
4931 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4932 instr->jmp.a.n_bits = fa->n_bits;
4933 instr->jmp.a.offset = fa->offset / 8;
4934 instr->jmp.b_val = b_val;
4939 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
4940 struct action *action,
4943 struct instruction *instr,
4944 struct instruction_data *data)
4946 char *a = tokens[2], *b = tokens[3];
4947 struct field *fa, *fb;
4949 uint32_t a_struct_id, b_struct_id;
4951 CHECK(n_tokens == 4, EINVAL);
4953 strcpy(data->jmp_label, tokens[1]);
4955 fa = struct_field_parse(p, action, a, &a_struct_id);
4957 CHECK(!fa->var_size, EINVAL);
4959 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
4960 fb = struct_field_parse(p, action, b, &b_struct_id);
4962 CHECK(!fb->var_size, EINVAL);
4964 instr->type = INSTR_JMP_LT;
4965 if (a[0] == 'h' && b[0] != 'h')
4966 instr->type = INSTR_JMP_LT_HM;
4967 if (a[0] != 'h' && b[0] == 'h')
4968 instr->type = INSTR_JMP_LT_MH;
4969 if (a[0] == 'h' && b[0] == 'h')
4970 instr->type = INSTR_JMP_LT_HH;
4971 instr->jmp.ip = NULL; /* Resolved later. */
4973 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4974 instr->jmp.a.n_bits = fa->n_bits;
4975 instr->jmp.a.offset = fa->offset / 8;
4976 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
4977 instr->jmp.b.n_bits = fb->n_bits;
4978 instr->jmp.b.offset = fb->offset / 8;
4982 /* JMP_LT_MI, JMP_LT_HI. */
4983 b_val = strtoull(b, &b, 0);
4984 CHECK(!b[0], EINVAL);
4986 instr->type = INSTR_JMP_LT_MI;
4988 instr->type = INSTR_JMP_LT_HI;
4989 instr->jmp.ip = NULL; /* Resolved later. */
4991 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
4992 instr->jmp.a.n_bits = fa->n_bits;
4993 instr->jmp.a.offset = fa->offset / 8;
4994 instr->jmp.b_val = b_val;
4999 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
5000 struct action *action,
5003 struct instruction *instr,
5004 struct instruction_data *data)
5006 char *a = tokens[2], *b = tokens[3];
5007 struct field *fa, *fb;
5009 uint32_t a_struct_id, b_struct_id;
5011 CHECK(n_tokens == 4, EINVAL);
5013 strcpy(data->jmp_label, tokens[1]);
5015 fa = struct_field_parse(p, action, a, &a_struct_id);
5017 CHECK(!fa->var_size, EINVAL);
5019 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
5020 fb = struct_field_parse(p, action, b, &b_struct_id);
5022 CHECK(!fb->var_size, EINVAL);
5024 instr->type = INSTR_JMP_GT;
5025 if (a[0] == 'h' && b[0] != 'h')
5026 instr->type = INSTR_JMP_GT_HM;
5027 if (a[0] != 'h' && b[0] == 'h')
5028 instr->type = INSTR_JMP_GT_MH;
5029 if (a[0] == 'h' && b[0] == 'h')
5030 instr->type = INSTR_JMP_GT_HH;
5031 instr->jmp.ip = NULL; /* Resolved later. */
5033 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5034 instr->jmp.a.n_bits = fa->n_bits;
5035 instr->jmp.a.offset = fa->offset / 8;
5036 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
5037 instr->jmp.b.n_bits = fb->n_bits;
5038 instr->jmp.b.offset = fb->offset / 8;
5042 /* JMP_GT_MI, JMP_GT_HI. */
5043 b_val = strtoull(b, &b, 0);
5044 CHECK(!b[0], EINVAL);
5046 instr->type = INSTR_JMP_GT_MI;
5048 instr->type = INSTR_JMP_GT_HI;
5049 instr->jmp.ip = NULL; /* Resolved later. */
5051 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
5052 instr->jmp.a.n_bits = fa->n_bits;
5053 instr->jmp.a.offset = fa->offset / 8;
5054 instr->jmp.b_val = b_val;
5059 instr_jmp_exec(struct rte_swx_pipeline *p)
5061 struct thread *t = &p->threads[p->thread_id];
5062 struct instruction *ip = t->ip;
5064 TRACE("[Thread %2u] jmp\n", p->thread_id);
5066 thread_ip_set(t, ip->jmp.ip);
5070 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
5072 struct thread *t = &p->threads[p->thread_id];
5073 struct instruction *ip = t->ip;
5074 uint32_t header_id = ip->jmp.header_id;
5076 TRACE("[Thread %2u] jmpv\n", p->thread_id);
5078 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
5082 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
5084 struct thread *t = &p->threads[p->thread_id];
5085 struct instruction *ip = t->ip;
5086 uint32_t header_id = ip->jmp.header_id;
5088 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
5090 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
5094 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
5096 struct thread *t = &p->threads[p->thread_id];
5097 struct instruction *ip = t->ip;
5098 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
5100 TRACE("[Thread %2u] jmph\n", p->thread_id);
5102 t->ip = ip_next[t->hit];
5106 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
5108 struct thread *t = &p->threads[p->thread_id];
5109 struct instruction *ip = t->ip;
5110 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
5112 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
5114 t->ip = ip_next[t->hit];
5118 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
5120 struct thread *t = &p->threads[p->thread_id];
5121 struct instruction *ip = t->ip;
5123 TRACE("[Thread %2u] jmpa\n", p->thread_id);
5125 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
5129 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
5131 struct thread *t = &p->threads[p->thread_id];
5132 struct instruction *ip = t->ip;
5134 TRACE("[Thread %2u] jmpna\n", p->thread_id);
5136 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
5140 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
5142 struct thread *t = &p->threads[p->thread_id];
5143 struct instruction *ip = t->ip;
5145 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
5151 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
5153 struct thread *t = &p->threads[p->thread_id];
5154 struct instruction *ip = t->ip;
5156 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
5158 JMP_CMP_MH(t, ip, ==);
5162 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
5164 struct thread *t = &p->threads[p->thread_id];
5165 struct instruction *ip = t->ip;
5167 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
5169 JMP_CMP_HM(t, ip, ==);
5173 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
5175 struct thread *t = &p->threads[p->thread_id];
5176 struct instruction *ip = t->ip;
5178 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
5180 JMP_CMP_HH_FAST(t, ip, ==);
5184 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
5186 struct thread *t = &p->threads[p->thread_id];
5187 struct instruction *ip = t->ip;
5189 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
5191 JMP_CMP_I(t, ip, ==);
5195 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
5197 struct thread *t = &p->threads[p->thread_id];
5198 struct instruction *ip = t->ip;
5200 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
5206 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
5208 struct thread *t = &p->threads[p->thread_id];
5209 struct instruction *ip = t->ip;
5211 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
5213 JMP_CMP_MH(t, ip, !=);
5217 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
5219 struct thread *t = &p->threads[p->thread_id];
5220 struct instruction *ip = t->ip;
5222 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
5224 JMP_CMP_HM(t, ip, !=);
5228 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
5230 struct thread *t = &p->threads[p->thread_id];
5231 struct instruction *ip = t->ip;
5233 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
5235 JMP_CMP_HH_FAST(t, ip, !=);
5239 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
5241 struct thread *t = &p->threads[p->thread_id];
5242 struct instruction *ip = t->ip;
5244 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
5246 JMP_CMP_I(t, ip, !=);
5250 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
5252 struct thread *t = &p->threads[p->thread_id];
5253 struct instruction *ip = t->ip;
5255 TRACE("[Thread %2u] jmplt\n", p->thread_id);
5261 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
5263 struct thread *t = &p->threads[p->thread_id];
5264 struct instruction *ip = t->ip;
5266 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
5268 JMP_CMP_MH(t, ip, <);
5272 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
5274 struct thread *t = &p->threads[p->thread_id];
5275 struct instruction *ip = t->ip;
5277 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
5279 JMP_CMP_HM(t, ip, <);
5283 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
5285 struct thread *t = &p->threads[p->thread_id];
5286 struct instruction *ip = t->ip;
5288 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
5290 JMP_CMP_HH(t, ip, <);
5294 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
5296 struct thread *t = &p->threads[p->thread_id];
5297 struct instruction *ip = t->ip;
5299 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
5301 JMP_CMP_MI(t, ip, <);
5305 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
5307 struct thread *t = &p->threads[p->thread_id];
5308 struct instruction *ip = t->ip;
5310 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
5312 JMP_CMP_HI(t, ip, <);
5316 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
5318 struct thread *t = &p->threads[p->thread_id];
5319 struct instruction *ip = t->ip;
5321 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
5327 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
5329 struct thread *t = &p->threads[p->thread_id];
5330 struct instruction *ip = t->ip;
5332 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
5334 JMP_CMP_MH(t, ip, >);
5338 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
5340 struct thread *t = &p->threads[p->thread_id];
5341 struct instruction *ip = t->ip;
5343 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
5345 JMP_CMP_HM(t, ip, >);
5349 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
5351 struct thread *t = &p->threads[p->thread_id];
5352 struct instruction *ip = t->ip;
5354 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
5356 JMP_CMP_HH(t, ip, >);
5360 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
5362 struct thread *t = &p->threads[p->thread_id];
5363 struct instruction *ip = t->ip;
5365 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
5367 JMP_CMP_MI(t, ip, >);
5371 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
5373 struct thread *t = &p->threads[p->thread_id];
5374 struct instruction *ip = t->ip;
5376 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
5378 JMP_CMP_HI(t, ip, >);
5385 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
5386 struct action *action,
5387 char **tokens __rte_unused,
5389 struct instruction *instr,
5390 struct instruction_data *data __rte_unused)
5392 CHECK(action, EINVAL);
5393 CHECK(n_tokens == 1, EINVAL);
5395 instr->type = INSTR_RETURN;
5400 instr_return_exec(struct rte_swx_pipeline *p)
5402 struct thread *t = &p->threads[p->thread_id];
5404 TRACE("[Thread %2u] return\n", p->thread_id);
5410 instr_translate(struct rte_swx_pipeline *p,
5411 struct action *action,
5413 struct instruction *instr,
5414 struct instruction_data *data)
5416 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
5417 int n_tokens = 0, tpos = 0;
5419 /* Parse the instruction string into tokens. */
5423 token = strtok_r(string, " \t\v", &string);
5427 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
5428 CHECK_NAME(token, EINVAL);
5430 tokens[n_tokens] = token;
5434 CHECK(n_tokens, EINVAL);
5436 /* Handle the optional instruction label. */
5437 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
5438 strcpy(data->label, tokens[0]);
5441 CHECK(n_tokens - tpos, EINVAL);
5444 /* Identify the instruction type. */
5445 if (!strcmp(tokens[tpos], "rx"))
5446 return instr_rx_translate(p,
5453 if (!strcmp(tokens[tpos], "tx"))
5454 return instr_tx_translate(p,
5461 if (!strcmp(tokens[tpos], "drop"))
5462 return instr_drop_translate(p,
5469 if (!strcmp(tokens[tpos], "extract"))
5470 return instr_hdr_extract_translate(p,
5477 if (!strcmp(tokens[tpos], "lookahead"))
5478 return instr_hdr_lookahead_translate(p,
5485 if (!strcmp(tokens[tpos], "emit"))
5486 return instr_hdr_emit_translate(p,
5493 if (!strcmp(tokens[tpos], "validate"))
5494 return instr_hdr_validate_translate(p,
5501 if (!strcmp(tokens[tpos], "invalidate"))
5502 return instr_hdr_invalidate_translate(p,
5509 if (!strcmp(tokens[tpos], "mov"))
5510 return instr_mov_translate(p,
5517 if (!strcmp(tokens[tpos], "add"))
5518 return instr_alu_add_translate(p,
5525 if (!strcmp(tokens[tpos], "sub"))
5526 return instr_alu_sub_translate(p,
5533 if (!strcmp(tokens[tpos], "ckadd"))
5534 return instr_alu_ckadd_translate(p,
5541 if (!strcmp(tokens[tpos], "cksub"))
5542 return instr_alu_cksub_translate(p,
5549 if (!strcmp(tokens[tpos], "and"))
5550 return instr_alu_and_translate(p,
5557 if (!strcmp(tokens[tpos], "or"))
5558 return instr_alu_or_translate(p,
5565 if (!strcmp(tokens[tpos], "xor"))
5566 return instr_alu_xor_translate(p,
5573 if (!strcmp(tokens[tpos], "shl"))
5574 return instr_alu_shl_translate(p,
5581 if (!strcmp(tokens[tpos], "shr"))
5582 return instr_alu_shr_translate(p,
5589 if (!strcmp(tokens[tpos], "regprefetch"))
5590 return instr_regprefetch_translate(p,
5597 if (!strcmp(tokens[tpos], "regrd"))
5598 return instr_regrd_translate(p,
5605 if (!strcmp(tokens[tpos], "regwr"))
5606 return instr_regwr_translate(p,
5613 if (!strcmp(tokens[tpos], "regadd"))
5614 return instr_regadd_translate(p,
5621 if (!strcmp(tokens[tpos], "metprefetch"))
5622 return instr_metprefetch_translate(p,
5629 if (!strcmp(tokens[tpos], "meter"))
5630 return instr_meter_translate(p,
5637 if (!strcmp(tokens[tpos], "table"))
5638 return instr_table_translate(p,
5645 if (!strcmp(tokens[tpos], "learn"))
5646 return instr_learn_translate(p,
5653 if (!strcmp(tokens[tpos], "forget"))
5654 return instr_forget_translate(p,
5661 if (!strcmp(tokens[tpos], "extern"))
5662 return instr_extern_translate(p,
5669 if (!strcmp(tokens[tpos], "jmp"))
5670 return instr_jmp_translate(p,
5677 if (!strcmp(tokens[tpos], "jmpv"))
5678 return instr_jmp_valid_translate(p,
5685 if (!strcmp(tokens[tpos], "jmpnv"))
5686 return instr_jmp_invalid_translate(p,
5693 if (!strcmp(tokens[tpos], "jmph"))
5694 return instr_jmp_hit_translate(p,
5701 if (!strcmp(tokens[tpos], "jmpnh"))
5702 return instr_jmp_miss_translate(p,
5709 if (!strcmp(tokens[tpos], "jmpa"))
5710 return instr_jmp_action_hit_translate(p,
5717 if (!strcmp(tokens[tpos], "jmpna"))
5718 return instr_jmp_action_miss_translate(p,
5725 if (!strcmp(tokens[tpos], "jmpeq"))
5726 return instr_jmp_eq_translate(p,
5733 if (!strcmp(tokens[tpos], "jmpneq"))
5734 return instr_jmp_neq_translate(p,
5741 if (!strcmp(tokens[tpos], "jmplt"))
5742 return instr_jmp_lt_translate(p,
5749 if (!strcmp(tokens[tpos], "jmpgt"))
5750 return instr_jmp_gt_translate(p,
5757 if (!strcmp(tokens[tpos], "return"))
5758 return instr_return_translate(p,
5768 static struct instruction_data *
5769 label_find(struct instruction_data *data, uint32_t n, const char *label)
5773 for (i = 0; i < n; i++)
5774 if (!strcmp(label, data[i].label))
5781 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
5783 uint32_t count = 0, i;
5788 for (i = 0; i < n; i++)
5789 if (!strcmp(label, data[i].jmp_label))
5796 instr_label_check(struct instruction_data *instruction_data,
5797 uint32_t n_instructions)
5801 /* Check that all instruction labels are unique. */
5802 for (i = 0; i < n_instructions; i++) {
5803 struct instruction_data *data = &instruction_data[i];
5804 char *label = data->label;
5810 for (j = i + 1; j < n_instructions; j++)
5811 CHECK(strcmp(label, data[j].label), EINVAL);
5814 /* Get users for each instruction label. */
5815 for (i = 0; i < n_instructions; i++) {
5816 struct instruction_data *data = &instruction_data[i];
5817 char *label = data->label;
5819 data->n_users = label_is_used(instruction_data,
5828 instr_jmp_resolve(struct instruction *instructions,
5829 struct instruction_data *instruction_data,
5830 uint32_t n_instructions)
5834 for (i = 0; i < n_instructions; i++) {
5835 struct instruction *instr = &instructions[i];
5836 struct instruction_data *data = &instruction_data[i];
5837 struct instruction_data *found;
5839 if (!instruction_is_jmp(instr))
5842 found = label_find(instruction_data,
5845 CHECK(found, EINVAL);
5847 instr->jmp.ip = &instructions[found - instruction_data];
5854 instr_verify(struct rte_swx_pipeline *p __rte_unused,
5856 struct instruction *instr,
5857 struct instruction_data *data __rte_unused,
5858 uint32_t n_instructions)
5861 enum instruction_type type;
5864 /* Check that the first instruction is rx. */
5865 CHECK(instr[0].type == INSTR_RX, EINVAL);
5867 /* Check that there is at least one tx instruction. */
5868 for (i = 0; i < n_instructions; i++) {
5869 type = instr[i].type;
5871 if (instruction_is_tx(type))
5874 CHECK(i < n_instructions, EINVAL);
5876 /* Check that the last instruction is either tx or unconditional
5879 type = instr[n_instructions - 1].type;
5880 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
5884 enum instruction_type type;
5887 /* Check that there is at least one return or tx instruction. */
5888 for (i = 0; i < n_instructions; i++) {
5889 type = instr[i].type;
5891 if ((type == INSTR_RETURN) || instruction_is_tx(type))
5894 CHECK(i < n_instructions, EINVAL);
5901 instr_compact(struct instruction *instructions,
5902 struct instruction_data *instruction_data,
5903 uint32_t n_instructions)
5905 uint32_t i, pos = 0;
5907 /* Eliminate the invalid instructions that have been optimized out. */
5908 for (i = 0; i < n_instructions; i++) {
5909 struct instruction *instr = &instructions[i];
5910 struct instruction_data *data = &instruction_data[i];
5916 memcpy(&instructions[pos], instr, sizeof(*instr));
5917 memcpy(&instruction_data[pos], data, sizeof(*data));
5927 instr_pattern_extract_many_search(struct instruction *instr,
5928 struct instruction_data *data,
5930 uint32_t *n_pattern_instr)
5934 for (i = 0; i < n_instr; i++) {
5935 if (data[i].invalid)
5938 if (instr[i].type != INSTR_HDR_EXTRACT)
5941 if (i == RTE_DIM(instr->io.hdr.header_id))
5944 if (i && data[i].n_users)
5951 *n_pattern_instr = i;
5956 instr_pattern_extract_many_replace(struct instruction *instr,
5957 struct instruction_data *data,
5962 for (i = 1; i < n_instr; i++) {
5964 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
5965 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
5966 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
5968 data[i].invalid = 1;
5973 instr_pattern_extract_many_optimize(struct instruction *instructions,
5974 struct instruction_data *instruction_data,
5975 uint32_t n_instructions)
5979 for (i = 0; i < n_instructions; ) {
5980 struct instruction *instr = &instructions[i];
5981 struct instruction_data *data = &instruction_data[i];
5982 uint32_t n_instr = 0;
5986 detected = instr_pattern_extract_many_search(instr,
5991 instr_pattern_extract_many_replace(instr,
5998 /* No pattern starting at the current instruction. */
6002 /* Eliminate the invalid instructions that have been optimized out. */
6003 n_instructions = instr_compact(instructions,
6007 return n_instructions;
6011 instr_pattern_emit_many_tx_search(struct instruction *instr,
6012 struct instruction_data *data,
6014 uint32_t *n_pattern_instr)
6018 for (i = 0; i < n_instr; i++) {
6019 if (data[i].invalid)
6022 if (instr[i].type != INSTR_HDR_EMIT)
6025 if (i == RTE_DIM(instr->io.hdr.header_id))
6028 if (i && data[i].n_users)
6035 if (!instruction_is_tx(instr[i].type))
6038 if (data[i].n_users)
6043 *n_pattern_instr = i;
6048 instr_pattern_emit_many_tx_replace(struct instruction *instr,
6049 struct instruction_data *data,
6054 /* Any emit instruction in addition to the first one. */
6055 for (i = 1; i < n_instr - 1; i++) {
6057 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
6058 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
6059 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
6061 data[i].invalid = 1;
6064 /* The TX instruction is the last one in the pattern. */
6066 instr[0].io.io.offset = instr[i].io.io.offset;
6067 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
6068 data[i].invalid = 1;
6072 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
6073 struct instruction_data *instruction_data,
6074 uint32_t n_instructions)
6078 for (i = 0; i < n_instructions; ) {
6079 struct instruction *instr = &instructions[i];
6080 struct instruction_data *data = &instruction_data[i];
6081 uint32_t n_instr = 0;
6084 /* Emit many + TX. */
6085 detected = instr_pattern_emit_many_tx_search(instr,
6090 instr_pattern_emit_many_tx_replace(instr,
6097 /* No pattern starting at the current instruction. */
6101 /* Eliminate the invalid instructions that have been optimized out. */
6102 n_instructions = instr_compact(instructions,
6106 return n_instructions;
6110 action_arg_src_mov_count(struct action *a,
6112 struct instruction *instructions,
6113 struct instruction_data *instruction_data,
6114 uint32_t n_instructions);
6117 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
6119 struct instruction *instr,
6120 struct instruction_data *data,
6122 struct instruction *instructions,
6123 struct instruction_data *instruction_data,
6124 uint32_t n_instructions,
6125 uint32_t *n_pattern_instr)
6128 uint32_t src_field_id, i, j;
6130 /* Prerequisites. */
6134 /* First instruction: MOV_HM. */
6135 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
6138 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6139 if (!h || h->st->var_size)
6142 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6143 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6146 if (src_field_id == a->st->n_fields)
6149 if (instr[0].mov.dst.offset ||
6150 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
6151 instr[0].mov.src.struct_id ||
6152 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
6153 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
6156 if ((n_instr < h->st->n_fields + 1) ||
6157 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
6160 /* Subsequent instructions: MOV_HM. */
6161 for (i = 1; i < h->st->n_fields; i++)
6162 if (data[i].invalid ||
6164 (instr[i].type != INSTR_MOV_HM) ||
6165 (instr[i].mov.dst.struct_id != h->struct_id) ||
6166 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
6167 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
6168 instr[i].mov.src.struct_id ||
6169 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
6170 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
6171 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
6174 /* Last instruction: HDR_VALIDATE. */
6175 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
6176 (instr[i].valid.header_id != h->id))
6179 /* Check that none of the action args that are used as source for this
6180 * DMA transfer are not used as source in any other mov instruction.
6182 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
6185 n_users = action_arg_src_mov_count(a,
6194 *n_pattern_instr = 1 + i;
6199 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
6201 struct instruction *instr,
6202 struct instruction_data *data,
6206 uint32_t src_field_id, src_offset, i;
6208 /* Read from the instructions before they are modified. */
6209 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
6213 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
6214 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
6217 if (src_field_id == a->st->n_fields)
6220 src_offset = instr[0].mov.src.offset;
6222 /* Modify the instructions. */
6223 instr[0].type = INSTR_DMA_HT;
6224 instr[0].dma.dst.header_id[0] = h->id;
6225 instr[0].dma.dst.struct_id[0] = h->struct_id;
6226 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
6227 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
6229 for (i = 1; i < n_instr; i++)
6230 data[i].invalid = 1;
6232 /* Update the endianness of the action arguments to header endianness. */
6233 for (i = 0; i < h->st->n_fields; i++)
6234 a->args_endianness[src_field_id + i] = 1;
6238 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
6240 struct instruction *instructions,
6241 struct instruction_data *instruction_data,
6242 uint32_t n_instructions)
6247 return n_instructions;
6249 for (i = 0; i < n_instructions; ) {
6250 struct instruction *instr = &instructions[i];
6251 struct instruction_data *data = &instruction_data[i];
6252 uint32_t n_instr = 0;
6255 /* Mov all + validate. */
6256 detected = instr_pattern_mov_all_validate_search(p,
6266 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
6271 /* No pattern starting at the current instruction. */
6275 /* Eliminate the invalid instructions that have been optimized out. */
6276 n_instructions = instr_compact(instructions,
6280 return n_instructions;
6284 instr_pattern_dma_many_search(struct instruction *instr,
6285 struct instruction_data *data,
6287 uint32_t *n_pattern_instr)
6291 for (i = 0; i < n_instr; i++) {
6292 if (data[i].invalid)
6295 if (instr[i].type != INSTR_DMA_HT)
6298 if (i == RTE_DIM(instr->dma.dst.header_id))
6301 if (i && data[i].n_users)
6308 *n_pattern_instr = i;
6313 instr_pattern_dma_many_replace(struct instruction *instr,
6314 struct instruction_data *data,
6319 for (i = 1; i < n_instr; i++) {
6321 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
6322 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
6323 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
6324 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
6326 data[i].invalid = 1;
6331 instr_pattern_dma_many_optimize(struct instruction *instructions,
6332 struct instruction_data *instruction_data,
6333 uint32_t n_instructions)
6337 for (i = 0; i < n_instructions; ) {
6338 struct instruction *instr = &instructions[i];
6339 struct instruction_data *data = &instruction_data[i];
6340 uint32_t n_instr = 0;
6344 detected = instr_pattern_dma_many_search(instr,
6349 instr_pattern_dma_many_replace(instr, data, n_instr);
6354 /* No pattern starting at the current instruction. */
6358 /* Eliminate the invalid instructions that have been optimized out. */
6359 n_instructions = instr_compact(instructions,
6363 return n_instructions;
6367 instr_optimize(struct rte_swx_pipeline *p,
6369 struct instruction *instructions,
6370 struct instruction_data *instruction_data,
6371 uint32_t n_instructions)
6374 n_instructions = instr_pattern_extract_many_optimize(instructions,
6378 /* Emit many + TX. */
6379 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
6383 /* Mov all + validate. */
6384 n_instructions = instr_pattern_mov_all_validate_optimize(p,
6391 n_instructions = instr_pattern_dma_many_optimize(instructions,
6395 return n_instructions;
6399 instruction_config(struct rte_swx_pipeline *p,
6401 const char **instructions,
6402 uint32_t n_instructions)
6404 struct instruction *instr = NULL;
6405 struct instruction_data *data = NULL;
6409 CHECK(n_instructions, EINVAL);
6410 CHECK(instructions, EINVAL);
6411 for (i = 0; i < n_instructions; i++)
6412 CHECK_INSTRUCTION(instructions[i], EINVAL);
6414 /* Memory allocation. */
6415 instr = calloc(n_instructions, sizeof(struct instruction));
6421 data = calloc(n_instructions, sizeof(struct instruction_data));
6427 for (i = 0; i < n_instructions; i++) {
6428 char *string = strdup(instructions[i]);
6434 err = instr_translate(p, a, string, &instr[i], &data[i]);
6443 err = instr_label_check(data, n_instructions);
6447 err = instr_verify(p, a, instr, data, n_instructions);
6451 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
6453 err = instr_jmp_resolve(instr, data, n_instructions);
6458 a->instructions = instr;
6459 a->n_instructions = n_instructions;
6461 p->instructions = instr;
6462 p->n_instructions = n_instructions;
6474 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
6476 static instr_exec_t instruction_table[] = {
6477 [INSTR_RX] = instr_rx_exec,
6478 [INSTR_TX] = instr_tx_exec,
6479 [INSTR_TX_I] = instr_tx_i_exec,
6481 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
6482 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
6483 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
6484 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
6485 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
6486 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
6487 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
6488 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
6489 [INSTR_HDR_EXTRACT_M] = instr_hdr_extract_m_exec,
6490 [INSTR_HDR_LOOKAHEAD] = instr_hdr_lookahead_exec,
6492 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
6493 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
6494 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
6495 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
6496 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
6497 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
6498 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
6499 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
6500 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
6502 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
6503 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
6505 [INSTR_MOV] = instr_mov_exec,
6506 [INSTR_MOV_MH] = instr_mov_mh_exec,
6507 [INSTR_MOV_HM] = instr_mov_hm_exec,
6508 [INSTR_MOV_HH] = instr_mov_hh_exec,
6509 [INSTR_MOV_I] = instr_mov_i_exec,
6511 [INSTR_DMA_HT] = instr_dma_ht_exec,
6512 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
6513 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
6514 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
6515 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
6516 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
6517 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
6518 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
6520 [INSTR_ALU_ADD] = instr_alu_add_exec,
6521 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
6522 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
6523 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
6524 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
6525 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
6527 [INSTR_ALU_SUB] = instr_alu_sub_exec,
6528 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
6529 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
6530 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
6531 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
6532 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
6534 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
6535 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
6536 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
6537 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
6539 [INSTR_ALU_AND] = instr_alu_and_exec,
6540 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
6541 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
6542 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
6543 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
6545 [INSTR_ALU_OR] = instr_alu_or_exec,
6546 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
6547 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
6548 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
6549 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
6551 [INSTR_ALU_XOR] = instr_alu_xor_exec,
6552 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
6553 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
6554 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
6555 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
6557 [INSTR_ALU_SHL] = instr_alu_shl_exec,
6558 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
6559 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
6560 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
6561 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
6562 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
6564 [INSTR_ALU_SHR] = instr_alu_shr_exec,
6565 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
6566 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
6567 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
6568 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
6569 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
6571 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
6572 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
6573 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
6575 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
6576 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
6577 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
6578 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
6579 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
6580 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
6582 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
6583 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
6584 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
6585 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
6586 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
6587 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
6588 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
6589 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
6590 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
6592 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
6593 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
6594 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
6595 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
6596 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
6597 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
6598 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
6599 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
6600 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
6602 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
6603 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
6604 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
6606 [INSTR_METER_HHM] = instr_meter_hhm_exec,
6607 [INSTR_METER_HHI] = instr_meter_hhi_exec,
6608 [INSTR_METER_HMM] = instr_meter_hmm_exec,
6609 [INSTR_METER_HMI] = instr_meter_hmi_exec,
6610 [INSTR_METER_MHM] = instr_meter_mhm_exec,
6611 [INSTR_METER_MHI] = instr_meter_mhi_exec,
6612 [INSTR_METER_MMM] = instr_meter_mmm_exec,
6613 [INSTR_METER_MMI] = instr_meter_mmi_exec,
6614 [INSTR_METER_IHM] = instr_meter_ihm_exec,
6615 [INSTR_METER_IHI] = instr_meter_ihi_exec,
6616 [INSTR_METER_IMM] = instr_meter_imm_exec,
6617 [INSTR_METER_IMI] = instr_meter_imi_exec,
6619 [INSTR_TABLE] = instr_table_exec,
6620 [INSTR_SELECTOR] = instr_selector_exec,
6621 [INSTR_LEARNER] = instr_learner_exec,
6622 [INSTR_LEARNER_LEARN] = instr_learn_exec,
6623 [INSTR_LEARNER_FORGET] = instr_forget_exec,
6624 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
6625 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
6627 [INSTR_JMP] = instr_jmp_exec,
6628 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
6629 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
6630 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
6631 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
6632 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
6633 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
6635 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
6636 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
6637 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
6638 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
6639 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
6641 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
6642 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
6643 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
6644 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
6645 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
6647 [INSTR_JMP_LT] = instr_jmp_lt_exec,
6648 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
6649 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
6650 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
6651 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
6652 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
6654 [INSTR_JMP_GT] = instr_jmp_gt_exec,
6655 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
6656 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
6657 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
6658 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
6659 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
6661 [INSTR_RETURN] = instr_return_exec,
6665 instr_exec(struct rte_swx_pipeline *p)
6667 struct thread *t = &p->threads[p->thread_id];
6668 struct instruction *ip = t->ip;
6669 instr_exec_t instr = instruction_table[ip->type];
6677 static struct action *
6678 action_find(struct rte_swx_pipeline *p, const char *name)
6680 struct action *elem;
6685 TAILQ_FOREACH(elem, &p->actions, node)
6686 if (strcmp(elem->name, name) == 0)
6692 static struct action *
6693 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6695 struct action *action = NULL;
6697 TAILQ_FOREACH(action, &p->actions, node)
6698 if (action->id == id)
6704 static struct field *
6705 action_field_find(struct action *a, const char *name)
6707 return a->st ? struct_type_field_find(a->st, name) : NULL;
6710 static struct field *
6711 action_field_parse(struct action *action, const char *name)
6713 if (name[0] != 't' || name[1] != '.')
6716 return action_field_find(action, &name[2]);
6720 action_has_nbo_args(struct action *a)
6724 /* Return if the action does not have any args. */
6726 return 0; /* FALSE */
6728 for (i = 0; i < a->st->n_fields; i++)
6729 if (a->args_endianness[i])
6730 return 1; /* TRUE */
6732 return 0; /* FALSE */
6736 action_does_learning(struct action *a)
6740 for (i = 0; i < a->n_instructions; i++)
6741 switch (a->instructions[i].type) {
6742 case INSTR_LEARNER_LEARN:
6743 return 1; /* TRUE */
6745 case INSTR_LEARNER_FORGET:
6746 return 1; /* TRUE */
6752 return 0; /* FALSE */
6756 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
6758 const char *args_struct_type_name,
6759 const char **instructions,
6760 uint32_t n_instructions)
6762 struct struct_type *args_struct_type = NULL;
6768 CHECK_NAME(name, EINVAL);
6769 CHECK(!action_find(p, name), EEXIST);
6771 if (args_struct_type_name) {
6772 CHECK_NAME(args_struct_type_name, EINVAL);
6773 args_struct_type = struct_type_find(p, args_struct_type_name);
6774 CHECK(args_struct_type, EINVAL);
6775 CHECK(!args_struct_type->var_size, EINVAL);
6778 /* Node allocation. */
6779 a = calloc(1, sizeof(struct action));
6781 if (args_struct_type) {
6782 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
6783 if (!a->args_endianness) {
6789 /* Node initialization. */
6790 strcpy(a->name, name);
6791 a->st = args_struct_type;
6792 a->id = p->n_actions;
6794 /* Instruction translation. */
6795 err = instruction_config(p, a, instructions, n_instructions);
6797 free(a->args_endianness);
6802 /* Node add to tailq. */
6803 TAILQ_INSERT_TAIL(&p->actions, a, node);
6810 action_build(struct rte_swx_pipeline *p)
6812 struct action *action;
6814 p->action_instructions = calloc(p->n_actions,
6815 sizeof(struct instruction *));
6816 CHECK(p->action_instructions, ENOMEM);
6818 TAILQ_FOREACH(action, &p->actions, node)
6819 p->action_instructions[action->id] = action->instructions;
6825 action_build_free(struct rte_swx_pipeline *p)
6827 free(p->action_instructions);
6828 p->action_instructions = NULL;
6832 action_free(struct rte_swx_pipeline *p)
6834 action_build_free(p);
6837 struct action *action;
6839 action = TAILQ_FIRST(&p->actions);
6843 TAILQ_REMOVE(&p->actions, action, node);
6844 free(action->instructions);
6850 action_arg_src_mov_count(struct action *a,
6852 struct instruction *instructions,
6853 struct instruction_data *instruction_data,
6854 uint32_t n_instructions)
6856 uint32_t offset, n_users = 0, i;
6859 (arg_id >= a->st->n_fields) ||
6861 !instruction_data ||
6865 offset = a->st->fields[arg_id].offset / 8;
6867 for (i = 0; i < n_instructions; i++) {
6868 struct instruction *instr = &instructions[i];
6869 struct instruction_data *data = &instruction_data[i];
6871 if (data->invalid ||
6872 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
6873 instr->mov.src.struct_id ||
6874 (instr->mov.src.offset != offset))
6886 static struct table_type *
6887 table_type_find(struct rte_swx_pipeline *p, const char *name)
6889 struct table_type *elem;
6891 TAILQ_FOREACH(elem, &p->table_types, node)
6892 if (strcmp(elem->name, name) == 0)
6898 static struct table_type *
6899 table_type_resolve(struct rte_swx_pipeline *p,
6900 const char *recommended_type_name,
6901 enum rte_swx_table_match_type match_type)
6903 struct table_type *elem;
6905 /* Only consider the recommended type if the match type is correct. */
6906 if (recommended_type_name)
6907 TAILQ_FOREACH(elem, &p->table_types, node)
6908 if (!strcmp(elem->name, recommended_type_name) &&
6909 (elem->match_type == match_type))
6912 /* Ignore the recommended type and get the first element with this match
6915 TAILQ_FOREACH(elem, &p->table_types, node)
6916 if (elem->match_type == match_type)
6922 static struct table *
6923 table_find(struct rte_swx_pipeline *p, const char *name)
6927 TAILQ_FOREACH(elem, &p->tables, node)
6928 if (strcmp(elem->name, name) == 0)
6934 static struct table *
6935 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
6937 struct table *table = NULL;
6939 TAILQ_FOREACH(table, &p->tables, node)
6940 if (table->id == id)
6947 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
6949 enum rte_swx_table_match_type match_type,
6950 struct rte_swx_table_ops *ops)
6952 struct table_type *elem;
6956 CHECK_NAME(name, EINVAL);
6957 CHECK(!table_type_find(p, name), EEXIST);
6960 CHECK(ops->create, EINVAL);
6961 CHECK(ops->lkp, EINVAL);
6962 CHECK(ops->free, EINVAL);
6964 /* Node allocation. */
6965 elem = calloc(1, sizeof(struct table_type));
6966 CHECK(elem, ENOMEM);
6968 /* Node initialization. */
6969 strcpy(elem->name, name);
6970 elem->match_type = match_type;
6971 memcpy(&elem->ops, ops, sizeof(*ops));
6973 /* Node add to tailq. */
6974 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
6980 table_match_type_resolve(struct rte_swx_match_field_params *fields,
6982 enum rte_swx_table_match_type *match_type)
6984 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
6986 for (i = 0; i < n_fields; i++) {
6987 struct rte_swx_match_field_params *f = &fields[i];
6989 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
6992 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
6996 if ((n_fields_lpm > 1) ||
6997 (n_fields_lpm && (n_fields_em != n_fields - 1)))
7000 *match_type = (n_fields_em == n_fields) ?
7001 RTE_SWX_TABLE_MATCH_EXACT :
7002 RTE_SWX_TABLE_MATCH_WILDCARD;
7008 table_match_fields_check(struct rte_swx_pipeline *p,
7009 struct rte_swx_pipeline_table_params *params,
7010 struct header **header)
7012 struct header *h0 = NULL;
7013 struct field *hf, *mf;
7014 uint32_t *offset = NULL, i;
7017 /* Return if no match fields. */
7018 if (!params->n_fields) {
7019 if (params->fields) {
7030 /* Memory allocation. */
7031 offset = calloc(params->n_fields, sizeof(uint32_t));
7037 /* Check that all the match fields belong to either the same header or
7040 hf = header_field_parse(p, params->fields[0].name, &h0);
7041 mf = metadata_field_parse(p, params->fields[0].name);
7042 if ((!hf && !mf) || (hf && hf->var_size)) {
7047 offset[0] = h0 ? hf->offset : mf->offset;
7049 for (i = 1; i < params->n_fields; i++)
7053 hf = header_field_parse(p, params->fields[i].name, &h);
7054 if (!hf || (h->id != h0->id) || hf->var_size) {
7059 offset[i] = hf->offset;
7061 mf = metadata_field_parse(p, params->fields[i].name);
7067 offset[i] = mf->offset;
7070 /* Check that there are no duplicated match fields. */
7071 for (i = 0; i < params->n_fields; i++) {
7074 for (j = 0; j < i; j++)
7075 if (offset[j] == offset[i]) {
7091 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
7093 struct rte_swx_pipeline_table_params *params,
7094 const char *recommended_table_type_name,
7098 struct table_type *type;
7100 struct action *default_action;
7101 struct header *header = NULL;
7102 uint32_t action_data_size_max = 0, i;
7107 CHECK_NAME(name, EINVAL);
7108 CHECK(!table_find(p, name), EEXIST);
7109 CHECK(!selector_find(p, name), EEXIST);
7110 CHECK(!learner_find(p, name), EEXIST);
7112 CHECK(params, EINVAL);
7115 status = table_match_fields_check(p, params, &header);
7119 /* Action checks. */
7120 CHECK(params->n_actions, EINVAL);
7121 CHECK(params->action_names, EINVAL);
7122 for (i = 0; i < params->n_actions; i++) {
7123 const char *action_name = params->action_names[i];
7125 uint32_t action_data_size;
7127 CHECK_NAME(action_name, EINVAL);
7129 a = action_find(p, action_name);
7131 CHECK(!action_does_learning(a), EINVAL);
7133 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7134 if (action_data_size > action_data_size_max)
7135 action_data_size_max = action_data_size;
7138 CHECK_NAME(params->default_action_name, EINVAL);
7139 for (i = 0; i < p->n_actions; i++)
7140 if (!strcmp(params->action_names[i],
7141 params->default_action_name))
7143 CHECK(i < params->n_actions, EINVAL);
7144 default_action = action_find(p, params->default_action_name);
7145 CHECK((default_action->st && params->default_action_data) ||
7146 !params->default_action_data, EINVAL);
7148 /* Table type checks. */
7149 if (recommended_table_type_name)
7150 CHECK_NAME(recommended_table_type_name, EINVAL);
7152 if (params->n_fields) {
7153 enum rte_swx_table_match_type match_type;
7155 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
7159 type = table_type_resolve(p, recommended_table_type_name, match_type);
7160 CHECK(type, EINVAL);
7165 /* Memory allocation. */
7166 t = calloc(1, sizeof(struct table));
7169 t->fields = calloc(params->n_fields, sizeof(struct match_field));
7175 t->actions = calloc(params->n_actions, sizeof(struct action *));
7182 if (action_data_size_max) {
7183 t->default_action_data = calloc(1, action_data_size_max);
7184 if (!t->default_action_data) {
7192 /* Node initialization. */
7193 strcpy(t->name, name);
7194 if (args && args[0])
7195 strcpy(t->args, args);
7198 for (i = 0; i < params->n_fields; i++) {
7199 struct rte_swx_match_field_params *field = ¶ms->fields[i];
7200 struct match_field *f = &t->fields[i];
7202 f->match_type = field->match_type;
7204 header_field_parse(p, field->name, NULL) :
7205 metadata_field_parse(p, field->name);
7207 t->n_fields = params->n_fields;
7210 for (i = 0; i < params->n_actions; i++)
7211 t->actions[i] = action_find(p, params->action_names[i]);
7212 t->default_action = default_action;
7213 if (default_action->st)
7214 memcpy(t->default_action_data,
7215 params->default_action_data,
7216 default_action->st->n_bits / 8);
7217 t->n_actions = params->n_actions;
7218 t->default_action_is_const = params->default_action_is_const;
7219 t->action_data_size_max = action_data_size_max;
7222 t->id = p->n_tables;
7224 /* Node add to tailq. */
7225 TAILQ_INSERT_TAIL(&p->tables, t, node);
7231 static struct rte_swx_table_params *
7232 table_params_get(struct table *table)
7234 struct rte_swx_table_params *params;
7235 struct field *first, *last;
7237 uint32_t key_size, key_offset, action_data_size, i;
7239 /* Memory allocation. */
7240 params = calloc(1, sizeof(struct rte_swx_table_params));
7244 /* Find first (smallest offset) and last (biggest offset) match fields. */
7245 first = table->fields[0].field;
7246 last = table->fields[0].field;
7248 for (i = 0; i < table->n_fields; i++) {
7249 struct field *f = table->fields[i].field;
7251 if (f->offset < first->offset)
7254 if (f->offset > last->offset)
7258 /* Key offset and size. */
7259 key_offset = first->offset / 8;
7260 key_size = (last->offset + last->n_bits - first->offset) / 8;
7262 /* Memory allocation. */
7263 key_mask = calloc(1, key_size);
7270 for (i = 0; i < table->n_fields; i++) {
7271 struct field *f = table->fields[i].field;
7272 uint32_t start = (f->offset - first->offset) / 8;
7273 size_t size = f->n_bits / 8;
7275 memset(&key_mask[start], 0xFF, size);
7278 /* Action data size. */
7279 action_data_size = 0;
7280 for (i = 0; i < table->n_actions; i++) {
7281 struct action *action = table->actions[i];
7282 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
7284 if (ads > action_data_size)
7285 action_data_size = ads;
7289 params->match_type = table->type->match_type;
7290 params->key_size = key_size;
7291 params->key_offset = key_offset;
7292 params->key_mask0 = key_mask;
7293 params->action_data_size = action_data_size;
7294 params->n_keys_max = table->size;
7300 table_params_free(struct rte_swx_table_params *params)
7305 free(params->key_mask0);
7310 table_stub_lkp(void *table __rte_unused,
7311 void *mailbox __rte_unused,
7312 uint8_t **key __rte_unused,
7313 uint64_t *action_id __rte_unused,
7314 uint8_t **action_data __rte_unused,
7318 return 1; /* DONE. */
7322 table_build(struct rte_swx_pipeline *p)
7326 /* Per pipeline: table statistics. */
7327 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
7328 CHECK(p->table_stats, ENOMEM);
7330 for (i = 0; i < p->n_tables; i++) {
7331 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
7332 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
7335 /* Per thread: table runt-time. */
7336 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7337 struct thread *t = &p->threads[i];
7338 struct table *table;
7340 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
7341 CHECK(t->tables, ENOMEM);
7343 TAILQ_FOREACH(table, &p->tables, node) {
7344 struct table_runtime *r = &t->tables[table->id];
7349 size = table->type->ops.mailbox_size_get();
7352 r->func = table->type->ops.lkp;
7356 r->mailbox = calloc(1, size);
7357 CHECK(r->mailbox, ENOMEM);
7361 r->key = table->header ?
7362 &t->structs[table->header->struct_id] :
7363 &t->structs[p->metadata_struct_id];
7365 r->func = table_stub_lkp;
7374 table_build_free(struct rte_swx_pipeline *p)
7378 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7379 struct thread *t = &p->threads[i];
7385 for (j = 0; j < p->n_tables; j++) {
7386 struct table_runtime *r = &t->tables[j];
7395 if (p->table_stats) {
7396 for (i = 0; i < p->n_tables; i++)
7397 free(p->table_stats[i].n_pkts_action);
7399 free(p->table_stats);
7404 table_free(struct rte_swx_pipeline *p)
7406 table_build_free(p);
7412 elem = TAILQ_FIRST(&p->tables);
7416 TAILQ_REMOVE(&p->tables, elem, node);
7418 free(elem->actions);
7419 free(elem->default_action_data);
7425 struct table_type *elem;
7427 elem = TAILQ_FIRST(&p->table_types);
7431 TAILQ_REMOVE(&p->table_types, elem, node);
7439 static struct selector *
7440 selector_find(struct rte_swx_pipeline *p, const char *name)
7444 TAILQ_FOREACH(s, &p->selectors, node)
7445 if (strcmp(s->name, name) == 0)
7451 static struct selector *
7452 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7454 struct selector *s = NULL;
7456 TAILQ_FOREACH(s, &p->selectors, node)
7464 selector_fields_check(struct rte_swx_pipeline *p,
7465 struct rte_swx_pipeline_selector_params *params,
7466 struct header **header)
7468 struct header *h0 = NULL;
7469 struct field *hf, *mf;
7472 /* Return if no selector fields. */
7473 if (!params->n_selector_fields || !params->selector_field_names)
7476 /* Check that all the selector fields either belong to the same header
7477 * or are all meta-data fields.
7479 hf = header_field_parse(p, params->selector_field_names[0], &h0);
7480 mf = metadata_field_parse(p, params->selector_field_names[0]);
7484 for (i = 1; i < params->n_selector_fields; i++)
7488 hf = header_field_parse(p, params->selector_field_names[i], &h);
7489 if (!hf || (h->id != h0->id))
7492 mf = metadata_field_parse(p, params->selector_field_names[i]);
7497 /* Check that there are no duplicated match fields. */
7498 for (i = 0; i < params->n_selector_fields; i++) {
7499 const char *field_name = params->selector_field_names[i];
7502 for (j = i + 1; j < params->n_selector_fields; j++)
7503 if (!strcmp(params->selector_field_names[j], field_name))
7515 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
7517 struct rte_swx_pipeline_selector_params *params)
7520 struct header *selector_header = NULL;
7521 struct field *group_id_field, *member_id_field;
7527 CHECK_NAME(name, EINVAL);
7528 CHECK(!table_find(p, name), EEXIST);
7529 CHECK(!selector_find(p, name), EEXIST);
7530 CHECK(!learner_find(p, name), EEXIST);
7532 CHECK(params, EINVAL);
7534 CHECK_NAME(params->group_id_field_name, EINVAL);
7535 group_id_field = metadata_field_parse(p, params->group_id_field_name);
7536 CHECK(group_id_field, EINVAL);
7538 for (i = 0; i < params->n_selector_fields; i++) {
7539 const char *field_name = params->selector_field_names[i];
7541 CHECK_NAME(field_name, EINVAL);
7543 status = selector_fields_check(p, params, &selector_header);
7547 CHECK_NAME(params->member_id_field_name, EINVAL);
7548 member_id_field = metadata_field_parse(p, params->member_id_field_name);
7549 CHECK(member_id_field, EINVAL);
7551 CHECK(params->n_groups_max, EINVAL);
7553 CHECK(params->n_members_per_group_max, EINVAL);
7555 /* Memory allocation. */
7556 s = calloc(1, sizeof(struct selector));
7562 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
7563 if (!s->selector_fields) {
7568 /* Node initialization. */
7569 strcpy(s->name, name);
7571 s->group_id_field = group_id_field;
7573 for (i = 0; i < params->n_selector_fields; i++) {
7574 const char *field_name = params->selector_field_names[i];
7576 s->selector_fields[i] = selector_header ?
7577 header_field_parse(p, field_name, NULL) :
7578 metadata_field_parse(p, field_name);
7581 s->n_selector_fields = params->n_selector_fields;
7583 s->selector_header = selector_header;
7585 s->member_id_field = member_id_field;
7587 s->n_groups_max = params->n_groups_max;
7589 s->n_members_per_group_max = params->n_members_per_group_max;
7591 s->id = p->n_selectors;
7593 /* Node add to tailq. */
7594 TAILQ_INSERT_TAIL(&p->selectors, s, node);
7603 free(s->selector_fields);
7611 selector_params_free(struct rte_swx_table_selector_params *params)
7616 free(params->selector_mask);
7621 static struct rte_swx_table_selector_params *
7622 selector_table_params_get(struct selector *s)
7624 struct rte_swx_table_selector_params *params = NULL;
7625 struct field *first, *last;
7628 /* Memory allocation. */
7629 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
7634 params->group_id_offset = s->group_id_field->offset / 8;
7636 /* Find first (smallest offset) and last (biggest offset) selector fields. */
7637 first = s->selector_fields[0];
7638 last = s->selector_fields[0];
7640 for (i = 0; i < s->n_selector_fields; i++) {
7641 struct field *f = s->selector_fields[i];
7643 if (f->offset < first->offset)
7646 if (f->offset > last->offset)
7650 /* Selector offset and size. */
7651 params->selector_offset = first->offset / 8;
7652 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
7654 /* Memory allocation. */
7655 params->selector_mask = calloc(1, params->selector_size);
7656 if (!params->selector_mask)
7659 /* Selector mask. */
7660 for (i = 0; i < s->n_selector_fields; i++) {
7661 struct field *f = s->selector_fields[i];
7662 uint32_t start = (f->offset - first->offset) / 8;
7663 size_t size = f->n_bits / 8;
7665 memset(¶ms->selector_mask[start], 0xFF, size);
7669 params->member_id_offset = s->member_id_field->offset / 8;
7671 /* Maximum number of groups. */
7672 params->n_groups_max = s->n_groups_max;
7674 /* Maximum number of members per group. */
7675 params->n_members_per_group_max = s->n_members_per_group_max;
7680 selector_params_free(params);
7685 selector_build_free(struct rte_swx_pipeline *p)
7689 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7690 struct thread *t = &p->threads[i];
7696 for (j = 0; j < p->n_selectors; j++) {
7697 struct selector_runtime *r = &t->selectors[j];
7703 t->selectors = NULL;
7706 free(p->selector_stats);
7707 p->selector_stats = NULL;
7711 selector_build(struct rte_swx_pipeline *p)
7716 /* Per pipeline: selector statistics. */
7717 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
7718 if (!p->selector_stats) {
7723 /* Per thread: selector run-time. */
7724 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
7725 struct thread *t = &p->threads[i];
7728 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
7729 if (!t->selectors) {
7734 TAILQ_FOREACH(s, &p->selectors, node) {
7735 struct selector_runtime *r = &t->selectors[s->id];
7739 size = rte_swx_table_selector_mailbox_size_get();
7741 r->mailbox = calloc(1, size);
7748 /* r->group_id_buffer. */
7749 r->group_id_buffer = &t->structs[p->metadata_struct_id];
7751 /* r->selector_buffer. */
7752 r->selector_buffer = s->selector_header ?
7753 &t->structs[s->selector_header->struct_id] :
7754 &t->structs[p->metadata_struct_id];
7756 /* r->member_id_buffer. */
7757 r->member_id_buffer = &t->structs[p->metadata_struct_id];
7764 selector_build_free(p);
7769 selector_free(struct rte_swx_pipeline *p)
7771 selector_build_free(p);
7773 /* Selector tables. */
7775 struct selector *elem;
7777 elem = TAILQ_FIRST(&p->selectors);
7781 TAILQ_REMOVE(&p->selectors, elem, node);
7782 free(elem->selector_fields);
7790 static struct learner *
7791 learner_find(struct rte_swx_pipeline *p, const char *name)
7795 TAILQ_FOREACH(l, &p->learners, node)
7796 if (!strcmp(l->name, name))
7802 static struct learner *
7803 learner_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
7805 struct learner *l = NULL;
7807 TAILQ_FOREACH(l, &p->learners, node)
7815 learner_match_fields_check(struct rte_swx_pipeline *p,
7816 struct rte_swx_pipeline_learner_params *params,
7817 struct header **header)
7819 struct header *h0 = NULL;
7820 struct field *hf, *mf;
7823 /* Return if no match fields. */
7824 if (!params->n_fields || !params->field_names)
7827 /* Check that all the match fields either belong to the same header
7828 * or are all meta-data fields.
7830 hf = header_field_parse(p, params->field_names[0], &h0);
7831 mf = metadata_field_parse(p, params->field_names[0]);
7835 for (i = 1; i < params->n_fields; i++)
7839 hf = header_field_parse(p, params->field_names[i], &h);
7840 if (!hf || (h->id != h0->id))
7843 mf = metadata_field_parse(p, params->field_names[i]);
7848 /* Check that there are no duplicated match fields. */
7849 for (i = 0; i < params->n_fields; i++) {
7850 const char *field_name = params->field_names[i];
7853 for (j = i + 1; j < params->n_fields; j++)
7854 if (!strcmp(params->field_names[j], field_name))
7866 learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name)
7868 struct struct_type *mst = p->metadata_st, *ast = a->st;
7869 struct field *mf, *af;
7879 /* Check that mf_name is the name of a valid meta-data field. */
7880 CHECK_NAME(mf_name, EINVAL);
7881 mf = metadata_field_parse(p, mf_name);
7884 /* Check that there are enough meta-data fields, starting with the mf_name field, to cover
7885 * all the action arguments.
7887 mf_pos = mf - mst->fields;
7888 CHECK(mst->n_fields - mf_pos >= ast->n_fields, EINVAL);
7890 /* Check that the size of each of the identified meta-data fields matches exactly the size
7891 * of the corresponding action argument.
7893 for (i = 0; i < ast->n_fields; i++) {
7894 mf = &mst->fields[mf_pos + i];
7895 af = &ast->fields[i];
7897 CHECK(mf->n_bits == af->n_bits, EINVAL);
7904 learner_action_learning_check(struct rte_swx_pipeline *p,
7905 struct action *action,
7906 const char **action_names,
7911 /* For each "learn" instruction of the current action, check that the learned action (i.e.
7912 * the action passed as argument to the "learn" instruction) is also enabled for the
7913 * current learner table.
7915 for (i = 0; i < action->n_instructions; i++) {
7916 struct instruction *instr = &action->instructions[i];
7917 uint32_t found = 0, j;
7919 if (instr->type != INSTR_LEARNER_LEARN)
7922 for (j = 0; j < n_actions; j++) {
7925 a = action_find(p, action_names[j]);
7929 if (a->id == instr->learn.action_id)
7941 rte_swx_pipeline_learner_config(struct rte_swx_pipeline *p,
7943 struct rte_swx_pipeline_learner_params *params,
7947 struct learner *l = NULL;
7948 struct action *default_action;
7949 struct header *header = NULL;
7950 uint32_t action_data_size_max = 0, i;
7955 CHECK_NAME(name, EINVAL);
7956 CHECK(!table_find(p, name), EEXIST);
7957 CHECK(!selector_find(p, name), EEXIST);
7958 CHECK(!learner_find(p, name), EEXIST);
7960 CHECK(params, EINVAL);
7963 status = learner_match_fields_check(p, params, &header);
7967 /* Action checks. */
7968 CHECK(params->n_actions, EINVAL);
7970 CHECK(params->action_names, EINVAL);
7971 for (i = 0; i < params->n_actions; i++) {
7972 const char *action_name = params->action_names[i];
7973 const char *action_field_name = params->action_field_names[i];
7975 uint32_t action_data_size;
7977 CHECK_NAME(action_name, EINVAL);
7979 a = action_find(p, action_name);
7982 status = learner_action_args_check(p, a, action_field_name);
7986 status = learner_action_learning_check(p,
7988 params->action_names,
7993 action_data_size = a->st ? a->st->n_bits / 8 : 0;
7994 if (action_data_size > action_data_size_max)
7995 action_data_size_max = action_data_size;
7998 CHECK_NAME(params->default_action_name, EINVAL);
7999 for (i = 0; i < p->n_actions; i++)
8000 if (!strcmp(params->action_names[i],
8001 params->default_action_name))
8003 CHECK(i < params->n_actions, EINVAL);
8005 default_action = action_find(p, params->default_action_name);
8006 CHECK((default_action->st && params->default_action_data) ||
8007 !params->default_action_data, EINVAL);
8009 /* Any other checks. */
8010 CHECK(size, EINVAL);
8011 CHECK(timeout, EINVAL);
8013 /* Memory allocation. */
8014 l = calloc(1, sizeof(struct learner));
8018 l->fields = calloc(params->n_fields, sizeof(struct field *));
8022 l->actions = calloc(params->n_actions, sizeof(struct action *));
8026 l->action_arg = calloc(params->n_actions, sizeof(struct field *));
8030 if (action_data_size_max) {
8031 l->default_action_data = calloc(1, action_data_size_max);
8032 if (!l->default_action_data)
8036 /* Node initialization. */
8037 strcpy(l->name, name);
8039 for (i = 0; i < params->n_fields; i++) {
8040 const char *field_name = params->field_names[i];
8042 l->fields[i] = header ?
8043 header_field_parse(p, field_name, NULL) :
8044 metadata_field_parse(p, field_name);
8047 l->n_fields = params->n_fields;
8051 for (i = 0; i < params->n_actions; i++) {
8052 const char *mf_name = params->action_field_names[i];
8054 l->actions[i] = action_find(p, params->action_names[i]);
8056 l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
8059 l->default_action = default_action;
8061 if (default_action->st)
8062 memcpy(l->default_action_data,
8063 params->default_action_data,
8064 default_action->st->n_bits / 8);
8066 l->n_actions = params->n_actions;
8068 l->default_action_is_const = params->default_action_is_const;
8070 l->action_data_size_max = action_data_size_max;
8074 l->timeout = timeout;
8076 l->id = p->n_learners;
8078 /* Node add to tailq. */
8079 TAILQ_INSERT_TAIL(&p->learners, l, node);
8088 free(l->action_arg);
8097 learner_params_free(struct rte_swx_table_learner_params *params)
8102 free(params->key_mask0);
8107 static struct rte_swx_table_learner_params *
8108 learner_params_get(struct learner *l)
8110 struct rte_swx_table_learner_params *params = NULL;
8111 struct field *first, *last;
8114 /* Memory allocation. */
8115 params = calloc(1, sizeof(struct rte_swx_table_learner_params));
8119 /* Find first (smallest offset) and last (biggest offset) match fields. */
8120 first = l->fields[0];
8121 last = l->fields[0];
8123 for (i = 0; i < l->n_fields; i++) {
8124 struct field *f = l->fields[i];
8126 if (f->offset < first->offset)
8129 if (f->offset > last->offset)
8133 /* Key offset and size. */
8134 params->key_offset = first->offset / 8;
8135 params->key_size = (last->offset + last->n_bits - first->offset) / 8;
8137 /* Memory allocation. */
8138 params->key_mask0 = calloc(1, params->key_size);
8139 if (!params->key_mask0)
8143 for (i = 0; i < l->n_fields; i++) {
8144 struct field *f = l->fields[i];
8145 uint32_t start = (f->offset - first->offset) / 8;
8146 size_t size = f->n_bits / 8;
8148 memset(¶ms->key_mask0[start], 0xFF, size);
8151 /* Action data size. */
8152 params->action_data_size = l->action_data_size_max;
8154 /* Maximum number of keys. */
8155 params->n_keys_max = l->size;
8158 params->key_timeout = l->timeout;
8163 learner_params_free(params);
8168 learner_build_free(struct rte_swx_pipeline *p)
8172 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8173 struct thread *t = &p->threads[i];
8179 for (j = 0; j < p->n_learners; j++) {
8180 struct learner_runtime *r = &t->learners[j];
8183 free(r->action_data);
8190 if (p->learner_stats) {
8191 for (i = 0; i < p->n_learners; i++)
8192 free(p->learner_stats[i].n_pkts_action);
8194 free(p->learner_stats);
8199 learner_build(struct rte_swx_pipeline *p)
8204 /* Per pipeline: learner statistics. */
8205 p->learner_stats = calloc(p->n_learners, sizeof(struct learner_statistics));
8206 CHECK(p->learner_stats, ENOMEM);
8208 for (i = 0; i < p->n_learners; i++) {
8209 p->learner_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8210 CHECK(p->learner_stats[i].n_pkts_action, ENOMEM);
8213 /* Per thread: learner run-time. */
8214 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8215 struct thread *t = &p->threads[i];
8218 t->learners = calloc(p->n_learners, sizeof(struct learner_runtime));
8224 TAILQ_FOREACH(l, &p->learners, node) {
8225 struct learner_runtime *r = &t->learners[l->id];
8230 size = rte_swx_table_learner_mailbox_size_get();
8232 r->mailbox = calloc(1, size);
8240 r->key = l->header ?
8241 &t->structs[l->header->struct_id] :
8242 &t->structs[p->metadata_struct_id];
8244 /* r->action_data. */
8245 r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
8246 if (!r->action_data) {
8251 for (j = 0; j < l->n_actions; j++) {
8252 struct action *a = l->actions[j];
8253 struct field *mf = l->action_arg[j];
8254 uint8_t *m = t->structs[p->metadata_struct_id];
8256 r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
8264 learner_build_free(p);
8269 learner_free(struct rte_swx_pipeline *p)
8271 learner_build_free(p);
8273 /* Learner tables. */
8277 l = TAILQ_FIRST(&p->learners);
8281 TAILQ_REMOVE(&p->learners, l, node);
8284 free(l->action_arg);
8285 free(l->default_action_data);
8294 table_state_build(struct rte_swx_pipeline *p)
8296 struct table *table;
8300 p->table_state = calloc(p->n_tables + p->n_selectors,
8301 sizeof(struct rte_swx_table_state));
8302 CHECK(p->table_state, ENOMEM);
8304 TAILQ_FOREACH(table, &p->tables, node) {
8305 struct rte_swx_table_state *ts = &p->table_state[table->id];
8308 struct rte_swx_table_params *params;
8311 params = table_params_get(table);
8312 CHECK(params, ENOMEM);
8314 ts->obj = table->type->ops.create(params,
8319 table_params_free(params);
8320 CHECK(ts->obj, ENODEV);
8323 /* ts->default_action_data. */
8324 if (table->action_data_size_max) {
8325 ts->default_action_data =
8326 malloc(table->action_data_size_max);
8327 CHECK(ts->default_action_data, ENOMEM);
8329 memcpy(ts->default_action_data,
8330 table->default_action_data,
8331 table->action_data_size_max);
8334 /* ts->default_action_id. */
8335 ts->default_action_id = table->default_action->id;
8338 TAILQ_FOREACH(s, &p->selectors, node) {
8339 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
8340 struct rte_swx_table_selector_params *params;
8343 params = selector_table_params_get(s);
8344 CHECK(params, ENOMEM);
8346 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
8348 selector_params_free(params);
8349 CHECK(ts->obj, ENODEV);
8352 TAILQ_FOREACH(l, &p->learners, node) {
8353 struct rte_swx_table_state *ts = &p->table_state[p->n_tables +
8354 p->n_selectors + l->id];
8355 struct rte_swx_table_learner_params *params;
8358 params = learner_params_get(l);
8359 CHECK(params, ENOMEM);
8361 ts->obj = rte_swx_table_learner_create(params, p->numa_node);
8362 learner_params_free(params);
8363 CHECK(ts->obj, ENODEV);
8365 /* ts->default_action_data. */
8366 if (l->action_data_size_max) {
8367 ts->default_action_data = malloc(l->action_data_size_max);
8368 CHECK(ts->default_action_data, ENOMEM);
8370 memcpy(ts->default_action_data,
8371 l->default_action_data,
8372 l->action_data_size_max);
8375 /* ts->default_action_id. */
8376 ts->default_action_id = l->default_action->id;
8383 table_state_build_free(struct rte_swx_pipeline *p)
8387 if (!p->table_state)
8390 for (i = 0; i < p->n_tables; i++) {
8391 struct rte_swx_table_state *ts = &p->table_state[i];
8392 struct table *table = table_find_by_id(p, i);
8395 if (table->type && ts->obj)
8396 table->type->ops.free(ts->obj);
8398 /* ts->default_action_data. */
8399 free(ts->default_action_data);
8402 for (i = 0; i < p->n_selectors; i++) {
8403 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
8407 rte_swx_table_selector_free(ts->obj);
8410 for (i = 0; i < p->n_learners; i++) {
8411 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + p->n_selectors + i];
8415 rte_swx_table_learner_free(ts->obj);
8417 /* ts->default_action_data. */
8418 free(ts->default_action_data);
8421 free(p->table_state);
8422 p->table_state = NULL;
8426 table_state_free(struct rte_swx_pipeline *p)
8428 table_state_build_free(p);
8434 static struct regarray *
8435 regarray_find(struct rte_swx_pipeline *p, const char *name)
8437 struct regarray *elem;
8439 TAILQ_FOREACH(elem, &p->regarrays, node)
8440 if (!strcmp(elem->name, name))
8446 static struct regarray *
8447 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8449 struct regarray *elem = NULL;
8451 TAILQ_FOREACH(elem, &p->regarrays, node)
8459 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8468 CHECK_NAME(name, EINVAL);
8469 CHECK(!regarray_find(p, name), EEXIST);
8471 CHECK(size, EINVAL);
8472 size = rte_align32pow2(size);
8474 /* Memory allocation. */
8475 r = calloc(1, sizeof(struct regarray));
8478 /* Node initialization. */
8479 strcpy(r->name, name);
8480 r->init_val = init_val;
8482 r->id = p->n_regarrays;
8484 /* Node add to tailq. */
8485 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8492 regarray_build(struct rte_swx_pipeline *p)
8494 struct regarray *regarray;
8496 if (!p->n_regarrays)
8499 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8500 CHECK(p->regarray_runtime, ENOMEM);
8502 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8503 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8506 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8507 RTE_CACHE_LINE_SIZE,
8509 CHECK(r->regarray, ENOMEM);
8511 if (regarray->init_val)
8512 for (i = 0; i < regarray->size; i++)
8513 r->regarray[i] = regarray->init_val;
8515 r->size_mask = regarray->size - 1;
8522 regarray_build_free(struct rte_swx_pipeline *p)
8526 if (!p->regarray_runtime)
8529 for (i = 0; i < p->n_regarrays; i++) {
8530 struct regarray *regarray = regarray_find_by_id(p, i);
8531 struct regarray_runtime *r = &p->regarray_runtime[i];
8533 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8536 free(p->regarray_runtime);
8537 p->regarray_runtime = NULL;
8541 regarray_free(struct rte_swx_pipeline *p)
8543 regarray_build_free(p);
8546 struct regarray *elem;
8548 elem = TAILQ_FIRST(&p->regarrays);
8552 TAILQ_REMOVE(&p->regarrays, elem, node);
8560 static struct meter_profile *
8561 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
8563 struct meter_profile *elem;
8565 TAILQ_FOREACH(elem, &p->meter_profiles, node)
8566 if (!strcmp(elem->name, name))
8572 static struct metarray *
8573 metarray_find(struct rte_swx_pipeline *p, const char *name)
8575 struct metarray *elem;
8577 TAILQ_FOREACH(elem, &p->metarrays, node)
8578 if (!strcmp(elem->name, name))
8584 static struct metarray *
8585 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8587 struct metarray *elem = NULL;
8589 TAILQ_FOREACH(elem, &p->metarrays, node)
8597 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
8605 CHECK_NAME(name, EINVAL);
8606 CHECK(!metarray_find(p, name), EEXIST);
8608 CHECK(size, EINVAL);
8609 size = rte_align32pow2(size);
8611 /* Memory allocation. */
8612 m = calloc(1, sizeof(struct metarray));
8615 /* Node initialization. */
8616 strcpy(m->name, name);
8618 m->id = p->n_metarrays;
8620 /* Node add to tailq. */
8621 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
8627 struct meter_profile meter_profile_default = {
8636 .cir_bytes_per_period = 1,
8638 .pir_bytes_per_period = 1,
8645 meter_init(struct meter *m)
8647 memset(m, 0, sizeof(struct meter));
8648 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
8649 m->profile = &meter_profile_default;
8650 m->color_mask = RTE_COLOR_GREEN;
8652 meter_profile_default.n_users++;
8656 metarray_build(struct rte_swx_pipeline *p)
8660 if (!p->n_metarrays)
8663 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
8664 CHECK(p->metarray_runtime, ENOMEM);
8666 TAILQ_FOREACH(m, &p->metarrays, node) {
8667 struct metarray_runtime *r = &p->metarray_runtime[m->id];
8670 r->metarray = env_malloc(m->size * sizeof(struct meter),
8671 RTE_CACHE_LINE_SIZE,
8673 CHECK(r->metarray, ENOMEM);
8675 for (i = 0; i < m->size; i++)
8676 meter_init(&r->metarray[i]);
8678 r->size_mask = m->size - 1;
8685 metarray_build_free(struct rte_swx_pipeline *p)
8689 if (!p->metarray_runtime)
8692 for (i = 0; i < p->n_metarrays; i++) {
8693 struct metarray *m = metarray_find_by_id(p, i);
8694 struct metarray_runtime *r = &p->metarray_runtime[i];
8696 env_free(r->metarray, m->size * sizeof(struct meter));
8699 free(p->metarray_runtime);
8700 p->metarray_runtime = NULL;
8704 metarray_free(struct rte_swx_pipeline *p)
8706 metarray_build_free(p);
8710 struct metarray *elem;
8712 elem = TAILQ_FIRST(&p->metarrays);
8716 TAILQ_REMOVE(&p->metarrays, elem, node);
8720 /* Meter profiles. */
8722 struct meter_profile *elem;
8724 elem = TAILQ_FIRST(&p->meter_profiles);
8728 TAILQ_REMOVE(&p->meter_profiles, elem, node);
8737 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
8739 struct rte_swx_pipeline *pipeline;
8741 /* Check input parameters. */
8744 /* Memory allocation. */
8745 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
8746 CHECK(pipeline, ENOMEM);
8748 /* Initialization. */
8749 TAILQ_INIT(&pipeline->struct_types);
8750 TAILQ_INIT(&pipeline->port_in_types);
8751 TAILQ_INIT(&pipeline->ports_in);
8752 TAILQ_INIT(&pipeline->port_out_types);
8753 TAILQ_INIT(&pipeline->ports_out);
8754 TAILQ_INIT(&pipeline->extern_types);
8755 TAILQ_INIT(&pipeline->extern_objs);
8756 TAILQ_INIT(&pipeline->extern_funcs);
8757 TAILQ_INIT(&pipeline->headers);
8758 TAILQ_INIT(&pipeline->actions);
8759 TAILQ_INIT(&pipeline->table_types);
8760 TAILQ_INIT(&pipeline->tables);
8761 TAILQ_INIT(&pipeline->selectors);
8762 TAILQ_INIT(&pipeline->learners);
8763 TAILQ_INIT(&pipeline->regarrays);
8764 TAILQ_INIT(&pipeline->meter_profiles);
8765 TAILQ_INIT(&pipeline->metarrays);
8767 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
8768 pipeline->numa_node = numa_node;
8775 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
8780 free(p->instructions);
8784 table_state_free(p);
8791 extern_func_free(p);
8801 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
8802 const char **instructions,
8803 uint32_t n_instructions)
8808 err = instruction_config(p, NULL, instructions, n_instructions);
8812 /* Thread instruction pointer reset. */
8813 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8814 struct thread *t = &p->threads[i];
8816 thread_ip_reset(p, t);
8823 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
8828 CHECK(p->build_done == 0, EEXIST);
8830 status = port_in_build(p);
8834 status = port_out_build(p);
8838 status = struct_build(p);
8842 status = extern_obj_build(p);
8846 status = extern_func_build(p);
8850 status = header_build(p);
8854 status = metadata_build(p);
8858 status = action_build(p);
8862 status = table_build(p);
8866 status = selector_build(p);
8870 status = learner_build(p);
8874 status = table_state_build(p);
8878 status = regarray_build(p);
8882 status = metarray_build(p);
8890 metarray_build_free(p);
8891 regarray_build_free(p);
8892 table_state_build_free(p);
8893 learner_build_free(p);
8894 selector_build_free(p);
8895 table_build_free(p);
8896 action_build_free(p);
8897 metadata_build_free(p);
8898 header_build_free(p);
8899 extern_func_build_free(p);
8900 extern_obj_build_free(p);
8901 port_out_build_free(p);
8902 port_in_build_free(p);
8903 struct_build_free(p);
8909 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
8913 for (i = 0; i < n_instructions; i++)
8918 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
8922 for (i = 0; i < p->n_ports_out; i++) {
8923 struct port_out_runtime *port = &p->out[i];
8926 port->flush(port->obj);
8934 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
8935 struct rte_swx_ctl_pipeline_info *pipeline)
8937 struct action *action;
8938 struct table *table;
8939 uint32_t n_actions = 0, n_tables = 0;
8941 if (!p || !pipeline)
8944 TAILQ_FOREACH(action, &p->actions, node)
8947 TAILQ_FOREACH(table, &p->tables, node)
8950 pipeline->n_ports_in = p->n_ports_in;
8951 pipeline->n_ports_out = p->n_ports_out;
8952 pipeline->n_actions = n_actions;
8953 pipeline->n_tables = n_tables;
8954 pipeline->n_selectors = p->n_selectors;
8955 pipeline->n_learners = p->n_learners;
8956 pipeline->n_regarrays = p->n_regarrays;
8957 pipeline->n_metarrays = p->n_metarrays;
8963 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
8965 if (!p || !numa_node)
8968 *numa_node = p->numa_node;
8973 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
8975 struct rte_swx_ctl_action_info *action)
8977 struct action *a = NULL;
8979 if (!p || (action_id >= p->n_actions) || !action)
8982 a = action_find_by_id(p, action_id);
8986 strcpy(action->name, a->name);
8987 action->n_args = a->st ? a->st->n_fields : 0;
8992 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
8994 uint32_t action_arg_id,
8995 struct rte_swx_ctl_action_arg_info *action_arg)
8997 struct action *a = NULL;
8998 struct field *arg = NULL;
9000 if (!p || (action_id >= p->n_actions) || !action_arg)
9003 a = action_find_by_id(p, action_id);
9004 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9007 arg = &a->st->fields[action_arg_id];
9008 strcpy(action_arg->name, arg->name);
9009 action_arg->n_bits = arg->n_bits;
9010 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
9016 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9018 struct rte_swx_ctl_table_info *table)
9020 struct table *t = NULL;
9025 t = table_find_by_id(p, table_id);
9029 strcpy(table->name, t->name);
9030 strcpy(table->args, t->args);
9031 table->n_match_fields = t->n_fields;
9032 table->n_actions = t->n_actions;
9033 table->default_action_is_const = t->default_action_is_const;
9034 table->size = t->size;
9039 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9041 uint32_t match_field_id,
9042 struct rte_swx_ctl_table_match_field_info *match_field)
9045 struct match_field *f;
9047 if (!p || (table_id >= p->n_tables) || !match_field)
9050 t = table_find_by_id(p, table_id);
9051 if (!t || (match_field_id >= t->n_fields))
9054 f = &t->fields[match_field_id];
9055 match_field->match_type = f->match_type;
9056 match_field->is_header = t->header ? 1 : 0;
9057 match_field->n_bits = f->field->n_bits;
9058 match_field->offset = f->field->offset;
9064 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9066 uint32_t table_action_id,
9067 struct rte_swx_ctl_table_action_info *table_action)
9071 if (!p || (table_id >= p->n_tables) || !table_action)
9074 t = table_find_by_id(p, table_id);
9075 if (!t || (table_action_id >= t->n_actions))
9078 table_action->action_id = t->actions[table_action_id]->id;
9084 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9086 struct rte_swx_table_ops *table_ops,
9091 if (!p || (table_id >= p->n_tables))
9094 t = table_find_by_id(p, table_id);
9100 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9110 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
9111 uint32_t selector_id,
9112 struct rte_swx_ctl_selector_info *selector)
9114 struct selector *s = NULL;
9116 if (!p || !selector)
9119 s = selector_find_by_id(p, selector_id);
9123 strcpy(selector->name, s->name);
9125 selector->n_selector_fields = s->n_selector_fields;
9126 selector->n_groups_max = s->n_groups_max;
9127 selector->n_members_per_group_max = s->n_members_per_group_max;
9133 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
9134 uint32_t selector_id,
9135 struct rte_swx_ctl_table_match_field_info *field)
9139 if (!p || (selector_id >= p->n_selectors) || !field)
9142 s = selector_find_by_id(p, selector_id);
9146 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9147 field->is_header = 0;
9148 field->n_bits = s->group_id_field->n_bits;
9149 field->offset = s->group_id_field->offset;
9155 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
9156 uint32_t selector_id,
9157 uint32_t selector_field_id,
9158 struct rte_swx_ctl_table_match_field_info *field)
9163 if (!p || (selector_id >= p->n_selectors) || !field)
9166 s = selector_find_by_id(p, selector_id);
9167 if (!s || (selector_field_id >= s->n_selector_fields))
9170 f = s->selector_fields[selector_field_id];
9171 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9172 field->is_header = s->selector_header ? 1 : 0;
9173 field->n_bits = f->n_bits;
9174 field->offset = f->offset;
9180 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
9181 uint32_t selector_id,
9182 struct rte_swx_ctl_table_match_field_info *field)
9186 if (!p || (selector_id >= p->n_selectors) || !field)
9189 s = selector_find_by_id(p, selector_id);
9193 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9194 field->is_header = 0;
9195 field->n_bits = s->member_id_field->n_bits;
9196 field->offset = s->member_id_field->offset;
9202 rte_swx_ctl_learner_info_get(struct rte_swx_pipeline *p,
9203 uint32_t learner_id,
9204 struct rte_swx_ctl_learner_info *learner)
9206 struct learner *l = NULL;
9211 l = learner_find_by_id(p, learner_id);
9215 strcpy(learner->name, l->name);
9217 learner->n_match_fields = l->n_fields;
9218 learner->n_actions = l->n_actions;
9219 learner->default_action_is_const = l->default_action_is_const;
9220 learner->size = l->size;
9226 rte_swx_ctl_learner_match_field_info_get(struct rte_swx_pipeline *p,
9227 uint32_t learner_id,
9228 uint32_t match_field_id,
9229 struct rte_swx_ctl_table_match_field_info *match_field)
9234 if (!p || (learner_id >= p->n_learners) || !match_field)
9237 l = learner_find_by_id(p, learner_id);
9238 if (!l || (match_field_id >= l->n_fields))
9241 f = l->fields[match_field_id];
9242 match_field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
9243 match_field->is_header = l->header ? 1 : 0;
9244 match_field->n_bits = f->n_bits;
9245 match_field->offset = f->offset;
9251 rte_swx_ctl_learner_action_info_get(struct rte_swx_pipeline *p,
9252 uint32_t learner_id,
9253 uint32_t learner_action_id,
9254 struct rte_swx_ctl_table_action_info *learner_action)
9258 if (!p || (learner_id >= p->n_learners) || !learner_action)
9261 l = learner_find_by_id(p, learner_id);
9262 if (!l || (learner_action_id >= l->n_actions))
9265 learner_action->action_id = l->actions[learner_action_id]->id;
9271 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9272 struct rte_swx_table_state **table_state)
9274 if (!p || !table_state || !p->build_done)
9277 *table_state = p->table_state;
9282 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9283 struct rte_swx_table_state *table_state)
9285 if (!p || !table_state || !p->build_done)
9288 p->table_state = table_state;
9293 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9295 struct rte_swx_port_in_stats *stats)
9297 struct port_in *port;
9302 port = port_in_find(p, port_id);
9306 port->type->ops.stats_read(port->obj, stats);
9311 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9313 struct rte_swx_port_out_stats *stats)
9315 struct port_out *port;
9320 port = port_out_find(p, port_id);
9324 port->type->ops.stats_read(port->obj, stats);
9329 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9330 const char *table_name,
9331 struct rte_swx_table_stats *stats)
9333 struct table *table;
9334 struct table_statistics *table_stats;
9336 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9339 table = table_find(p, table_name);
9343 table_stats = &p->table_stats[table->id];
9345 memcpy(stats->n_pkts_action,
9346 table_stats->n_pkts_action,
9347 p->n_actions * sizeof(uint64_t));
9349 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9350 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9356 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
9357 const char *selector_name,
9358 struct rte_swx_pipeline_selector_stats *stats)
9362 if (!p || !selector_name || !selector_name[0] || !stats)
9365 s = selector_find(p, selector_name);
9369 stats->n_pkts = p->selector_stats[s->id].n_pkts;
9375 rte_swx_ctl_pipeline_learner_stats_read(struct rte_swx_pipeline *p,
9376 const char *learner_name,
9377 struct rte_swx_learner_stats *stats)
9380 struct learner_statistics *learner_stats;
9382 if (!p || !learner_name || !learner_name[0] || !stats || !stats->n_pkts_action)
9385 l = learner_find(p, learner_name);
9389 learner_stats = &p->learner_stats[l->id];
9391 memcpy(stats->n_pkts_action,
9392 learner_stats->n_pkts_action,
9393 p->n_actions * sizeof(uint64_t));
9395 stats->n_pkts_hit = learner_stats->n_pkts_hit[1];
9396 stats->n_pkts_miss = learner_stats->n_pkts_hit[0];
9398 stats->n_pkts_learn_ok = learner_stats->n_pkts_learn[0];
9399 stats->n_pkts_learn_err = learner_stats->n_pkts_learn[1];
9401 stats->n_pkts_forget = learner_stats->n_pkts_forget;
9407 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9408 uint32_t regarray_id,
9409 struct rte_swx_ctl_regarray_info *regarray)
9413 if (!p || !regarray)
9416 r = regarray_find_by_id(p, regarray_id);
9420 strcpy(regarray->name, r->name);
9421 regarray->size = r->size;
9426 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9427 const char *regarray_name,
9428 uint32_t regarray_index,
9431 struct regarray *regarray;
9432 struct regarray_runtime *r;
9434 if (!p || !regarray_name || !value)
9437 regarray = regarray_find(p, regarray_name);
9438 if (!regarray || (regarray_index >= regarray->size))
9441 r = &p->regarray_runtime[regarray->id];
9442 *value = r->regarray[regarray_index];
9447 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9448 const char *regarray_name,
9449 uint32_t regarray_index,
9452 struct regarray *regarray;
9453 struct regarray_runtime *r;
9455 if (!p || !regarray_name)
9458 regarray = regarray_find(p, regarray_name);
9459 if (!regarray || (regarray_index >= regarray->size))
9462 r = &p->regarray_runtime[regarray->id];
9463 r->regarray[regarray_index] = value;
9468 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9469 uint32_t metarray_id,
9470 struct rte_swx_ctl_metarray_info *metarray)
9474 if (!p || !metarray)
9477 m = metarray_find_by_id(p, metarray_id);
9481 strcpy(metarray->name, m->name);
9482 metarray->size = m->size;
9487 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9489 struct rte_meter_trtcm_params *params)
9491 struct meter_profile *mp;
9495 CHECK_NAME(name, EINVAL);
9496 CHECK(params, EINVAL);
9497 CHECK(!meter_profile_find(p, name), EEXIST);
9499 /* Node allocation. */
9500 mp = calloc(1, sizeof(struct meter_profile));
9503 /* Node initialization. */
9504 strcpy(mp->name, name);
9505 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9506 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9512 /* Node add to tailq. */
9513 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9519 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9522 struct meter_profile *mp;
9525 CHECK_NAME(name, EINVAL);
9527 mp = meter_profile_find(p, name);
9529 CHECK(!mp->n_users, EBUSY);
9531 /* Remove node from tailq. */
9532 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9539 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9540 const char *metarray_name,
9541 uint32_t metarray_index)
9543 struct meter_profile *mp_old;
9544 struct metarray *metarray;
9545 struct metarray_runtime *metarray_runtime;
9549 CHECK_NAME(metarray_name, EINVAL);
9551 metarray = metarray_find(p, metarray_name);
9552 CHECK(metarray, EINVAL);
9553 CHECK(metarray_index < metarray->size, EINVAL);
9555 metarray_runtime = &p->metarray_runtime[metarray->id];
9556 m = &metarray_runtime->metarray[metarray_index];
9557 mp_old = m->profile;
9567 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9568 const char *metarray_name,
9569 uint32_t metarray_index,
9570 const char *profile_name)
9572 struct meter_profile *mp, *mp_old;
9573 struct metarray *metarray;
9574 struct metarray_runtime *metarray_runtime;
9578 CHECK_NAME(metarray_name, EINVAL);
9580 metarray = metarray_find(p, metarray_name);
9581 CHECK(metarray, EINVAL);
9582 CHECK(metarray_index < metarray->size, EINVAL);
9584 mp = meter_profile_find(p, profile_name);
9587 metarray_runtime = &p->metarray_runtime[metarray->id];
9588 m = &metarray_runtime->metarray[metarray_index];
9589 mp_old = m->profile;
9591 memset(m, 0, sizeof(struct meter));
9592 rte_meter_trtcm_config(&m->m, &mp->profile);
9594 m->color_mask = RTE_COLORS;
9603 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9604 const char *metarray_name,
9605 uint32_t metarray_index,
9606 struct rte_swx_ctl_meter_stats *stats)
9608 struct metarray *metarray;
9609 struct metarray_runtime *metarray_runtime;
9613 CHECK_NAME(metarray_name, EINVAL);
9615 metarray = metarray_find(p, metarray_name);
9616 CHECK(metarray, EINVAL);
9617 CHECK(metarray_index < metarray->size, EINVAL);
9619 CHECK(stats, EINVAL);
9621 metarray_runtime = &p->metarray_runtime[metarray->id];
9622 m = &metarray_runtime->metarray[metarray_index];
9624 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9625 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));