1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <arpa/inet.h>
12 #include <rte_common.h>
13 #include <rte_prefetch.h>
14 #include <rte_byteorder.h>
15 #include <rte_cycles.h>
16 #include <rte_meter.h>
18 #include <rte_swx_table_selector.h>
20 #include "rte_swx_pipeline.h"
21 #include "rte_swx_ctl.h"
23 #define CHECK(condition, err_code) \
29 #define CHECK_NAME(name, err_code) \
32 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
35 #define CHECK_INSTRUCTION(instr, err_code) \
38 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
39 RTE_SWX_INSTRUCTION_SIZE), \
47 #define TRACE(...) printf(__VA_ARGS__)
55 #define ntoh64(x) rte_be_to_cpu_64(x)
56 #define hton64(x) rte_cpu_to_be_64(x)
58 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
60 #include <rte_malloc.h>
63 env_malloc(size_t size, size_t alignment, int numa_node)
65 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
69 env_free(void *start, size_t size __rte_unused)
79 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
83 if (numa_available() == -1)
86 start = numa_alloc_onnode(size, numa_node);
90 memset(start, 0, size);
95 env_free(void *start, size_t size)
97 if (numa_available() == -1)
100 numa_free(start, size);
109 char name[RTE_SWX_NAME_SIZE];
116 TAILQ_ENTRY(struct_type) node;
117 char name[RTE_SWX_NAME_SIZE];
118 struct field *fields;
125 TAILQ_HEAD(struct_type_tailq, struct_type);
130 struct port_in_type {
131 TAILQ_ENTRY(port_in_type) node;
132 char name[RTE_SWX_NAME_SIZE];
133 struct rte_swx_port_in_ops ops;
136 TAILQ_HEAD(port_in_type_tailq, port_in_type);
139 TAILQ_ENTRY(port_in) node;
140 struct port_in_type *type;
145 TAILQ_HEAD(port_in_tailq, port_in);
147 struct port_in_runtime {
148 rte_swx_port_in_pkt_rx_t pkt_rx;
155 struct port_out_type {
156 TAILQ_ENTRY(port_out_type) node;
157 char name[RTE_SWX_NAME_SIZE];
158 struct rte_swx_port_out_ops ops;
161 TAILQ_HEAD(port_out_type_tailq, port_out_type);
164 TAILQ_ENTRY(port_out) node;
165 struct port_out_type *type;
170 TAILQ_HEAD(port_out_tailq, port_out);
172 struct port_out_runtime {
173 rte_swx_port_out_pkt_tx_t pkt_tx;
174 rte_swx_port_out_flush_t flush;
181 struct extern_type_member_func {
182 TAILQ_ENTRY(extern_type_member_func) node;
183 char name[RTE_SWX_NAME_SIZE];
184 rte_swx_extern_type_member_func_t func;
188 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
191 TAILQ_ENTRY(extern_type) node;
192 char name[RTE_SWX_NAME_SIZE];
193 struct struct_type *mailbox_struct_type;
194 rte_swx_extern_type_constructor_t constructor;
195 rte_swx_extern_type_destructor_t destructor;
196 struct extern_type_member_func_tailq funcs;
200 TAILQ_HEAD(extern_type_tailq, extern_type);
203 TAILQ_ENTRY(extern_obj) node;
204 char name[RTE_SWX_NAME_SIZE];
205 struct extern_type *type;
211 TAILQ_HEAD(extern_obj_tailq, extern_obj);
213 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
214 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
217 struct extern_obj_runtime {
220 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
227 TAILQ_ENTRY(extern_func) node;
228 char name[RTE_SWX_NAME_SIZE];
229 struct struct_type *mailbox_struct_type;
230 rte_swx_extern_func_t func;
235 TAILQ_HEAD(extern_func_tailq, extern_func);
237 struct extern_func_runtime {
239 rte_swx_extern_func_t func;
246 TAILQ_ENTRY(header) node;
247 char name[RTE_SWX_NAME_SIZE];
248 struct struct_type *st;
253 TAILQ_HEAD(header_tailq, header);
255 struct header_runtime {
260 struct header_out_runtime {
270 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
271 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
272 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
273 * when transferred to packet meta-data and in NBO when transferred to packet
277 /* Notation conventions:
278 * -Header field: H = h.header.field (dst/src)
279 * -Meta-data field: M = m.field (dst/src)
280 * -Extern object mailbox field: E = e.field (dst/src)
281 * -Extern function mailbox field: F = f.field (dst/src)
282 * -Table action data field: T = t.field (src only)
283 * -Immediate value: I = 32-bit unsigned value (src only)
286 enum instruction_type {
293 INSTR_TX, /* port_out = M */
294 INSTR_TX_I, /* port_out = I */
296 /* extract h.header */
317 /* validate h.header */
320 /* invalidate h.header */
321 INSTR_HDR_INVALIDATE,
325 * dst = HMEF, src = HMEFTI
327 INSTR_MOV, /* dst = MEF, src = MEFT */
328 INSTR_MOV_MH, /* dst = MEF, src = H */
329 INSTR_MOV_HM, /* dst = H, src = MEFT */
330 INSTR_MOV_HH, /* dst = H, src = H */
331 INSTR_MOV_I, /* dst = HMEF, src = I */
333 /* dma h.header t.field
334 * memcpy(h.header, t.field, sizeof(h.header))
347 * dst = HMEF, src = HMEFTI
349 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
350 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
351 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
352 INSTR_ALU_ADD_HH, /* dst = H, src = H */
353 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
354 INSTR_ALU_ADD_HI, /* dst = H, src = I */
358 * dst = HMEF, src = HMEFTI
360 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
361 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
362 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
363 INSTR_ALU_SUB_HH, /* dst = H, src = H */
364 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
365 INSTR_ALU_SUB_HI, /* dst = H, src = I */
368 * dst = dst '+ src[0:1] '+ src[2:3] + ...
369 * dst = H, src = {H, h.header}
371 INSTR_ALU_CKADD_FIELD, /* src = H */
372 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
373 INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
379 INSTR_ALU_CKSUB_FIELD,
383 * dst = HMEF, src = HMEFTI
385 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
386 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
387 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
388 INSTR_ALU_AND_HH, /* dst = H, src = H */
389 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
393 * dst = HMEF, src = HMEFTI
395 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
396 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
397 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
398 INSTR_ALU_OR_HH, /* dst = H, src = H */
399 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
403 * dst = HMEF, src = HMEFTI
405 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
406 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
407 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
408 INSTR_ALU_XOR_HH, /* dst = H, src = H */
409 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
413 * dst = HMEF, src = HMEFTI
415 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
416 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
417 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
418 INSTR_ALU_SHL_HH, /* dst = H, src = H */
419 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
420 INSTR_ALU_SHL_HI, /* dst = H, src = I */
424 * dst = HMEF, src = HMEFTI
426 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
427 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
428 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
429 INSTR_ALU_SHR_HH, /* dst = H, src = H */
430 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
431 INSTR_ALU_SHR_HI, /* dst = H, src = I */
433 /* regprefetch REGARRAY index
434 * prefetch REGARRAY[index]
437 INSTR_REGPREFETCH_RH, /* index = H */
438 INSTR_REGPREFETCH_RM, /* index = MEFT */
439 INSTR_REGPREFETCH_RI, /* index = I */
441 /* regrd dst REGARRAY index
442 * dst = REGARRAY[index]
443 * dst = HMEF, index = HMEFTI
445 INSTR_REGRD_HRH, /* dst = H, index = H */
446 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
447 INSTR_REGRD_HRI, /* dst = H, index = I */
448 INSTR_REGRD_MRH, /* dst = MEF, index = H */
449 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
450 INSTR_REGRD_MRI, /* dst = MEF, index = I */
452 /* regwr REGARRAY index src
453 * REGARRAY[index] = src
454 * index = HMEFTI, src = HMEFTI
456 INSTR_REGWR_RHH, /* index = H, src = H */
457 INSTR_REGWR_RHM, /* index = H, src = MEFT */
458 INSTR_REGWR_RHI, /* index = H, src = I */
459 INSTR_REGWR_RMH, /* index = MEFT, src = H */
460 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
461 INSTR_REGWR_RMI, /* index = MEFT, src = I */
462 INSTR_REGWR_RIH, /* index = I, src = H */
463 INSTR_REGWR_RIM, /* index = I, src = MEFT */
464 INSTR_REGWR_RII, /* index = I, src = I */
466 /* regadd REGARRAY index src
467 * REGARRAY[index] += src
468 * index = HMEFTI, src = HMEFTI
470 INSTR_REGADD_RHH, /* index = H, src = H */
471 INSTR_REGADD_RHM, /* index = H, src = MEFT */
472 INSTR_REGADD_RHI, /* index = H, src = I */
473 INSTR_REGADD_RMH, /* index = MEFT, src = H */
474 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
475 INSTR_REGADD_RMI, /* index = MEFT, src = I */
476 INSTR_REGADD_RIH, /* index = I, src = H */
477 INSTR_REGADD_RIM, /* index = I, src = MEFT */
478 INSTR_REGADD_RII, /* index = I, src = I */
480 /* metprefetch METARRAY index
481 * prefetch METARRAY[index]
484 INSTR_METPREFETCH_H, /* index = H */
485 INSTR_METPREFETCH_M, /* index = MEFT */
486 INSTR_METPREFETCH_I, /* index = I */
488 /* meter METARRAY index length color_in color_out
489 * color_out = meter(METARRAY[index], length, color_in)
490 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
492 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
493 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
494 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
495 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
496 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
497 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
498 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
499 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
500 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
501 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
502 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
503 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
509 /* extern e.obj.func */
520 /* jmpv LABEL h.header
521 * Jump if header is valid
525 /* jmpnv LABEL h.header
526 * Jump if header is invalid
531 * Jump if table lookup hit
536 * Jump if table lookup miss
543 INSTR_JMP_ACTION_HIT,
545 /* jmpna LABEL ACTION
546 * Jump if action not run
548 INSTR_JMP_ACTION_MISS,
551 * Jump if a is equal to b
552 * a = HMEFT, b = HMEFTI
554 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
555 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
556 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
557 INSTR_JMP_EQ_HH, /* a = H, b = H */
558 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
561 * Jump if a is not equal to b
562 * a = HMEFT, b = HMEFTI
564 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
565 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
566 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
567 INSTR_JMP_NEQ_HH, /* a = H, b = H */
568 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
571 * Jump if a is less than b
572 * a = HMEFT, b = HMEFTI
574 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
575 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
576 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
577 INSTR_JMP_LT_HH, /* a = H, b = H */
578 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
579 INSTR_JMP_LT_HI, /* a = H, b = I */
582 * Jump if a is greater than b
583 * a = HMEFT, b = HMEFTI
585 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
586 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
587 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
588 INSTR_JMP_GT_HH, /* a = H, b = H */
589 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
590 INSTR_JMP_GT_HI, /* a = H, b = I */
598 struct instr_operand {
619 uint8_t header_id[8];
620 uint8_t struct_id[8];
625 struct instr_hdr_validity {
633 struct instr_extern_obj {
638 struct instr_extern_func {
642 struct instr_dst_src {
643 struct instr_operand dst;
645 struct instr_operand src;
650 struct instr_regarray {
655 struct instr_operand idx;
660 struct instr_operand dstsrc;
670 struct instr_operand idx;
674 struct instr_operand length;
677 struct instr_operand color_in;
678 uint32_t color_in_val;
681 struct instr_operand color_out;
686 uint8_t header_id[8];
687 uint8_t struct_id[8];
698 struct instruction *ip;
701 struct instr_operand a;
707 struct instr_operand b;
713 enum instruction_type type;
716 struct instr_hdr_validity valid;
717 struct instr_dst_src mov;
718 struct instr_regarray regarray;
719 struct instr_meter meter;
720 struct instr_dma dma;
721 struct instr_dst_src alu;
722 struct instr_table table;
723 struct instr_extern_obj ext_obj;
724 struct instr_extern_func ext_func;
725 struct instr_jmp jmp;
729 struct instruction_data {
730 char label[RTE_SWX_NAME_SIZE];
731 char jmp_label[RTE_SWX_NAME_SIZE];
732 uint32_t n_users; /* user = jmp instruction to this instruction. */
740 TAILQ_ENTRY(action) node;
741 char name[RTE_SWX_NAME_SIZE];
742 struct struct_type *st;
743 int *args_endianness; /* 0 = Host Byte Order (HBO). */
744 struct instruction *instructions;
745 uint32_t n_instructions;
749 TAILQ_HEAD(action_tailq, action);
755 TAILQ_ENTRY(table_type) node;
756 char name[RTE_SWX_NAME_SIZE];
757 enum rte_swx_table_match_type match_type;
758 struct rte_swx_table_ops ops;
761 TAILQ_HEAD(table_type_tailq, table_type);
764 enum rte_swx_table_match_type match_type;
769 TAILQ_ENTRY(table) node;
770 char name[RTE_SWX_NAME_SIZE];
771 char args[RTE_SWX_NAME_SIZE];
772 struct table_type *type; /* NULL when n_fields == 0. */
775 struct match_field *fields;
777 struct header *header; /* Only valid when n_fields > 0. */
780 struct action **actions;
781 struct action *default_action;
782 uint8_t *default_action_data;
784 int default_action_is_const;
785 uint32_t action_data_size_max;
791 TAILQ_HEAD(table_tailq, table);
793 struct table_runtime {
794 rte_swx_table_lookup_t func;
799 struct table_statistics {
800 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
801 uint64_t *n_pkts_action;
808 TAILQ_ENTRY(selector) node;
809 char name[RTE_SWX_NAME_SIZE];
811 struct field *group_id_field;
812 struct field **selector_fields;
813 uint32_t n_selector_fields;
814 struct header *selector_header;
815 struct field *member_id_field;
817 uint32_t n_groups_max;
818 uint32_t n_members_per_group_max;
823 TAILQ_HEAD(selector_tailq, selector);
825 struct selector_runtime {
827 uint8_t **group_id_buffer;
828 uint8_t **selector_buffer;
829 uint8_t **member_id_buffer;
832 struct selector_statistics {
840 TAILQ_ENTRY(regarray) node;
841 char name[RTE_SWX_NAME_SIZE];
847 TAILQ_HEAD(regarray_tailq, regarray);
849 struct regarray_runtime {
857 struct meter_profile {
858 TAILQ_ENTRY(meter_profile) node;
859 char name[RTE_SWX_NAME_SIZE];
860 struct rte_meter_trtcm_params params;
861 struct rte_meter_trtcm_profile profile;
865 TAILQ_HEAD(meter_profile_tailq, meter_profile);
868 TAILQ_ENTRY(metarray) node;
869 char name[RTE_SWX_NAME_SIZE];
874 TAILQ_HEAD(metarray_tailq, metarray);
877 struct rte_meter_trtcm m;
878 struct meter_profile *profile;
879 enum rte_color color_mask;
882 uint64_t n_pkts[RTE_COLORS];
883 uint64_t n_bytes[RTE_COLORS];
886 struct metarray_runtime {
887 struct meter *metarray;
896 struct rte_swx_pkt pkt;
902 /* Packet headers. */
903 struct header_runtime *headers; /* Extracted or generated headers. */
904 struct header_out_runtime *headers_out; /* Emitted headers. */
905 uint8_t *header_storage;
906 uint8_t *header_out_storage;
907 uint64_t valid_headers;
908 uint32_t n_headers_out;
910 /* Packet meta-data. */
914 struct table_runtime *tables;
915 struct selector_runtime *selectors;
916 struct rte_swx_table_state *table_state;
918 int hit; /* 0 = Miss, 1 = Hit. */
920 /* Extern objects and functions. */
921 struct extern_obj_runtime *extern_objs;
922 struct extern_func_runtime *extern_funcs;
925 struct instruction *ip;
926 struct instruction *ret;
929 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
930 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
931 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
933 #define HEADER_VALID(thread, header_id) \
934 MASK64_BIT_GET((thread)->valid_headers, header_id)
936 #define ALU(thread, ip, operator) \
938 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
939 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
940 uint64_t dst64 = *dst64_ptr; \
941 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
942 uint64_t dst = dst64 & dst64_mask; \
944 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
945 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
946 uint64_t src64 = *src64_ptr; \
947 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
948 uint64_t src = src64 & src64_mask; \
950 uint64_t result = dst operator src; \
952 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
955 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
957 #define ALU_MH(thread, ip, operator) \
959 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
960 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
961 uint64_t dst64 = *dst64_ptr; \
962 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
963 uint64_t dst = dst64 & dst64_mask; \
965 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
966 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
967 uint64_t src64 = *src64_ptr; \
968 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
970 uint64_t result = dst operator src; \
972 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
975 #define ALU_HM(thread, ip, operator) \
977 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
978 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
979 uint64_t dst64 = *dst64_ptr; \
980 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
981 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
983 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
984 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
985 uint64_t src64 = *src64_ptr; \
986 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
987 uint64_t src = src64 & src64_mask; \
989 uint64_t result = dst operator src; \
990 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
992 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
995 #define ALU_HM_FAST(thread, ip, operator) \
997 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
998 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
999 uint64_t dst64 = *dst64_ptr; \
1000 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1001 uint64_t dst = dst64 & dst64_mask; \
1003 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1004 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1005 uint64_t src64 = *src64_ptr; \
1006 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1007 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1009 uint64_t result = dst operator src; \
1011 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1014 #define ALU_HH(thread, ip, operator) \
1016 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1017 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1018 uint64_t dst64 = *dst64_ptr; \
1019 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1020 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1022 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1023 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1024 uint64_t src64 = *src64_ptr; \
1025 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1027 uint64_t result = dst operator src; \
1028 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1030 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1033 #define ALU_HH_FAST(thread, ip, operator) \
1035 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1036 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1037 uint64_t dst64 = *dst64_ptr; \
1038 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1039 uint64_t dst = dst64 & dst64_mask; \
1041 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1042 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1043 uint64_t src64 = *src64_ptr; \
1044 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1046 uint64_t result = dst operator src; \
1048 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1055 #define ALU_HM_FAST ALU
1057 #define ALU_HH_FAST ALU
1061 #define ALU_I(thread, ip, operator) \
1063 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1064 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1065 uint64_t dst64 = *dst64_ptr; \
1066 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1067 uint64_t dst = dst64 & dst64_mask; \
1069 uint64_t src = (ip)->alu.src_val; \
1071 uint64_t result = dst operator src; \
1073 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1076 #define ALU_MI ALU_I
1078 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1080 #define ALU_HI(thread, ip, operator) \
1082 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1083 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1084 uint64_t dst64 = *dst64_ptr; \
1085 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1086 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1088 uint64_t src = (ip)->alu.src_val; \
1090 uint64_t result = dst operator src; \
1091 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1093 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1098 #define ALU_HI ALU_I
1102 #define MOV(thread, ip) \
1104 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1105 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1106 uint64_t dst64 = *dst64_ptr; \
1107 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1109 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1110 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1111 uint64_t src64 = *src64_ptr; \
1112 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1113 uint64_t src = src64 & src64_mask; \
1115 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1118 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1120 #define MOV_MH(thread, ip) \
1122 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1123 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1124 uint64_t dst64 = *dst64_ptr; \
1125 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1127 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1128 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1129 uint64_t src64 = *src64_ptr; \
1130 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1132 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1135 #define MOV_HM(thread, ip) \
1137 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1138 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1139 uint64_t dst64 = *dst64_ptr; \
1140 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1142 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1143 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1144 uint64_t src64 = *src64_ptr; \
1145 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1146 uint64_t src = src64 & src64_mask; \
1148 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1149 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1152 #define MOV_HH(thread, ip) \
1154 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1155 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1156 uint64_t dst64 = *dst64_ptr; \
1157 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1159 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1160 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1161 uint64_t src64 = *src64_ptr; \
1163 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1164 src = src >> (64 - (ip)->mov.dst.n_bits); \
1165 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1176 #define MOV_I(thread, ip) \
1178 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1179 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1180 uint64_t dst64 = *dst64_ptr; \
1181 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1183 uint64_t src = (ip)->mov.src_val; \
1185 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1188 #define JMP_CMP(thread, ip, operator) \
1190 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1191 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1192 uint64_t a64 = *a64_ptr; \
1193 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1194 uint64_t a = a64 & a64_mask; \
1196 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1197 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1198 uint64_t b64 = *b64_ptr; \
1199 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1200 uint64_t b = b64 & b64_mask; \
1202 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1205 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1207 #define JMP_CMP_MH(thread, ip, operator) \
1209 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1210 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1211 uint64_t a64 = *a64_ptr; \
1212 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1213 uint64_t a = a64 & a64_mask; \
1215 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1216 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1217 uint64_t b64 = *b64_ptr; \
1218 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1220 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1223 #define JMP_CMP_HM(thread, ip, operator) \
1225 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1226 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1227 uint64_t a64 = *a64_ptr; \
1228 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1230 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1231 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1232 uint64_t b64 = *b64_ptr; \
1233 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1234 uint64_t b = b64 & b64_mask; \
1236 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1239 #define JMP_CMP_HH(thread, ip, operator) \
1241 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1242 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1243 uint64_t a64 = *a64_ptr; \
1244 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1246 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1247 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1248 uint64_t b64 = *b64_ptr; \
1249 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1251 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1254 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1256 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1257 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1258 uint64_t a64 = *a64_ptr; \
1259 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1261 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1262 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1263 uint64_t b64 = *b64_ptr; \
1264 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1266 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1271 #define JMP_CMP_MH JMP_CMP
1272 #define JMP_CMP_HM JMP_CMP
1273 #define JMP_CMP_HH JMP_CMP
1274 #define JMP_CMP_HH_FAST JMP_CMP
1278 #define JMP_CMP_I(thread, ip, operator) \
1280 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1281 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1282 uint64_t a64 = *a64_ptr; \
1283 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1284 uint64_t a = a64 & a64_mask; \
1286 uint64_t b = (ip)->jmp.b_val; \
1288 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1291 #define JMP_CMP_MI JMP_CMP_I
1293 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1295 #define JMP_CMP_HI(thread, ip, operator) \
1297 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1298 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1299 uint64_t a64 = *a64_ptr; \
1300 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1302 uint64_t b = (ip)->jmp.b_val; \
1304 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1309 #define JMP_CMP_HI JMP_CMP_I
1313 #define METADATA_READ(thread, offset, n_bits) \
1315 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1316 uint64_t m64 = *m64_ptr; \
1317 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1321 #define METADATA_WRITE(thread, offset, n_bits, value) \
1323 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1324 uint64_t m64 = *m64_ptr; \
1325 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1327 uint64_t m_new = value; \
1329 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1332 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1333 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1336 struct rte_swx_pipeline {
1337 struct struct_type_tailq struct_types;
1338 struct port_in_type_tailq port_in_types;
1339 struct port_in_tailq ports_in;
1340 struct port_out_type_tailq port_out_types;
1341 struct port_out_tailq ports_out;
1342 struct extern_type_tailq extern_types;
1343 struct extern_obj_tailq extern_objs;
1344 struct extern_func_tailq extern_funcs;
1345 struct header_tailq headers;
1346 struct struct_type *metadata_st;
1347 uint32_t metadata_struct_id;
1348 struct action_tailq actions;
1349 struct table_type_tailq table_types;
1350 struct table_tailq tables;
1351 struct selector_tailq selectors;
1352 struct regarray_tailq regarrays;
1353 struct meter_profile_tailq meter_profiles;
1354 struct metarray_tailq metarrays;
1356 struct port_in_runtime *in;
1357 struct port_out_runtime *out;
1358 struct instruction **action_instructions;
1359 struct rte_swx_table_state *table_state;
1360 struct table_statistics *table_stats;
1361 struct selector_statistics *selector_stats;
1362 struct regarray_runtime *regarray_runtime;
1363 struct metarray_runtime *metarray_runtime;
1364 struct instruction *instructions;
1365 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1368 uint32_t n_ports_in;
1369 uint32_t n_ports_out;
1370 uint32_t n_extern_objs;
1371 uint32_t n_extern_funcs;
1374 uint32_t n_selectors;
1375 uint32_t n_regarrays;
1376 uint32_t n_metarrays;
1380 uint32_t n_instructions;
1388 static struct struct_type *
1389 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1391 struct struct_type *elem;
1393 TAILQ_FOREACH(elem, &p->struct_types, node)
1394 if (strcmp(elem->name, name) == 0)
1400 static struct field *
1401 struct_type_field_find(struct struct_type *st, const char *name)
1405 for (i = 0; i < st->n_fields; i++) {
1406 struct field *f = &st->fields[i];
1408 if (strcmp(f->name, name) == 0)
1416 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1418 struct rte_swx_field_params *fields,
1420 int last_field_has_variable_size)
1422 struct struct_type *st;
1426 CHECK_NAME(name, EINVAL);
1427 CHECK(fields, EINVAL);
1428 CHECK(n_fields, EINVAL);
1430 for (i = 0; i < n_fields; i++) {
1431 struct rte_swx_field_params *f = &fields[i];
1432 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1435 CHECK_NAME(f->name, EINVAL);
1436 CHECK(f->n_bits, EINVAL);
1437 CHECK((f->n_bits <= 64) || var_size, EINVAL);
1438 CHECK((f->n_bits & 7) == 0, EINVAL);
1440 for (j = 0; j < i; j++) {
1441 struct rte_swx_field_params *f_prev = &fields[j];
1443 CHECK(strcmp(f->name, f_prev->name), EINVAL);
1447 CHECK(!struct_type_find(p, name), EEXIST);
1449 /* Node allocation. */
1450 st = calloc(1, sizeof(struct struct_type));
1453 st->fields = calloc(n_fields, sizeof(struct field));
1459 /* Node initialization. */
1460 strcpy(st->name, name);
1461 for (i = 0; i < n_fields; i++) {
1462 struct field *dst = &st->fields[i];
1463 struct rte_swx_field_params *src = &fields[i];
1464 int var_size = ((i == n_fields - 1) && last_field_has_variable_size) ? 1 : 0;
1466 strcpy(dst->name, src->name);
1467 dst->n_bits = src->n_bits;
1468 dst->offset = st->n_bits;
1469 dst->var_size = var_size;
1471 st->n_bits += src->n_bits;
1472 st->n_bits_min += var_size ? 0 : src->n_bits;
1474 st->n_fields = n_fields;
1475 st->var_size = last_field_has_variable_size;
1477 /* Node add to tailq. */
1478 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1484 struct_build(struct rte_swx_pipeline *p)
1488 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1489 struct thread *t = &p->threads[i];
1491 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1492 CHECK(t->structs, ENOMEM);
1499 struct_build_free(struct rte_swx_pipeline *p)
1503 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1504 struct thread *t = &p->threads[i];
1512 struct_free(struct rte_swx_pipeline *p)
1514 struct_build_free(p);
1518 struct struct_type *elem;
1520 elem = TAILQ_FIRST(&p->struct_types);
1524 TAILQ_REMOVE(&p->struct_types, elem, node);
1533 static struct port_in_type *
1534 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1536 struct port_in_type *elem;
1541 TAILQ_FOREACH(elem, &p->port_in_types, node)
1542 if (strcmp(elem->name, name) == 0)
1549 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1551 struct rte_swx_port_in_ops *ops)
1553 struct port_in_type *elem;
1556 CHECK_NAME(name, EINVAL);
1558 CHECK(ops->create, EINVAL);
1559 CHECK(ops->free, EINVAL);
1560 CHECK(ops->pkt_rx, EINVAL);
1561 CHECK(ops->stats_read, EINVAL);
1563 CHECK(!port_in_type_find(p, name), EEXIST);
1565 /* Node allocation. */
1566 elem = calloc(1, sizeof(struct port_in_type));
1567 CHECK(elem, ENOMEM);
1569 /* Node initialization. */
1570 strcpy(elem->name, name);
1571 memcpy(&elem->ops, ops, sizeof(*ops));
1573 /* Node add to tailq. */
1574 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1579 static struct port_in *
1580 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1582 struct port_in *port;
1584 TAILQ_FOREACH(port, &p->ports_in, node)
1585 if (port->id == port_id)
1592 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1594 const char *port_type_name,
1597 struct port_in_type *type = NULL;
1598 struct port_in *port = NULL;
1603 CHECK(!port_in_find(p, port_id), EINVAL);
1605 CHECK_NAME(port_type_name, EINVAL);
1606 type = port_in_type_find(p, port_type_name);
1607 CHECK(type, EINVAL);
1609 obj = type->ops.create(args);
1612 /* Node allocation. */
1613 port = calloc(1, sizeof(struct port_in));
1614 CHECK(port, ENOMEM);
1616 /* Node initialization. */
1621 /* Node add to tailq. */
1622 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1623 if (p->n_ports_in < port_id + 1)
1624 p->n_ports_in = port_id + 1;
1630 port_in_build(struct rte_swx_pipeline *p)
1632 struct port_in *port;
1635 CHECK(p->n_ports_in, EINVAL);
1636 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1638 for (i = 0; i < p->n_ports_in; i++)
1639 CHECK(port_in_find(p, i), EINVAL);
1641 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1642 CHECK(p->in, ENOMEM);
1644 TAILQ_FOREACH(port, &p->ports_in, node) {
1645 struct port_in_runtime *in = &p->in[port->id];
1647 in->pkt_rx = port->type->ops.pkt_rx;
1648 in->obj = port->obj;
1655 port_in_build_free(struct rte_swx_pipeline *p)
1662 port_in_free(struct rte_swx_pipeline *p)
1664 port_in_build_free(p);
1668 struct port_in *port;
1670 port = TAILQ_FIRST(&p->ports_in);
1674 TAILQ_REMOVE(&p->ports_in, port, node);
1675 port->type->ops.free(port->obj);
1679 /* Input port types. */
1681 struct port_in_type *elem;
1683 elem = TAILQ_FIRST(&p->port_in_types);
1687 TAILQ_REMOVE(&p->port_in_types, elem, node);
1695 static struct port_out_type *
1696 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1698 struct port_out_type *elem;
1703 TAILQ_FOREACH(elem, &p->port_out_types, node)
1704 if (!strcmp(elem->name, name))
1711 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1713 struct rte_swx_port_out_ops *ops)
1715 struct port_out_type *elem;
1718 CHECK_NAME(name, EINVAL);
1720 CHECK(ops->create, EINVAL);
1721 CHECK(ops->free, EINVAL);
1722 CHECK(ops->pkt_tx, EINVAL);
1723 CHECK(ops->stats_read, EINVAL);
1725 CHECK(!port_out_type_find(p, name), EEXIST);
1727 /* Node allocation. */
1728 elem = calloc(1, sizeof(struct port_out_type));
1729 CHECK(elem, ENOMEM);
1731 /* Node initialization. */
1732 strcpy(elem->name, name);
1733 memcpy(&elem->ops, ops, sizeof(*ops));
1735 /* Node add to tailq. */
1736 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1741 static struct port_out *
1742 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1744 struct port_out *port;
1746 TAILQ_FOREACH(port, &p->ports_out, node)
1747 if (port->id == port_id)
1754 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1756 const char *port_type_name,
1759 struct port_out_type *type = NULL;
1760 struct port_out *port = NULL;
1765 CHECK(!port_out_find(p, port_id), EINVAL);
1767 CHECK_NAME(port_type_name, EINVAL);
1768 type = port_out_type_find(p, port_type_name);
1769 CHECK(type, EINVAL);
1771 obj = type->ops.create(args);
1774 /* Node allocation. */
1775 port = calloc(1, sizeof(struct port_out));
1776 CHECK(port, ENOMEM);
1778 /* Node initialization. */
1783 /* Node add to tailq. */
1784 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1785 if (p->n_ports_out < port_id + 1)
1786 p->n_ports_out = port_id + 1;
1792 port_out_build(struct rte_swx_pipeline *p)
1794 struct port_out *port;
1797 CHECK(p->n_ports_out, EINVAL);
1799 for (i = 0; i < p->n_ports_out; i++)
1800 CHECK(port_out_find(p, i), EINVAL);
1802 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1803 CHECK(p->out, ENOMEM);
1805 TAILQ_FOREACH(port, &p->ports_out, node) {
1806 struct port_out_runtime *out = &p->out[port->id];
1808 out->pkt_tx = port->type->ops.pkt_tx;
1809 out->flush = port->type->ops.flush;
1810 out->obj = port->obj;
1817 port_out_build_free(struct rte_swx_pipeline *p)
1824 port_out_free(struct rte_swx_pipeline *p)
1826 port_out_build_free(p);
1830 struct port_out *port;
1832 port = TAILQ_FIRST(&p->ports_out);
1836 TAILQ_REMOVE(&p->ports_out, port, node);
1837 port->type->ops.free(port->obj);
1841 /* Output port types. */
1843 struct port_out_type *elem;
1845 elem = TAILQ_FIRST(&p->port_out_types);
1849 TAILQ_REMOVE(&p->port_out_types, elem, node);
1857 static struct extern_type *
1858 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1860 struct extern_type *elem;
1862 TAILQ_FOREACH(elem, &p->extern_types, node)
1863 if (strcmp(elem->name, name) == 0)
1869 static struct extern_type_member_func *
1870 extern_type_member_func_find(struct extern_type *type, const char *name)
1872 struct extern_type_member_func *elem;
1874 TAILQ_FOREACH(elem, &type->funcs, node)
1875 if (strcmp(elem->name, name) == 0)
1881 static struct extern_obj *
1882 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1884 struct extern_obj *elem;
1886 TAILQ_FOREACH(elem, &p->extern_objs, node)
1887 if (strcmp(elem->name, name) == 0)
1893 static struct extern_type_member_func *
1894 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1896 struct extern_obj **obj)
1898 struct extern_obj *object;
1899 struct extern_type_member_func *func;
1900 char *object_name, *func_name;
1902 if (name[0] != 'e' || name[1] != '.')
1905 object_name = strdup(&name[2]);
1909 func_name = strchr(object_name, '.');
1918 object = extern_obj_find(p, object_name);
1924 func = extern_type_member_func_find(object->type, func_name);
1937 static struct field *
1938 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1940 struct extern_obj **object)
1942 struct extern_obj *obj;
1944 char *obj_name, *field_name;
1946 if ((name[0] != 'e') || (name[1] != '.'))
1949 obj_name = strdup(&name[2]);
1953 field_name = strchr(obj_name, '.');
1962 obj = extern_obj_find(p, obj_name);
1968 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1982 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1984 const char *mailbox_struct_type_name,
1985 rte_swx_extern_type_constructor_t constructor,
1986 rte_swx_extern_type_destructor_t destructor)
1988 struct extern_type *elem;
1989 struct struct_type *mailbox_struct_type;
1993 CHECK_NAME(name, EINVAL);
1994 CHECK(!extern_type_find(p, name), EEXIST);
1996 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1997 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1998 CHECK(mailbox_struct_type, EINVAL);
1999 CHECK(!mailbox_struct_type->var_size, EINVAL);
2001 CHECK(constructor, EINVAL);
2002 CHECK(destructor, EINVAL);
2004 /* Node allocation. */
2005 elem = calloc(1, sizeof(struct extern_type));
2006 CHECK(elem, ENOMEM);
2008 /* Node initialization. */
2009 strcpy(elem->name, name);
2010 elem->mailbox_struct_type = mailbox_struct_type;
2011 elem->constructor = constructor;
2012 elem->destructor = destructor;
2013 TAILQ_INIT(&elem->funcs);
2015 /* Node add to tailq. */
2016 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
2022 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
2023 const char *extern_type_name,
2025 rte_swx_extern_type_member_func_t member_func)
2027 struct extern_type *type;
2028 struct extern_type_member_func *type_member;
2032 CHECK_NAME(extern_type_name, EINVAL);
2033 type = extern_type_find(p, extern_type_name);
2034 CHECK(type, EINVAL);
2035 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
2037 CHECK_NAME(name, EINVAL);
2038 CHECK(!extern_type_member_func_find(type, name), EEXIST);
2040 CHECK(member_func, EINVAL);
2042 /* Node allocation. */
2043 type_member = calloc(1, sizeof(struct extern_type_member_func));
2044 CHECK(type_member, ENOMEM);
2046 /* Node initialization. */
2047 strcpy(type_member->name, name);
2048 type_member->func = member_func;
2049 type_member->id = type->n_funcs;
2051 /* Node add to tailq. */
2052 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2059 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2060 const char *extern_type_name,
2064 struct extern_type *type;
2065 struct extern_obj *obj;
2070 CHECK_NAME(extern_type_name, EINVAL);
2071 type = extern_type_find(p, extern_type_name);
2072 CHECK(type, EINVAL);
2074 CHECK_NAME(name, EINVAL);
2075 CHECK(!extern_obj_find(p, name), EEXIST);
2077 /* Node allocation. */
2078 obj = calloc(1, sizeof(struct extern_obj));
2081 /* Object construction. */
2082 obj_handle = type->constructor(args);
2088 /* Node initialization. */
2089 strcpy(obj->name, name);
2091 obj->obj = obj_handle;
2092 obj->struct_id = p->n_structs;
2093 obj->id = p->n_extern_objs;
2095 /* Node add to tailq. */
2096 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2104 extern_obj_build(struct rte_swx_pipeline *p)
2108 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2109 struct thread *t = &p->threads[i];
2110 struct extern_obj *obj;
2112 t->extern_objs = calloc(p->n_extern_objs,
2113 sizeof(struct extern_obj_runtime));
2114 CHECK(t->extern_objs, ENOMEM);
2116 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2117 struct extern_obj_runtime *r =
2118 &t->extern_objs[obj->id];
2119 struct extern_type_member_func *func;
2120 uint32_t mailbox_size =
2121 obj->type->mailbox_struct_type->n_bits / 8;
2125 r->mailbox = calloc(1, mailbox_size);
2126 CHECK(r->mailbox, ENOMEM);
2128 TAILQ_FOREACH(func, &obj->type->funcs, node)
2129 r->funcs[func->id] = func->func;
2131 t->structs[obj->struct_id] = r->mailbox;
2139 extern_obj_build_free(struct rte_swx_pipeline *p)
2143 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2144 struct thread *t = &p->threads[i];
2147 if (!t->extern_objs)
2150 for (j = 0; j < p->n_extern_objs; j++) {
2151 struct extern_obj_runtime *r = &t->extern_objs[j];
2156 free(t->extern_objs);
2157 t->extern_objs = NULL;
2162 extern_obj_free(struct rte_swx_pipeline *p)
2164 extern_obj_build_free(p);
2166 /* Extern objects. */
2168 struct extern_obj *elem;
2170 elem = TAILQ_FIRST(&p->extern_objs);
2174 TAILQ_REMOVE(&p->extern_objs, elem, node);
2176 elem->type->destructor(elem->obj);
2182 struct extern_type *elem;
2184 elem = TAILQ_FIRST(&p->extern_types);
2188 TAILQ_REMOVE(&p->extern_types, elem, node);
2191 struct extern_type_member_func *func;
2193 func = TAILQ_FIRST(&elem->funcs);
2197 TAILQ_REMOVE(&elem->funcs, func, node);
2208 static struct extern_func *
2209 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2211 struct extern_func *elem;
2213 TAILQ_FOREACH(elem, &p->extern_funcs, node)
2214 if (strcmp(elem->name, name) == 0)
2220 static struct extern_func *
2221 extern_func_parse(struct rte_swx_pipeline *p,
2224 if (name[0] != 'f' || name[1] != '.')
2227 return extern_func_find(p, &name[2]);
2230 static struct field *
2231 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2233 struct extern_func **function)
2235 struct extern_func *func;
2237 char *func_name, *field_name;
2239 if ((name[0] != 'f') || (name[1] != '.'))
2242 func_name = strdup(&name[2]);
2246 field_name = strchr(func_name, '.');
2255 func = extern_func_find(p, func_name);
2261 f = struct_type_field_find(func->mailbox_struct_type, field_name);
2275 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2277 const char *mailbox_struct_type_name,
2278 rte_swx_extern_func_t func)
2280 struct extern_func *f;
2281 struct struct_type *mailbox_struct_type;
2285 CHECK_NAME(name, EINVAL);
2286 CHECK(!extern_func_find(p, name), EEXIST);
2288 CHECK_NAME(mailbox_struct_type_name, EINVAL);
2289 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2290 CHECK(mailbox_struct_type, EINVAL);
2291 CHECK(!mailbox_struct_type->var_size, EINVAL);
2293 CHECK(func, EINVAL);
2295 /* Node allocation. */
2296 f = calloc(1, sizeof(struct extern_func));
2297 CHECK(func, ENOMEM);
2299 /* Node initialization. */
2300 strcpy(f->name, name);
2301 f->mailbox_struct_type = mailbox_struct_type;
2303 f->struct_id = p->n_structs;
2304 f->id = p->n_extern_funcs;
2306 /* Node add to tailq. */
2307 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2308 p->n_extern_funcs++;
2315 extern_func_build(struct rte_swx_pipeline *p)
2319 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2320 struct thread *t = &p->threads[i];
2321 struct extern_func *func;
2323 /* Memory allocation. */
2324 t->extern_funcs = calloc(p->n_extern_funcs,
2325 sizeof(struct extern_func_runtime));
2326 CHECK(t->extern_funcs, ENOMEM);
2328 /* Extern function. */
2329 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2330 struct extern_func_runtime *r =
2331 &t->extern_funcs[func->id];
2332 uint32_t mailbox_size =
2333 func->mailbox_struct_type->n_bits / 8;
2335 r->func = func->func;
2337 r->mailbox = calloc(1, mailbox_size);
2338 CHECK(r->mailbox, ENOMEM);
2340 t->structs[func->struct_id] = r->mailbox;
2348 extern_func_build_free(struct rte_swx_pipeline *p)
2352 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2353 struct thread *t = &p->threads[i];
2356 if (!t->extern_funcs)
2359 for (j = 0; j < p->n_extern_funcs; j++) {
2360 struct extern_func_runtime *r = &t->extern_funcs[j];
2365 free(t->extern_funcs);
2366 t->extern_funcs = NULL;
2371 extern_func_free(struct rte_swx_pipeline *p)
2373 extern_func_build_free(p);
2376 struct extern_func *elem;
2378 elem = TAILQ_FIRST(&p->extern_funcs);
2382 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2390 static struct header *
2391 header_find(struct rte_swx_pipeline *p, const char *name)
2393 struct header *elem;
2395 TAILQ_FOREACH(elem, &p->headers, node)
2396 if (strcmp(elem->name, name) == 0)
2402 static struct header *
2403 header_find_by_struct_id(struct rte_swx_pipeline *p, uint32_t struct_id)
2405 struct header *elem;
2407 TAILQ_FOREACH(elem, &p->headers, node)
2408 if (elem->struct_id == struct_id)
2414 static struct header *
2415 header_parse(struct rte_swx_pipeline *p,
2418 if (name[0] != 'h' || name[1] != '.')
2421 return header_find(p, &name[2]);
2424 static struct field *
2425 header_field_parse(struct rte_swx_pipeline *p,
2427 struct header **header)
2431 char *header_name, *field_name;
2433 if ((name[0] != 'h') || (name[1] != '.'))
2436 header_name = strdup(&name[2]);
2440 field_name = strchr(header_name, '.');
2449 h = header_find(p, header_name);
2455 f = struct_type_field_find(h->st, field_name);
2469 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2471 const char *struct_type_name)
2473 struct struct_type *st;
2475 size_t n_headers_max;
2478 CHECK_NAME(name, EINVAL);
2479 CHECK_NAME(struct_type_name, EINVAL);
2481 CHECK(!header_find(p, name), EEXIST);
2483 st = struct_type_find(p, struct_type_name);
2486 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2487 CHECK(p->n_headers < n_headers_max, ENOSPC);
2489 /* Node allocation. */
2490 h = calloc(1, sizeof(struct header));
2493 /* Node initialization. */
2494 strcpy(h->name, name);
2496 h->struct_id = p->n_structs;
2497 h->id = p->n_headers;
2499 /* Node add to tailq. */
2500 TAILQ_INSERT_TAIL(&p->headers, h, node);
2508 header_build(struct rte_swx_pipeline *p)
2511 uint32_t n_bytes = 0, i;
2513 TAILQ_FOREACH(h, &p->headers, node) {
2514 n_bytes += h->st->n_bits / 8;
2517 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2518 struct thread *t = &p->threads[i];
2519 uint32_t offset = 0;
2521 t->headers = calloc(p->n_headers,
2522 sizeof(struct header_runtime));
2523 CHECK(t->headers, ENOMEM);
2525 t->headers_out = calloc(p->n_headers,
2526 sizeof(struct header_out_runtime));
2527 CHECK(t->headers_out, ENOMEM);
2529 t->header_storage = calloc(1, n_bytes);
2530 CHECK(t->header_storage, ENOMEM);
2532 t->header_out_storage = calloc(1, n_bytes);
2533 CHECK(t->header_out_storage, ENOMEM);
2535 TAILQ_FOREACH(h, &p->headers, node) {
2536 uint8_t *header_storage;
2537 uint32_t n_bytes = h->st->n_bits / 8;
2539 header_storage = &t->header_storage[offset];
2542 t->headers[h->id].ptr0 = header_storage;
2543 t->headers[h->id].n_bytes = n_bytes;
2545 t->structs[h->struct_id] = header_storage;
2553 header_build_free(struct rte_swx_pipeline *p)
2557 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2558 struct thread *t = &p->threads[i];
2560 free(t->headers_out);
2561 t->headers_out = NULL;
2566 free(t->header_out_storage);
2567 t->header_out_storage = NULL;
2569 free(t->header_storage);
2570 t->header_storage = NULL;
2575 header_free(struct rte_swx_pipeline *p)
2577 header_build_free(p);
2580 struct header *elem;
2582 elem = TAILQ_FIRST(&p->headers);
2586 TAILQ_REMOVE(&p->headers, elem, node);
2594 static struct field *
2595 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2597 if (!p->metadata_st)
2600 if (name[0] != 'm' || name[1] != '.')
2603 return struct_type_field_find(p->metadata_st, &name[2]);
2607 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2608 const char *struct_type_name)
2610 struct struct_type *st = NULL;
2614 CHECK_NAME(struct_type_name, EINVAL);
2615 st = struct_type_find(p, struct_type_name);
2617 CHECK(!st->var_size, EINVAL);
2618 CHECK(!p->metadata_st, EINVAL);
2620 p->metadata_st = st;
2621 p->metadata_struct_id = p->n_structs;
2629 metadata_build(struct rte_swx_pipeline *p)
2631 uint32_t n_bytes = p->metadata_st->n_bits / 8;
2634 /* Thread-level initialization. */
2635 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2636 struct thread *t = &p->threads[i];
2639 metadata = calloc(1, n_bytes);
2640 CHECK(metadata, ENOMEM);
2642 t->metadata = metadata;
2643 t->structs[p->metadata_struct_id] = metadata;
2650 metadata_build_free(struct rte_swx_pipeline *p)
2654 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2655 struct thread *t = &p->threads[i];
2663 metadata_free(struct rte_swx_pipeline *p)
2665 metadata_build_free(p);
2672 instruction_is_tx(enum instruction_type type)
2685 instruction_is_jmp(struct instruction *instr)
2687 switch (instr->type) {
2689 case INSTR_JMP_VALID:
2690 case INSTR_JMP_INVALID:
2692 case INSTR_JMP_MISS:
2693 case INSTR_JMP_ACTION_HIT:
2694 case INSTR_JMP_ACTION_MISS:
2696 case INSTR_JMP_EQ_MH:
2697 case INSTR_JMP_EQ_HM:
2698 case INSTR_JMP_EQ_HH:
2699 case INSTR_JMP_EQ_I:
2701 case INSTR_JMP_NEQ_MH:
2702 case INSTR_JMP_NEQ_HM:
2703 case INSTR_JMP_NEQ_HH:
2704 case INSTR_JMP_NEQ_I:
2706 case INSTR_JMP_LT_MH:
2707 case INSTR_JMP_LT_HM:
2708 case INSTR_JMP_LT_HH:
2709 case INSTR_JMP_LT_MI:
2710 case INSTR_JMP_LT_HI:
2712 case INSTR_JMP_GT_MH:
2713 case INSTR_JMP_GT_HM:
2714 case INSTR_JMP_GT_HH:
2715 case INSTR_JMP_GT_MI:
2716 case INSTR_JMP_GT_HI:
2724 static struct field *
2725 action_field_parse(struct action *action, const char *name);
2727 static struct field *
2728 struct_field_parse(struct rte_swx_pipeline *p,
2729 struct action *action,
2731 uint32_t *struct_id)
2738 struct header *header;
2740 f = header_field_parse(p, name, &header);
2744 *struct_id = header->struct_id;
2750 f = metadata_field_parse(p, name);
2754 *struct_id = p->metadata_struct_id;
2763 f = action_field_parse(action, name);
2773 struct extern_obj *obj;
2775 f = extern_obj_mailbox_field_parse(p, name, &obj);
2779 *struct_id = obj->struct_id;
2785 struct extern_func *func;
2787 f = extern_func_mailbox_field_parse(p, name, &func);
2791 *struct_id = func->struct_id;
2801 pipeline_port_inc(struct rte_swx_pipeline *p)
2803 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2807 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2809 t->ip = p->instructions;
2813 thread_ip_set(struct thread *t, struct instruction *ip)
2819 thread_ip_action_call(struct rte_swx_pipeline *p,
2824 t->ip = p->action_instructions[action_id];
2828 thread_ip_inc(struct rte_swx_pipeline *p);
2831 thread_ip_inc(struct rte_swx_pipeline *p)
2833 struct thread *t = &p->threads[p->thread_id];
2839 thread_ip_inc_cond(struct thread *t, int cond)
2845 thread_yield(struct rte_swx_pipeline *p)
2847 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2851 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2853 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2860 instr_rx_translate(struct rte_swx_pipeline *p,
2861 struct action *action,
2864 struct instruction *instr,
2865 struct instruction_data *data __rte_unused)
2869 CHECK(!action, EINVAL);
2870 CHECK(n_tokens == 2, EINVAL);
2872 f = metadata_field_parse(p, tokens[1]);
2875 instr->type = INSTR_RX;
2876 instr->io.io.offset = f->offset / 8;
2877 instr->io.io.n_bits = f->n_bits;
2882 instr_rx_exec(struct rte_swx_pipeline *p);
2885 instr_rx_exec(struct rte_swx_pipeline *p)
2887 struct thread *t = &p->threads[p->thread_id];
2888 struct instruction *ip = t->ip;
2889 struct port_in_runtime *port = &p->in[p->port_id];
2890 struct rte_swx_pkt *pkt = &t->pkt;
2894 pkt_received = port->pkt_rx(port->obj, pkt);
2895 t->ptr = &pkt->pkt[pkt->offset];
2896 rte_prefetch0(t->ptr);
2898 TRACE("[Thread %2u] rx %s from port %u\n",
2900 pkt_received ? "1 pkt" : "0 pkts",
2904 t->valid_headers = 0;
2905 t->n_headers_out = 0;
2908 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2911 t->table_state = p->table_state;
2914 pipeline_port_inc(p);
2915 thread_ip_inc_cond(t, pkt_received);
2923 instr_tx_translate(struct rte_swx_pipeline *p,
2924 struct action *action __rte_unused,
2927 struct instruction *instr,
2928 struct instruction_data *data __rte_unused)
2930 char *port = tokens[1];
2934 CHECK(n_tokens == 2, EINVAL);
2936 f = metadata_field_parse(p, port);
2938 instr->type = INSTR_TX;
2939 instr->io.io.offset = f->offset / 8;
2940 instr->io.io.n_bits = f->n_bits;
2945 port_val = strtoul(port, &port, 0);
2946 CHECK(!port[0], EINVAL);
2948 instr->type = INSTR_TX_I;
2949 instr->io.io.val = port_val;
2954 instr_drop_translate(struct rte_swx_pipeline *p,
2955 struct action *action __rte_unused,
2956 char **tokens __rte_unused,
2958 struct instruction *instr,
2959 struct instruction_data *data __rte_unused)
2961 CHECK(n_tokens == 1, EINVAL);
2964 instr->type = INSTR_TX_I;
2965 instr->io.io.val = p->n_ports_out - 1;
2970 emit_handler(struct thread *t)
2972 struct header_out_runtime *h0 = &t->headers_out[0];
2973 struct header_out_runtime *h1 = &t->headers_out[1];
2974 uint32_t offset = 0, i;
2976 /* No header change or header decapsulation. */
2977 if ((t->n_headers_out == 1) &&
2978 (h0->ptr + h0->n_bytes == t->ptr)) {
2979 TRACE("Emit handler: no header change or header decap.\n");
2981 t->pkt.offset -= h0->n_bytes;
2982 t->pkt.length += h0->n_bytes;
2987 /* Header encapsulation (optionally, with prior header decasulation). */
2988 if ((t->n_headers_out == 2) &&
2989 (h1->ptr + h1->n_bytes == t->ptr) &&
2990 (h0->ptr == h0->ptr0)) {
2993 TRACE("Emit handler: header encapsulation.\n");
2995 offset = h0->n_bytes + h1->n_bytes;
2996 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2997 t->pkt.offset -= offset;
2998 t->pkt.length += offset;
3003 /* Header insertion. */
3006 /* Header extraction. */
3009 /* For any other case. */
3010 TRACE("Emit handler: complex case.\n");
3012 for (i = 0; i < t->n_headers_out; i++) {
3013 struct header_out_runtime *h = &t->headers_out[i];
3015 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
3016 offset += h->n_bytes;
3020 memcpy(t->ptr - offset, t->header_out_storage, offset);
3021 t->pkt.offset -= offset;
3022 t->pkt.length += offset;
3027 instr_tx_exec(struct rte_swx_pipeline *p);
3030 instr_tx_exec(struct rte_swx_pipeline *p)
3032 struct thread *t = &p->threads[p->thread_id];
3033 struct instruction *ip = t->ip;
3034 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
3035 struct port_out_runtime *port = &p->out[port_id];
3036 struct rte_swx_pkt *pkt = &t->pkt;
3038 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
3046 port->pkt_tx(port->obj, pkt);
3049 thread_ip_reset(p, t);
3054 instr_tx_i_exec(struct rte_swx_pipeline *p)
3056 struct thread *t = &p->threads[p->thread_id];
3057 struct instruction *ip = t->ip;
3058 uint64_t port_id = ip->io.io.val;
3059 struct port_out_runtime *port = &p->out[port_id];
3060 struct rte_swx_pkt *pkt = &t->pkt;
3062 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3070 port->pkt_tx(port->obj, pkt);
3073 thread_ip_reset(p, t);
3081 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3082 struct action *action,
3085 struct instruction *instr,
3086 struct instruction_data *data __rte_unused)
3090 CHECK(!action, EINVAL);
3091 CHECK(n_tokens == 2, EINVAL);
3093 h = header_parse(p, tokens[1]);
3095 CHECK(!h->st->var_size, EINVAL);
3097 instr->type = INSTR_HDR_EXTRACT;
3098 instr->io.hdr.header_id[0] = h->id;
3099 instr->io.hdr.struct_id[0] = h->struct_id;
3100 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3105 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3108 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3110 struct thread *t = &p->threads[p->thread_id];
3111 struct instruction *ip = t->ip;
3112 uint64_t valid_headers = t->valid_headers;
3113 uint8_t *ptr = t->ptr;
3114 uint32_t offset = t->pkt.offset;
3115 uint32_t length = t->pkt.length;
3118 for (i = 0; i < n_extract; i++) {
3119 uint32_t header_id = ip->io.hdr.header_id[i];
3120 uint32_t struct_id = ip->io.hdr.struct_id[i];
3121 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3123 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3129 t->structs[struct_id] = ptr;
3130 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3139 t->valid_headers = valid_headers;
3142 t->pkt.offset = offset;
3143 t->pkt.length = length;
3148 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3150 __instr_hdr_extract_exec(p, 1);
3157 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3159 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3162 __instr_hdr_extract_exec(p, 2);
3169 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3171 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3174 __instr_hdr_extract_exec(p, 3);
3181 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3183 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3186 __instr_hdr_extract_exec(p, 4);
3193 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3195 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3198 __instr_hdr_extract_exec(p, 5);
3205 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3207 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3210 __instr_hdr_extract_exec(p, 6);
3217 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3219 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3222 __instr_hdr_extract_exec(p, 7);
3229 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3231 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3234 __instr_hdr_extract_exec(p, 8);
3244 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3245 struct action *action __rte_unused,
3248 struct instruction *instr,
3249 struct instruction_data *data __rte_unused)
3253 CHECK(n_tokens == 2, EINVAL);
3255 h = header_parse(p, tokens[1]);
3258 instr->type = INSTR_HDR_EMIT;
3259 instr->io.hdr.header_id[0] = h->id;
3260 instr->io.hdr.struct_id[0] = h->struct_id;
3261 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3266 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3269 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3271 struct thread *t = &p->threads[p->thread_id];
3272 struct instruction *ip = t->ip;
3273 uint64_t valid_headers = t->valid_headers;
3274 uint32_t n_headers_out = t->n_headers_out;
3275 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3276 uint8_t *ho_ptr = NULL;
3277 uint32_t ho_nbytes = 0, first = 1, i;
3279 for (i = 0; i < n_emit; i++) {
3280 uint32_t header_id = ip->io.hdr.header_id[i];
3281 uint32_t struct_id = ip->io.hdr.struct_id[i];
3283 struct header_runtime *hi = &t->headers[header_id];
3284 uint8_t *hi_ptr0 = hi->ptr0;
3285 uint32_t n_bytes = hi->n_bytes;
3287 uint8_t *hi_ptr = t->structs[struct_id];
3289 if (!MASK64_BIT_GET(valid_headers, header_id))
3292 TRACE("[Thread %2u]: emit header %u\n",
3300 if (!t->n_headers_out) {
3301 ho = &t->headers_out[0];
3307 ho_nbytes = n_bytes;
3314 ho_nbytes = ho->n_bytes;
3318 if (ho_ptr + ho_nbytes == hi_ptr) {
3319 ho_nbytes += n_bytes;
3321 ho->n_bytes = ho_nbytes;
3328 ho_nbytes = n_bytes;
3334 ho->n_bytes = ho_nbytes;
3335 t->n_headers_out = n_headers_out;
3339 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3341 __instr_hdr_emit_exec(p, 1);
3348 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3350 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3353 __instr_hdr_emit_exec(p, 1);
3358 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3360 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3363 __instr_hdr_emit_exec(p, 2);
3368 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3370 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3373 __instr_hdr_emit_exec(p, 3);
3378 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3380 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3383 __instr_hdr_emit_exec(p, 4);
3388 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3390 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3393 __instr_hdr_emit_exec(p, 5);
3398 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3400 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3403 __instr_hdr_emit_exec(p, 6);
3408 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3410 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3413 __instr_hdr_emit_exec(p, 7);
3418 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3420 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3423 __instr_hdr_emit_exec(p, 8);
3431 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3432 struct action *action __rte_unused,
3435 struct instruction *instr,
3436 struct instruction_data *data __rte_unused)
3440 CHECK(n_tokens == 2, EINVAL);
3442 h = header_parse(p, tokens[1]);
3445 instr->type = INSTR_HDR_VALIDATE;
3446 instr->valid.header_id = h->id;
3451 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3453 struct thread *t = &p->threads[p->thread_id];
3454 struct instruction *ip = t->ip;
3455 uint32_t header_id = ip->valid.header_id;
3457 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3460 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3470 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3471 struct action *action __rte_unused,
3474 struct instruction *instr,
3475 struct instruction_data *data __rte_unused)
3479 CHECK(n_tokens == 2, EINVAL);
3481 h = header_parse(p, tokens[1]);
3484 instr->type = INSTR_HDR_INVALIDATE;
3485 instr->valid.header_id = h->id;
3490 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3492 struct thread *t = &p->threads[p->thread_id];
3493 struct instruction *ip = t->ip;
3494 uint32_t header_id = ip->valid.header_id;
3496 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3499 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3508 static struct table *
3509 table_find(struct rte_swx_pipeline *p, const char *name);
3511 static struct selector *
3512 selector_find(struct rte_swx_pipeline *p, const char *name);
3515 instr_table_translate(struct rte_swx_pipeline *p,
3516 struct action *action,
3519 struct instruction *instr,
3520 struct instruction_data *data __rte_unused)
3525 CHECK(!action, EINVAL);
3526 CHECK(n_tokens == 2, EINVAL);
3528 t = table_find(p, tokens[1]);
3530 instr->type = INSTR_TABLE;
3531 instr->table.table_id = t->id;
3535 s = selector_find(p, tokens[1]);
3537 instr->type = INSTR_SELECTOR;
3538 instr->table.table_id = s->id;
3546 instr_table_exec(struct rte_swx_pipeline *p)
3548 struct thread *t = &p->threads[p->thread_id];
3549 struct instruction *ip = t->ip;
3550 uint32_t table_id = ip->table.table_id;
3551 struct rte_swx_table_state *ts = &t->table_state[table_id];
3552 struct table_runtime *table = &t->tables[table_id];
3553 struct table_statistics *stats = &p->table_stats[table_id];
3554 uint64_t action_id, n_pkts_hit, n_pkts_action;
3555 uint8_t *action_data;
3559 done = table->func(ts->obj,
3567 TRACE("[Thread %2u] table %u (not finalized)\n",
3575 action_id = hit ? action_id : ts->default_action_id;
3576 action_data = hit ? action_data : ts->default_action_data;
3577 n_pkts_hit = stats->n_pkts_hit[hit];
3578 n_pkts_action = stats->n_pkts_action[action_id];
3580 TRACE("[Thread %2u] table %u (%s, action %u)\n",
3583 hit ? "hit" : "miss",
3584 (uint32_t)action_id);
3586 t->action_id = action_id;
3587 t->structs[0] = action_data;
3589 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3590 stats->n_pkts_action[action_id] = n_pkts_action + 1;
3593 thread_ip_action_call(p, t, action_id);
3597 instr_selector_exec(struct rte_swx_pipeline *p)
3599 struct thread *t = &p->threads[p->thread_id];
3600 struct instruction *ip = t->ip;
3601 uint32_t selector_id = ip->table.table_id;
3602 struct rte_swx_table_state *ts = &t->table_state[p->n_tables + selector_id];
3603 struct selector_runtime *selector = &t->selectors[selector_id];
3604 struct selector_statistics *stats = &p->selector_stats[selector_id];
3605 uint64_t n_pkts = stats->n_pkts;
3609 done = rte_swx_table_selector_select(ts->obj,
3611 selector->group_id_buffer,
3612 selector->selector_buffer,
3613 selector->member_id_buffer);
3616 TRACE("[Thread %2u] selector %u (not finalized)\n",
3625 TRACE("[Thread %2u] selector %u\n",
3629 stats->n_pkts = n_pkts + 1;
3639 instr_extern_translate(struct rte_swx_pipeline *p,
3640 struct action *action __rte_unused,
3643 struct instruction *instr,
3644 struct instruction_data *data __rte_unused)
3646 char *token = tokens[1];
3648 CHECK(n_tokens == 2, EINVAL);
3650 if (token[0] == 'e') {
3651 struct extern_obj *obj;
3652 struct extern_type_member_func *func;
3654 func = extern_obj_member_func_parse(p, token, &obj);
3655 CHECK(func, EINVAL);
3657 instr->type = INSTR_EXTERN_OBJ;
3658 instr->ext_obj.ext_obj_id = obj->id;
3659 instr->ext_obj.func_id = func->id;
3664 if (token[0] == 'f') {
3665 struct extern_func *func;
3667 func = extern_func_parse(p, token);
3668 CHECK(func, EINVAL);
3670 instr->type = INSTR_EXTERN_FUNC;
3671 instr->ext_func.ext_func_id = func->id;
3680 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3682 struct thread *t = &p->threads[p->thread_id];
3683 struct instruction *ip = t->ip;
3684 uint32_t obj_id = ip->ext_obj.ext_obj_id;
3685 uint32_t func_id = ip->ext_obj.func_id;
3686 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3687 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3689 TRACE("[Thread %2u] extern obj %u member func %u\n",
3694 /* Extern object member function execute. */
3695 uint32_t done = func(obj->obj, obj->mailbox);
3698 thread_ip_inc_cond(t, done);
3699 thread_yield_cond(p, done ^ 1);
3703 instr_extern_func_exec(struct rte_swx_pipeline *p)
3705 struct thread *t = &p->threads[p->thread_id];
3706 struct instruction *ip = t->ip;
3707 uint32_t ext_func_id = ip->ext_func.ext_func_id;
3708 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3709 rte_swx_extern_func_t func = ext_func->func;
3711 TRACE("[Thread %2u] extern func %u\n",
3715 /* Extern function execute. */
3716 uint32_t done = func(ext_func->mailbox);
3719 thread_ip_inc_cond(t, done);
3720 thread_yield_cond(p, done ^ 1);
3727 instr_mov_translate(struct rte_swx_pipeline *p,
3728 struct action *action,
3731 struct instruction *instr,
3732 struct instruction_data *data __rte_unused)
3734 char *dst = tokens[1], *src = tokens[2];
3735 struct field *fdst, *fsrc;
3737 uint32_t dst_struct_id = 0, src_struct_id = 0;
3739 CHECK(n_tokens == 3, EINVAL);
3741 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3742 CHECK(fdst, EINVAL);
3743 CHECK(!fdst->var_size, EINVAL);
3745 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3746 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3748 CHECK(!fsrc->var_size, EINVAL);
3750 instr->type = INSTR_MOV;
3751 if (dst[0] != 'h' && src[0] == 'h')
3752 instr->type = INSTR_MOV_MH;
3753 if (dst[0] == 'h' && src[0] != 'h')
3754 instr->type = INSTR_MOV_HM;
3755 if (dst[0] == 'h' && src[0] == 'h')
3756 instr->type = INSTR_MOV_HH;
3758 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3759 instr->mov.dst.n_bits = fdst->n_bits;
3760 instr->mov.dst.offset = fdst->offset / 8;
3761 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3762 instr->mov.src.n_bits = fsrc->n_bits;
3763 instr->mov.src.offset = fsrc->offset / 8;
3768 src_val = strtoull(src, &src, 0);
3769 CHECK(!src[0], EINVAL);
3772 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3774 instr->type = INSTR_MOV_I;
3775 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3776 instr->mov.dst.n_bits = fdst->n_bits;
3777 instr->mov.dst.offset = fdst->offset / 8;
3778 instr->mov.src_val = src_val;
3783 instr_mov_exec(struct rte_swx_pipeline *p)
3785 struct thread *t = &p->threads[p->thread_id];
3786 struct instruction *ip = t->ip;
3788 TRACE("[Thread %2u] mov\n",
3798 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3800 struct thread *t = &p->threads[p->thread_id];
3801 struct instruction *ip = t->ip;
3803 TRACE("[Thread %2u] mov (mh)\n",
3813 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3815 struct thread *t = &p->threads[p->thread_id];
3816 struct instruction *ip = t->ip;
3818 TRACE("[Thread %2u] mov (hm)\n",
3828 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3830 struct thread *t = &p->threads[p->thread_id];
3831 struct instruction *ip = t->ip;
3833 TRACE("[Thread %2u] mov (hh)\n",
3843 instr_mov_i_exec(struct rte_swx_pipeline *p)
3845 struct thread *t = &p->threads[p->thread_id];
3846 struct instruction *ip = t->ip;
3848 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3862 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3865 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3867 struct thread *t = &p->threads[p->thread_id];
3868 struct instruction *ip = t->ip;
3869 uint8_t *action_data = t->structs[0];
3870 uint64_t valid_headers = t->valid_headers;
3873 for (i = 0; i < n_dma; i++) {
3874 uint32_t header_id = ip->dma.dst.header_id[i];
3875 uint32_t struct_id = ip->dma.dst.struct_id[i];
3876 uint32_t offset = ip->dma.src.offset[i];
3877 uint32_t n_bytes = ip->dma.n_bytes[i];
3879 struct header_runtime *h = &t->headers[header_id];
3880 uint8_t *h_ptr0 = h->ptr0;
3881 uint8_t *h_ptr = t->structs[struct_id];
3883 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3885 void *src = &action_data[offset];
3887 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3890 memcpy(dst, src, n_bytes);
3891 t->structs[struct_id] = dst;
3892 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3895 t->valid_headers = valid_headers;
3899 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3901 __instr_dma_ht_exec(p, 1);
3908 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3910 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3913 __instr_dma_ht_exec(p, 2);
3920 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3922 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3925 __instr_dma_ht_exec(p, 3);
3932 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3934 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3937 __instr_dma_ht_exec(p, 4);
3944 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3946 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3949 __instr_dma_ht_exec(p, 5);
3956 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3958 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3961 __instr_dma_ht_exec(p, 6);
3968 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3970 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3973 __instr_dma_ht_exec(p, 7);
3980 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3982 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3985 __instr_dma_ht_exec(p, 8);
3995 instr_alu_add_translate(struct rte_swx_pipeline *p,
3996 struct action *action,
3999 struct instruction *instr,
4000 struct instruction_data *data __rte_unused)
4002 char *dst = tokens[1], *src = tokens[2];
4003 struct field *fdst, *fsrc;
4005 uint32_t dst_struct_id = 0, src_struct_id = 0;
4007 CHECK(n_tokens == 3, EINVAL);
4009 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4010 CHECK(fdst, EINVAL);
4011 CHECK(!fdst->var_size, EINVAL);
4013 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
4014 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4016 CHECK(!fsrc->var_size, EINVAL);
4018 instr->type = INSTR_ALU_ADD;
4019 if (dst[0] == 'h' && src[0] != 'h')
4020 instr->type = INSTR_ALU_ADD_HM;
4021 if (dst[0] != 'h' && src[0] == 'h')
4022 instr->type = INSTR_ALU_ADD_MH;
4023 if (dst[0] == 'h' && src[0] == 'h')
4024 instr->type = INSTR_ALU_ADD_HH;
4026 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4027 instr->alu.dst.n_bits = fdst->n_bits;
4028 instr->alu.dst.offset = fdst->offset / 8;
4029 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4030 instr->alu.src.n_bits = fsrc->n_bits;
4031 instr->alu.src.offset = fsrc->offset / 8;
4035 /* ADD_MI, ADD_HI. */
4036 src_val = strtoull(src, &src, 0);
4037 CHECK(!src[0], EINVAL);
4039 instr->type = INSTR_ALU_ADD_MI;
4041 instr->type = INSTR_ALU_ADD_HI;
4043 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4044 instr->alu.dst.n_bits = fdst->n_bits;
4045 instr->alu.dst.offset = fdst->offset / 8;
4046 instr->alu.src_val = src_val;
4051 instr_alu_sub_translate(struct rte_swx_pipeline *p,
4052 struct action *action,
4055 struct instruction *instr,
4056 struct instruction_data *data __rte_unused)
4058 char *dst = tokens[1], *src = tokens[2];
4059 struct field *fdst, *fsrc;
4061 uint32_t dst_struct_id = 0, src_struct_id = 0;
4063 CHECK(n_tokens == 3, EINVAL);
4065 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4066 CHECK(fdst, EINVAL);
4067 CHECK(!fdst->var_size, EINVAL);
4069 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
4070 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4072 CHECK(!fsrc->var_size, EINVAL);
4074 instr->type = INSTR_ALU_SUB;
4075 if (dst[0] == 'h' && src[0] != 'h')
4076 instr->type = INSTR_ALU_SUB_HM;
4077 if (dst[0] != 'h' && src[0] == 'h')
4078 instr->type = INSTR_ALU_SUB_MH;
4079 if (dst[0] == 'h' && src[0] == 'h')
4080 instr->type = INSTR_ALU_SUB_HH;
4082 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4083 instr->alu.dst.n_bits = fdst->n_bits;
4084 instr->alu.dst.offset = fdst->offset / 8;
4085 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4086 instr->alu.src.n_bits = fsrc->n_bits;
4087 instr->alu.src.offset = fsrc->offset / 8;
4091 /* SUB_MI, SUB_HI. */
4092 src_val = strtoull(src, &src, 0);
4093 CHECK(!src[0], EINVAL);
4095 instr->type = INSTR_ALU_SUB_MI;
4097 instr->type = INSTR_ALU_SUB_HI;
4099 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4100 instr->alu.dst.n_bits = fdst->n_bits;
4101 instr->alu.dst.offset = fdst->offset / 8;
4102 instr->alu.src_val = src_val;
4107 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4108 struct action *action __rte_unused,
4111 struct instruction *instr,
4112 struct instruction_data *data __rte_unused)
4114 char *dst = tokens[1], *src = tokens[2];
4115 struct header *hdst, *hsrc;
4116 struct field *fdst, *fsrc;
4118 CHECK(n_tokens == 3, EINVAL);
4120 fdst = header_field_parse(p, dst, &hdst);
4121 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4122 CHECK(!fdst->var_size, EINVAL);
4125 fsrc = header_field_parse(p, src, &hsrc);
4127 CHECK(!fsrc->var_size, EINVAL);
4129 instr->type = INSTR_ALU_CKADD_FIELD;
4130 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4131 instr->alu.dst.n_bits = fdst->n_bits;
4132 instr->alu.dst.offset = fdst->offset / 8;
4133 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4134 instr->alu.src.n_bits = fsrc->n_bits;
4135 instr->alu.src.offset = fsrc->offset / 8;
4139 /* CKADD_STRUCT, CKADD_STRUCT20. */
4140 hsrc = header_parse(p, src);
4141 CHECK(hsrc, EINVAL);
4142 CHECK(!hsrc->st->var_size, EINVAL);
4144 instr->type = INSTR_ALU_CKADD_STRUCT;
4145 if ((hsrc->st->n_bits / 8) == 20)
4146 instr->type = INSTR_ALU_CKADD_STRUCT20;
4148 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4149 instr->alu.dst.n_bits = fdst->n_bits;
4150 instr->alu.dst.offset = fdst->offset / 8;
4151 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4152 instr->alu.src.n_bits = hsrc->st->n_bits;
4153 instr->alu.src.offset = 0; /* Unused. */
4158 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4159 struct action *action __rte_unused,
4162 struct instruction *instr,
4163 struct instruction_data *data __rte_unused)
4165 char *dst = tokens[1], *src = tokens[2];
4166 struct header *hdst, *hsrc;
4167 struct field *fdst, *fsrc;
4169 CHECK(n_tokens == 3, EINVAL);
4171 fdst = header_field_parse(p, dst, &hdst);
4172 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4173 CHECK(!fdst->var_size, EINVAL);
4175 fsrc = header_field_parse(p, src, &hsrc);
4176 CHECK(fsrc, EINVAL);
4177 CHECK(!fsrc->var_size, EINVAL);
4179 instr->type = INSTR_ALU_CKSUB_FIELD;
4180 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4181 instr->alu.dst.n_bits = fdst->n_bits;
4182 instr->alu.dst.offset = fdst->offset / 8;
4183 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4184 instr->alu.src.n_bits = fsrc->n_bits;
4185 instr->alu.src.offset = fsrc->offset / 8;
4190 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4191 struct action *action,
4194 struct instruction *instr,
4195 struct instruction_data *data __rte_unused)
4197 char *dst = tokens[1], *src = tokens[2];
4198 struct field *fdst, *fsrc;
4200 uint32_t dst_struct_id = 0, src_struct_id = 0;
4202 CHECK(n_tokens == 3, EINVAL);
4204 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4205 CHECK(fdst, EINVAL);
4206 CHECK(!fdst->var_size, EINVAL);
4208 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4209 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4211 CHECK(!fsrc->var_size, EINVAL);
4213 instr->type = INSTR_ALU_SHL;
4214 if (dst[0] == 'h' && src[0] != 'h')
4215 instr->type = INSTR_ALU_SHL_HM;
4216 if (dst[0] != 'h' && src[0] == 'h')
4217 instr->type = INSTR_ALU_SHL_MH;
4218 if (dst[0] == 'h' && src[0] == 'h')
4219 instr->type = INSTR_ALU_SHL_HH;
4221 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4222 instr->alu.dst.n_bits = fdst->n_bits;
4223 instr->alu.dst.offset = fdst->offset / 8;
4224 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4225 instr->alu.src.n_bits = fsrc->n_bits;
4226 instr->alu.src.offset = fsrc->offset / 8;
4230 /* SHL_MI, SHL_HI. */
4231 src_val = strtoull(src, &src, 0);
4232 CHECK(!src[0], EINVAL);
4234 instr->type = INSTR_ALU_SHL_MI;
4236 instr->type = INSTR_ALU_SHL_HI;
4238 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4239 instr->alu.dst.n_bits = fdst->n_bits;
4240 instr->alu.dst.offset = fdst->offset / 8;
4241 instr->alu.src_val = src_val;
4246 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4247 struct action *action,
4250 struct instruction *instr,
4251 struct instruction_data *data __rte_unused)
4253 char *dst = tokens[1], *src = tokens[2];
4254 struct field *fdst, *fsrc;
4256 uint32_t dst_struct_id = 0, src_struct_id = 0;
4258 CHECK(n_tokens == 3, EINVAL);
4260 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4261 CHECK(fdst, EINVAL);
4262 CHECK(!fdst->var_size, EINVAL);
4264 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4265 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4267 CHECK(!fsrc->var_size, EINVAL);
4269 instr->type = INSTR_ALU_SHR;
4270 if (dst[0] == 'h' && src[0] != 'h')
4271 instr->type = INSTR_ALU_SHR_HM;
4272 if (dst[0] != 'h' && src[0] == 'h')
4273 instr->type = INSTR_ALU_SHR_MH;
4274 if (dst[0] == 'h' && src[0] == 'h')
4275 instr->type = INSTR_ALU_SHR_HH;
4277 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4278 instr->alu.dst.n_bits = fdst->n_bits;
4279 instr->alu.dst.offset = fdst->offset / 8;
4280 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4281 instr->alu.src.n_bits = fsrc->n_bits;
4282 instr->alu.src.offset = fsrc->offset / 8;
4286 /* SHR_MI, SHR_HI. */
4287 src_val = strtoull(src, &src, 0);
4288 CHECK(!src[0], EINVAL);
4290 instr->type = INSTR_ALU_SHR_MI;
4292 instr->type = INSTR_ALU_SHR_HI;
4294 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4295 instr->alu.dst.n_bits = fdst->n_bits;
4296 instr->alu.dst.offset = fdst->offset / 8;
4297 instr->alu.src_val = src_val;
4302 instr_alu_and_translate(struct rte_swx_pipeline *p,
4303 struct action *action,
4306 struct instruction *instr,
4307 struct instruction_data *data __rte_unused)
4309 char *dst = tokens[1], *src = tokens[2];
4310 struct field *fdst, *fsrc;
4312 uint32_t dst_struct_id = 0, src_struct_id = 0;
4314 CHECK(n_tokens == 3, EINVAL);
4316 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4317 CHECK(fdst, EINVAL);
4318 CHECK(!fdst->var_size, EINVAL);
4320 /* AND, AND_MH, AND_HM, AND_HH. */
4321 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4323 CHECK(!fsrc->var_size, EINVAL);
4325 instr->type = INSTR_ALU_AND;
4326 if (dst[0] != 'h' && src[0] == 'h')
4327 instr->type = INSTR_ALU_AND_MH;
4328 if (dst[0] == 'h' && src[0] != 'h')
4329 instr->type = INSTR_ALU_AND_HM;
4330 if (dst[0] == 'h' && src[0] == 'h')
4331 instr->type = INSTR_ALU_AND_HH;
4333 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4334 instr->alu.dst.n_bits = fdst->n_bits;
4335 instr->alu.dst.offset = fdst->offset / 8;
4336 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4337 instr->alu.src.n_bits = fsrc->n_bits;
4338 instr->alu.src.offset = fsrc->offset / 8;
4343 src_val = strtoull(src, &src, 0);
4344 CHECK(!src[0], EINVAL);
4347 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4349 instr->type = INSTR_ALU_AND_I;
4350 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4351 instr->alu.dst.n_bits = fdst->n_bits;
4352 instr->alu.dst.offset = fdst->offset / 8;
4353 instr->alu.src_val = src_val;
4358 instr_alu_or_translate(struct rte_swx_pipeline *p,
4359 struct action *action,
4362 struct instruction *instr,
4363 struct instruction_data *data __rte_unused)
4365 char *dst = tokens[1], *src = tokens[2];
4366 struct field *fdst, *fsrc;
4368 uint32_t dst_struct_id = 0, src_struct_id = 0;
4370 CHECK(n_tokens == 3, EINVAL);
4372 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4373 CHECK(fdst, EINVAL);
4374 CHECK(!fdst->var_size, EINVAL);
4376 /* OR, OR_MH, OR_HM, OR_HH. */
4377 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4379 CHECK(!fsrc->var_size, EINVAL);
4381 instr->type = INSTR_ALU_OR;
4382 if (dst[0] != 'h' && src[0] == 'h')
4383 instr->type = INSTR_ALU_OR_MH;
4384 if (dst[0] == 'h' && src[0] != 'h')
4385 instr->type = INSTR_ALU_OR_HM;
4386 if (dst[0] == 'h' && src[0] == 'h')
4387 instr->type = INSTR_ALU_OR_HH;
4389 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4390 instr->alu.dst.n_bits = fdst->n_bits;
4391 instr->alu.dst.offset = fdst->offset / 8;
4392 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4393 instr->alu.src.n_bits = fsrc->n_bits;
4394 instr->alu.src.offset = fsrc->offset / 8;
4399 src_val = strtoull(src, &src, 0);
4400 CHECK(!src[0], EINVAL);
4403 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4405 instr->type = INSTR_ALU_OR_I;
4406 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4407 instr->alu.dst.n_bits = fdst->n_bits;
4408 instr->alu.dst.offset = fdst->offset / 8;
4409 instr->alu.src_val = src_val;
4414 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4415 struct action *action,
4418 struct instruction *instr,
4419 struct instruction_data *data __rte_unused)
4421 char *dst = tokens[1], *src = tokens[2];
4422 struct field *fdst, *fsrc;
4424 uint32_t dst_struct_id = 0, src_struct_id = 0;
4426 CHECK(n_tokens == 3, EINVAL);
4428 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4429 CHECK(fdst, EINVAL);
4430 CHECK(!fdst->var_size, EINVAL);
4432 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4433 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4435 CHECK(!fsrc->var_size, EINVAL);
4437 instr->type = INSTR_ALU_XOR;
4438 if (dst[0] != 'h' && src[0] == 'h')
4439 instr->type = INSTR_ALU_XOR_MH;
4440 if (dst[0] == 'h' && src[0] != 'h')
4441 instr->type = INSTR_ALU_XOR_HM;
4442 if (dst[0] == 'h' && src[0] == 'h')
4443 instr->type = INSTR_ALU_XOR_HH;
4445 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4446 instr->alu.dst.n_bits = fdst->n_bits;
4447 instr->alu.dst.offset = fdst->offset / 8;
4448 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4449 instr->alu.src.n_bits = fsrc->n_bits;
4450 instr->alu.src.offset = fsrc->offset / 8;
4455 src_val = strtoull(src, &src, 0);
4456 CHECK(!src[0], EINVAL);
4459 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4461 instr->type = INSTR_ALU_XOR_I;
4462 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4463 instr->alu.dst.n_bits = fdst->n_bits;
4464 instr->alu.dst.offset = fdst->offset / 8;
4465 instr->alu.src_val = src_val;
4470 instr_alu_add_exec(struct rte_swx_pipeline *p)
4472 struct thread *t = &p->threads[p->thread_id];
4473 struct instruction *ip = t->ip;
4475 TRACE("[Thread %2u] add\n", p->thread_id);
4485 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4487 struct thread *t = &p->threads[p->thread_id];
4488 struct instruction *ip = t->ip;
4490 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4500 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4502 struct thread *t = &p->threads[p->thread_id];
4503 struct instruction *ip = t->ip;
4505 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4515 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4517 struct thread *t = &p->threads[p->thread_id];
4518 struct instruction *ip = t->ip;
4520 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4530 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4532 struct thread *t = &p->threads[p->thread_id];
4533 struct instruction *ip = t->ip;
4535 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4545 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4547 struct thread *t = &p->threads[p->thread_id];
4548 struct instruction *ip = t->ip;
4550 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4560 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4562 struct thread *t = &p->threads[p->thread_id];
4563 struct instruction *ip = t->ip;
4565 TRACE("[Thread %2u] sub\n", p->thread_id);
4575 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4577 struct thread *t = &p->threads[p->thread_id];
4578 struct instruction *ip = t->ip;
4580 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4590 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4592 struct thread *t = &p->threads[p->thread_id];
4593 struct instruction *ip = t->ip;
4595 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4605 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4607 struct thread *t = &p->threads[p->thread_id];
4608 struct instruction *ip = t->ip;
4610 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4620 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4622 struct thread *t = &p->threads[p->thread_id];
4623 struct instruction *ip = t->ip;
4625 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4635 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4637 struct thread *t = &p->threads[p->thread_id];
4638 struct instruction *ip = t->ip;
4640 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4650 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4652 struct thread *t = &p->threads[p->thread_id];
4653 struct instruction *ip = t->ip;
4655 TRACE("[Thread %2u] shl\n", p->thread_id);
4665 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4667 struct thread *t = &p->threads[p->thread_id];
4668 struct instruction *ip = t->ip;
4670 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4680 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4682 struct thread *t = &p->threads[p->thread_id];
4683 struct instruction *ip = t->ip;
4685 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4695 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4697 struct thread *t = &p->threads[p->thread_id];
4698 struct instruction *ip = t->ip;
4700 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4710 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4712 struct thread *t = &p->threads[p->thread_id];
4713 struct instruction *ip = t->ip;
4715 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4725 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4727 struct thread *t = &p->threads[p->thread_id];
4728 struct instruction *ip = t->ip;
4730 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4740 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4742 struct thread *t = &p->threads[p->thread_id];
4743 struct instruction *ip = t->ip;
4745 TRACE("[Thread %2u] shr\n", p->thread_id);
4755 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4757 struct thread *t = &p->threads[p->thread_id];
4758 struct instruction *ip = t->ip;
4760 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4770 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4772 struct thread *t = &p->threads[p->thread_id];
4773 struct instruction *ip = t->ip;
4775 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4785 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4787 struct thread *t = &p->threads[p->thread_id];
4788 struct instruction *ip = t->ip;
4790 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4800 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4802 struct thread *t = &p->threads[p->thread_id];
4803 struct instruction *ip = t->ip;
4805 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4815 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4817 struct thread *t = &p->threads[p->thread_id];
4818 struct instruction *ip = t->ip;
4820 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4830 instr_alu_and_exec(struct rte_swx_pipeline *p)
4832 struct thread *t = &p->threads[p->thread_id];
4833 struct instruction *ip = t->ip;
4835 TRACE("[Thread %2u] and\n", p->thread_id);
4845 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4847 struct thread *t = &p->threads[p->thread_id];
4848 struct instruction *ip = t->ip;
4850 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4860 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4862 struct thread *t = &p->threads[p->thread_id];
4863 struct instruction *ip = t->ip;
4865 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4868 ALU_HM_FAST(t, ip, &);
4875 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4877 struct thread *t = &p->threads[p->thread_id];
4878 struct instruction *ip = t->ip;
4880 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4883 ALU_HH_FAST(t, ip, &);
4890 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4892 struct thread *t = &p->threads[p->thread_id];
4893 struct instruction *ip = t->ip;
4895 TRACE("[Thread %2u] and (i)\n", p->thread_id);
4905 instr_alu_or_exec(struct rte_swx_pipeline *p)
4907 struct thread *t = &p->threads[p->thread_id];
4908 struct instruction *ip = t->ip;
4910 TRACE("[Thread %2u] or\n", p->thread_id);
4920 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4922 struct thread *t = &p->threads[p->thread_id];
4923 struct instruction *ip = t->ip;
4925 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
4935 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4937 struct thread *t = &p->threads[p->thread_id];
4938 struct instruction *ip = t->ip;
4940 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
4943 ALU_HM_FAST(t, ip, |);
4950 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4952 struct thread *t = &p->threads[p->thread_id];
4953 struct instruction *ip = t->ip;
4955 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
4958 ALU_HH_FAST(t, ip, |);
4965 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4967 struct thread *t = &p->threads[p->thread_id];
4968 struct instruction *ip = t->ip;
4970 TRACE("[Thread %2u] or (i)\n", p->thread_id);
4980 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4982 struct thread *t = &p->threads[p->thread_id];
4983 struct instruction *ip = t->ip;
4985 TRACE("[Thread %2u] xor\n", p->thread_id);
4995 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4997 struct thread *t = &p->threads[p->thread_id];
4998 struct instruction *ip = t->ip;
5000 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
5010 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
5012 struct thread *t = &p->threads[p->thread_id];
5013 struct instruction *ip = t->ip;
5015 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
5018 ALU_HM_FAST(t, ip, ^);
5025 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
5027 struct thread *t = &p->threads[p->thread_id];
5028 struct instruction *ip = t->ip;
5030 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
5033 ALU_HH_FAST(t, ip, ^);
5040 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
5042 struct thread *t = &p->threads[p->thread_id];
5043 struct instruction *ip = t->ip;
5045 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
5055 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
5057 struct thread *t = &p->threads[p->thread_id];
5058 struct instruction *ip = t->ip;
5059 uint8_t *dst_struct, *src_struct;
5060 uint16_t *dst16_ptr, dst;
5061 uint64_t *src64_ptr, src64, src64_mask, src;
5064 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
5067 dst_struct = t->structs[ip->alu.dst.struct_id];
5068 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5071 src_struct = t->structs[ip->alu.src.struct_id];
5072 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5074 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5075 src = src64 & src64_mask;
5080 /* The first input (r) is a 16-bit number. The second and the third
5081 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
5082 * three numbers (output r) is a 34-bit number.
5084 r += (src >> 32) + (src & 0xFFFFFFFF);
5086 /* The first input is a 16-bit number. The second input is an 18-bit
5087 * number. In the worst case scenario, the sum of the two numbers is a
5090 r = (r & 0xFFFF) + (r >> 16);
5092 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5093 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
5095 r = (r & 0xFFFF) + (r >> 16);
5097 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5098 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5099 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
5100 * therefore the output r is always a 16-bit number.
5102 r = (r & 0xFFFF) + (r >> 16);
5107 *dst16_ptr = (uint16_t)r;
5114 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
5116 struct thread *t = &p->threads[p->thread_id];
5117 struct instruction *ip = t->ip;
5118 uint8_t *dst_struct, *src_struct;
5119 uint16_t *dst16_ptr, dst;
5120 uint64_t *src64_ptr, src64, src64_mask, src;
5123 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5126 dst_struct = t->structs[ip->alu.dst.struct_id];
5127 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5130 src_struct = t->structs[ip->alu.src.struct_id];
5131 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5133 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5134 src = src64 & src64_mask;
5139 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5140 * the following sequence of operations in 2's complement arithmetic:
5141 * a '- b = (a - b) % 0xFFFF.
5143 * In order to prevent an underflow for the below subtraction, in which
5144 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5145 * minuend), we first add a multiple of the 0xFFFF modulus to the
5146 * minuend. The number we add to the minuend needs to be a 34-bit number
5147 * or higher, so for readability reasons we picked the 36-bit multiple.
5148 * We are effectively turning the 16-bit minuend into a 36-bit number:
5149 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5151 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5153 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5154 * result (the output r) is a 36-bit number.
5156 r -= (src >> 32) + (src & 0xFFFFFFFF);
5158 /* The first input is a 16-bit number. The second input is a 20-bit
5159 * number. Their sum is a 21-bit number.
5161 r = (r & 0xFFFF) + (r >> 16);
5163 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5164 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5166 r = (r & 0xFFFF) + (r >> 16);
5168 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5169 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5170 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5171 * generated, therefore the output r is always a 16-bit number.
5173 r = (r & 0xFFFF) + (r >> 16);
5178 *dst16_ptr = (uint16_t)r;
5185 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5187 struct thread *t = &p->threads[p->thread_id];
5188 struct instruction *ip = t->ip;
5189 uint8_t *dst_struct, *src_struct;
5190 uint16_t *dst16_ptr;
5191 uint32_t *src32_ptr;
5194 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5197 dst_struct = t->structs[ip->alu.dst.struct_id];
5198 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5200 src_struct = t->structs[ip->alu.src.struct_id];
5201 src32_ptr = (uint32_t *)&src_struct[0];
5203 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5204 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5205 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5206 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5207 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5209 /* The first input is a 16-bit number. The second input is a 19-bit
5210 * number. Their sum is a 20-bit number.
5212 r0 = (r0 & 0xFFFF) + (r0 >> 16);
5214 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5215 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5217 r0 = (r0 & 0xFFFF) + (r0 >> 16);
5219 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5220 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5221 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5222 * generated, therefore the output r is always a 16-bit number.
5224 r0 = (r0 & 0xFFFF) + (r0 >> 16);
5227 r0 = r0 ? r0 : 0xFFFF;
5229 *dst16_ptr = (uint16_t)r0;
5236 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5238 struct thread *t = &p->threads[p->thread_id];
5239 struct instruction *ip = t->ip;
5240 uint8_t *dst_struct, *src_struct;
5241 uint16_t *dst16_ptr;
5242 uint32_t *src32_ptr;
5246 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5249 dst_struct = t->structs[ip->alu.dst.struct_id];
5250 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5252 src_struct = t->structs[ip->alu.src.struct_id];
5253 src32_ptr = (uint32_t *)&src_struct[0];
5255 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5256 * Therefore, in the worst case scenario, a 35-bit number is added to a
5257 * 16-bit number (the input r), so the output r is 36-bit number.
5259 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5262 /* The first input is a 16-bit number. The second input is a 20-bit
5263 * number. Their sum is a 21-bit number.
5265 r = (r & 0xFFFF) + (r >> 16);
5267 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5268 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5270 r = (r & 0xFFFF) + (r >> 16);
5272 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5273 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5274 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5275 * generated, therefore the output r is always a 16-bit number.
5277 r = (r & 0xFFFF) + (r >> 16);
5282 *dst16_ptr = (uint16_t)r;
5291 static struct regarray *
5292 regarray_find(struct rte_swx_pipeline *p, const char *name);
5295 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5296 struct action *action,
5299 struct instruction *instr,
5300 struct instruction_data *data __rte_unused)
5302 char *regarray = tokens[1], *idx = tokens[2];
5305 uint32_t idx_struct_id, idx_val;
5307 CHECK(n_tokens == 3, EINVAL);
5309 r = regarray_find(p, regarray);
5312 /* REGPREFETCH_RH, REGPREFETCH_RM. */
5313 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5315 CHECK(!fidx->var_size, EINVAL);
5317 instr->type = INSTR_REGPREFETCH_RM;
5319 instr->type = INSTR_REGPREFETCH_RH;
5321 instr->regarray.regarray_id = r->id;
5322 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5323 instr->regarray.idx.n_bits = fidx->n_bits;
5324 instr->regarray.idx.offset = fidx->offset / 8;
5325 instr->regarray.dstsrc_val = 0; /* Unused. */
5329 /* REGPREFETCH_RI. */
5330 idx_val = strtoul(idx, &idx, 0);
5331 CHECK(!idx[0], EINVAL);
5333 instr->type = INSTR_REGPREFETCH_RI;
5334 instr->regarray.regarray_id = r->id;
5335 instr->regarray.idx_val = idx_val;
5336 instr->regarray.dstsrc_val = 0; /* Unused. */
5341 instr_regrd_translate(struct rte_swx_pipeline *p,
5342 struct action *action,
5345 struct instruction *instr,
5346 struct instruction_data *data __rte_unused)
5348 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5350 struct field *fdst, *fidx;
5351 uint32_t dst_struct_id, idx_struct_id, idx_val;
5353 CHECK(n_tokens == 4, EINVAL);
5355 r = regarray_find(p, regarray);
5358 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5359 CHECK(fdst, EINVAL);
5360 CHECK(!fdst->var_size, EINVAL);
5362 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5363 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5365 CHECK(!fidx->var_size, EINVAL);
5367 instr->type = INSTR_REGRD_MRM;
5368 if (dst[0] == 'h' && idx[0] != 'h')
5369 instr->type = INSTR_REGRD_HRM;
5370 if (dst[0] != 'h' && idx[0] == 'h')
5371 instr->type = INSTR_REGRD_MRH;
5372 if (dst[0] == 'h' && idx[0] == 'h')
5373 instr->type = INSTR_REGRD_HRH;
5375 instr->regarray.regarray_id = r->id;
5376 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5377 instr->regarray.idx.n_bits = fidx->n_bits;
5378 instr->regarray.idx.offset = fidx->offset / 8;
5379 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5380 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5381 instr->regarray.dstsrc.offset = fdst->offset / 8;
5385 /* REGRD_MRI, REGRD_HRI. */
5386 idx_val = strtoul(idx, &idx, 0);
5387 CHECK(!idx[0], EINVAL);
5389 instr->type = INSTR_REGRD_MRI;
5391 instr->type = INSTR_REGRD_HRI;
5393 instr->regarray.regarray_id = r->id;
5394 instr->regarray.idx_val = idx_val;
5395 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5396 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5397 instr->regarray.dstsrc.offset = fdst->offset / 8;
5402 instr_regwr_translate(struct rte_swx_pipeline *p,
5403 struct action *action,
5406 struct instruction *instr,
5407 struct instruction_data *data __rte_unused)
5409 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5411 struct field *fidx, *fsrc;
5413 uint32_t idx_struct_id, idx_val, src_struct_id;
5415 CHECK(n_tokens == 4, EINVAL);
5417 r = regarray_find(p, regarray);
5420 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5421 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5422 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5424 CHECK(!fidx->var_size, EINVAL);
5425 CHECK(!fsrc->var_size, EINVAL);
5427 instr->type = INSTR_REGWR_RMM;
5428 if (idx[0] == 'h' && src[0] != 'h')
5429 instr->type = INSTR_REGWR_RHM;
5430 if (idx[0] != 'h' && src[0] == 'h')
5431 instr->type = INSTR_REGWR_RMH;
5432 if (idx[0] == 'h' && src[0] == 'h')
5433 instr->type = INSTR_REGWR_RHH;
5435 instr->regarray.regarray_id = r->id;
5436 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5437 instr->regarray.idx.n_bits = fidx->n_bits;
5438 instr->regarray.idx.offset = fidx->offset / 8;
5439 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5440 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5441 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5445 /* REGWR_RHI, REGWR_RMI. */
5446 if (fidx && !fsrc) {
5447 CHECK(!fidx->var_size, EINVAL);
5449 src_val = strtoull(src, &src, 0);
5450 CHECK(!src[0], EINVAL);
5452 instr->type = INSTR_REGWR_RMI;
5454 instr->type = INSTR_REGWR_RHI;
5456 instr->regarray.regarray_id = r->id;
5457 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5458 instr->regarray.idx.n_bits = fidx->n_bits;
5459 instr->regarray.idx.offset = fidx->offset / 8;
5460 instr->regarray.dstsrc_val = src_val;
5464 /* REGWR_RIH, REGWR_RIM. */
5465 if (!fidx && fsrc) {
5466 idx_val = strtoul(idx, &idx, 0);
5467 CHECK(!idx[0], EINVAL);
5469 CHECK(!fsrc->var_size, EINVAL);
5471 instr->type = INSTR_REGWR_RIM;
5473 instr->type = INSTR_REGWR_RIH;
5475 instr->regarray.regarray_id = r->id;
5476 instr->regarray.idx_val = idx_val;
5477 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5478 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5479 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5484 src_val = strtoull(src, &src, 0);
5485 CHECK(!src[0], EINVAL);
5487 idx_val = strtoul(idx, &idx, 0);
5488 CHECK(!idx[0], EINVAL);
5490 instr->type = INSTR_REGWR_RII;
5491 instr->regarray.idx_val = idx_val;
5492 instr->regarray.dstsrc_val = src_val;
5498 instr_regadd_translate(struct rte_swx_pipeline *p,
5499 struct action *action,
5502 struct instruction *instr,
5503 struct instruction_data *data __rte_unused)
5505 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5507 struct field *fidx, *fsrc;
5509 uint32_t idx_struct_id, idx_val, src_struct_id;
5511 CHECK(n_tokens == 4, EINVAL);
5513 r = regarray_find(p, regarray);
5516 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5517 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5518 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5520 CHECK(!fidx->var_size, EINVAL);
5521 CHECK(!fsrc->var_size, EINVAL);
5523 instr->type = INSTR_REGADD_RMM;
5524 if (idx[0] == 'h' && src[0] != 'h')
5525 instr->type = INSTR_REGADD_RHM;
5526 if (idx[0] != 'h' && src[0] == 'h')
5527 instr->type = INSTR_REGADD_RMH;
5528 if (idx[0] == 'h' && src[0] == 'h')
5529 instr->type = INSTR_REGADD_RHH;
5531 instr->regarray.regarray_id = r->id;
5532 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5533 instr->regarray.idx.n_bits = fidx->n_bits;
5534 instr->regarray.idx.offset = fidx->offset / 8;
5535 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5536 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5537 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5541 /* REGADD_RHI, REGADD_RMI. */
5542 if (fidx && !fsrc) {
5543 CHECK(!fidx->var_size, EINVAL);
5545 src_val = strtoull(src, &src, 0);
5546 CHECK(!src[0], EINVAL);
5548 instr->type = INSTR_REGADD_RMI;
5550 instr->type = INSTR_REGADD_RHI;
5552 instr->regarray.regarray_id = r->id;
5553 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5554 instr->regarray.idx.n_bits = fidx->n_bits;
5555 instr->regarray.idx.offset = fidx->offset / 8;
5556 instr->regarray.dstsrc_val = src_val;
5560 /* REGADD_RIH, REGADD_RIM. */
5561 if (!fidx && fsrc) {
5562 idx_val = strtoul(idx, &idx, 0);
5563 CHECK(!idx[0], EINVAL);
5565 CHECK(!fsrc->var_size, EINVAL);
5567 instr->type = INSTR_REGADD_RIM;
5569 instr->type = INSTR_REGADD_RIH;
5571 instr->regarray.regarray_id = r->id;
5572 instr->regarray.idx_val = idx_val;
5573 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5574 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5575 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5580 src_val = strtoull(src, &src, 0);
5581 CHECK(!src[0], EINVAL);
5583 idx_val = strtoul(idx, &idx, 0);
5584 CHECK(!idx[0], EINVAL);
5586 instr->type = INSTR_REGADD_RII;
5587 instr->regarray.idx_val = idx_val;
5588 instr->regarray.dstsrc_val = src_val;
5592 static inline uint64_t *
5593 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5595 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5599 static inline uint64_t
5600 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5602 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5604 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5605 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5606 uint64_t idx64 = *idx64_ptr;
5607 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5608 uint64_t idx = idx64 & idx64_mask & r->size_mask;
5613 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5615 static inline uint64_t
5616 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5618 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5620 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5621 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5622 uint64_t idx64 = *idx64_ptr;
5623 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5630 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5634 static inline uint64_t
5635 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5637 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5639 uint64_t idx = ip->regarray.idx_val & r->size_mask;
5644 static inline uint64_t
5645 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5647 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5648 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5649 uint64_t src64 = *src64_ptr;
5650 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5651 uint64_t src = src64 & src64_mask;
5656 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5658 static inline uint64_t
5659 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5661 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5662 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5663 uint64_t src64 = *src64_ptr;
5664 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5671 #define instr_regarray_src_nbo instr_regarray_src_hbo
5676 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5678 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5679 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5680 uint64_t dst64 = *dst64_ptr;
5681 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5683 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5687 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5690 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5692 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5693 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5694 uint64_t dst64 = *dst64_ptr;
5695 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5697 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5698 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5703 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5708 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5710 struct thread *t = &p->threads[p->thread_id];
5711 struct instruction *ip = t->ip;
5712 uint64_t *regarray, idx;
5714 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5717 regarray = instr_regarray_regarray(p, ip);
5718 idx = instr_regarray_idx_nbo(p, t, ip);
5719 rte_prefetch0(®array[idx]);
5726 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5728 struct thread *t = &p->threads[p->thread_id];
5729 struct instruction *ip = t->ip;
5730 uint64_t *regarray, idx;
5732 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5735 regarray = instr_regarray_regarray(p, ip);
5736 idx = instr_regarray_idx_hbo(p, t, ip);
5737 rte_prefetch0(®array[idx]);
5744 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5746 struct thread *t = &p->threads[p->thread_id];
5747 struct instruction *ip = t->ip;
5748 uint64_t *regarray, idx;
5750 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5753 regarray = instr_regarray_regarray(p, ip);
5754 idx = instr_regarray_idx_imm(p, ip);
5755 rte_prefetch0(®array[idx]);
5762 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5764 struct thread *t = &p->threads[p->thread_id];
5765 struct instruction *ip = t->ip;
5766 uint64_t *regarray, idx;
5768 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5771 regarray = instr_regarray_regarray(p, ip);
5772 idx = instr_regarray_idx_nbo(p, t, ip);
5773 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5780 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5782 struct thread *t = &p->threads[p->thread_id];
5783 struct instruction *ip = t->ip;
5784 uint64_t *regarray, idx;
5786 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5789 regarray = instr_regarray_regarray(p, ip);
5790 idx = instr_regarray_idx_hbo(p, t, ip);
5791 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5798 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5800 struct thread *t = &p->threads[p->thread_id];
5801 struct instruction *ip = t->ip;
5802 uint64_t *regarray, idx;
5804 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5807 regarray = instr_regarray_regarray(p, ip);
5808 idx = instr_regarray_idx_nbo(p, t, ip);
5809 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5816 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5818 struct thread *t = &p->threads[p->thread_id];
5819 struct instruction *ip = t->ip;
5820 uint64_t *regarray, idx;
5823 regarray = instr_regarray_regarray(p, ip);
5824 idx = instr_regarray_idx_hbo(p, t, ip);
5825 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5832 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5834 struct thread *t = &p->threads[p->thread_id];
5835 struct instruction *ip = t->ip;
5836 uint64_t *regarray, idx;
5838 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5841 regarray = instr_regarray_regarray(p, ip);
5842 idx = instr_regarray_idx_imm(p, ip);
5843 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5850 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5852 struct thread *t = &p->threads[p->thread_id];
5853 struct instruction *ip = t->ip;
5854 uint64_t *regarray, idx;
5856 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5859 regarray = instr_regarray_regarray(p, ip);
5860 idx = instr_regarray_idx_imm(p, ip);
5861 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5868 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5870 struct thread *t = &p->threads[p->thread_id];
5871 struct instruction *ip = t->ip;
5872 uint64_t *regarray, idx, src;
5874 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5877 regarray = instr_regarray_regarray(p, ip);
5878 idx = instr_regarray_idx_nbo(p, t, ip);
5879 src = instr_regarray_src_nbo(t, ip);
5880 regarray[idx] = src;
5887 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5889 struct thread *t = &p->threads[p->thread_id];
5890 struct instruction *ip = t->ip;
5891 uint64_t *regarray, idx, src;
5893 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5896 regarray = instr_regarray_regarray(p, ip);
5897 idx = instr_regarray_idx_nbo(p, t, ip);
5898 src = instr_regarray_src_hbo(t, ip);
5899 regarray[idx] = src;
5906 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5908 struct thread *t = &p->threads[p->thread_id];
5909 struct instruction *ip = t->ip;
5910 uint64_t *regarray, idx, src;
5912 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5915 regarray = instr_regarray_regarray(p, ip);
5916 idx = instr_regarray_idx_hbo(p, t, ip);
5917 src = instr_regarray_src_nbo(t, ip);
5918 regarray[idx] = src;
5925 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5927 struct thread *t = &p->threads[p->thread_id];
5928 struct instruction *ip = t->ip;
5929 uint64_t *regarray, idx, src;
5931 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5934 regarray = instr_regarray_regarray(p, ip);
5935 idx = instr_regarray_idx_hbo(p, t, ip);
5936 src = instr_regarray_src_hbo(t, ip);
5937 regarray[idx] = src;
5944 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5946 struct thread *t = &p->threads[p->thread_id];
5947 struct instruction *ip = t->ip;
5948 uint64_t *regarray, idx, src;
5950 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5953 regarray = instr_regarray_regarray(p, ip);
5954 idx = instr_regarray_idx_nbo(p, t, ip);
5955 src = ip->regarray.dstsrc_val;
5956 regarray[idx] = src;
5963 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5965 struct thread *t = &p->threads[p->thread_id];
5966 struct instruction *ip = t->ip;
5967 uint64_t *regarray, idx, src;
5969 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5972 regarray = instr_regarray_regarray(p, ip);
5973 idx = instr_regarray_idx_hbo(p, t, ip);
5974 src = ip->regarray.dstsrc_val;
5975 regarray[idx] = src;
5982 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5984 struct thread *t = &p->threads[p->thread_id];
5985 struct instruction *ip = t->ip;
5986 uint64_t *regarray, idx, src;
5988 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5991 regarray = instr_regarray_regarray(p, ip);
5992 idx = instr_regarray_idx_imm(p, ip);
5993 src = instr_regarray_src_nbo(t, ip);
5994 regarray[idx] = src;
6001 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
6003 struct thread *t = &p->threads[p->thread_id];
6004 struct instruction *ip = t->ip;
6005 uint64_t *regarray, idx, src;
6007 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
6010 regarray = instr_regarray_regarray(p, ip);
6011 idx = instr_regarray_idx_imm(p, ip);
6012 src = instr_regarray_src_hbo(t, ip);
6013 regarray[idx] = src;
6020 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
6022 struct thread *t = &p->threads[p->thread_id];
6023 struct instruction *ip = t->ip;
6024 uint64_t *regarray, idx, src;
6026 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
6029 regarray = instr_regarray_regarray(p, ip);
6030 idx = instr_regarray_idx_imm(p, ip);
6031 src = ip->regarray.dstsrc_val;
6032 regarray[idx] = src;
6039 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
6041 struct thread *t = &p->threads[p->thread_id];
6042 struct instruction *ip = t->ip;
6043 uint64_t *regarray, idx, src;
6045 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
6048 regarray = instr_regarray_regarray(p, ip);
6049 idx = instr_regarray_idx_nbo(p, t, ip);
6050 src = instr_regarray_src_nbo(t, ip);
6051 regarray[idx] += src;
6058 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
6060 struct thread *t = &p->threads[p->thread_id];
6061 struct instruction *ip = t->ip;
6062 uint64_t *regarray, idx, src;
6064 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
6067 regarray = instr_regarray_regarray(p, ip);
6068 idx = instr_regarray_idx_nbo(p, t, ip);
6069 src = instr_regarray_src_hbo(t, ip);
6070 regarray[idx] += src;
6077 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
6079 struct thread *t = &p->threads[p->thread_id];
6080 struct instruction *ip = t->ip;
6081 uint64_t *regarray, idx, src;
6083 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
6086 regarray = instr_regarray_regarray(p, ip);
6087 idx = instr_regarray_idx_hbo(p, t, ip);
6088 src = instr_regarray_src_nbo(t, ip);
6089 regarray[idx] += src;
6096 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
6098 struct thread *t = &p->threads[p->thread_id];
6099 struct instruction *ip = t->ip;
6100 uint64_t *regarray, idx, src;
6102 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
6105 regarray = instr_regarray_regarray(p, ip);
6106 idx = instr_regarray_idx_hbo(p, t, ip);
6107 src = instr_regarray_src_hbo(t, ip);
6108 regarray[idx] += src;
6115 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
6117 struct thread *t = &p->threads[p->thread_id];
6118 struct instruction *ip = t->ip;
6119 uint64_t *regarray, idx, src;
6121 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
6124 regarray = instr_regarray_regarray(p, ip);
6125 idx = instr_regarray_idx_nbo(p, t, ip);
6126 src = ip->regarray.dstsrc_val;
6127 regarray[idx] += src;
6134 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
6136 struct thread *t = &p->threads[p->thread_id];
6137 struct instruction *ip = t->ip;
6138 uint64_t *regarray, idx, src;
6140 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6143 regarray = instr_regarray_regarray(p, ip);
6144 idx = instr_regarray_idx_hbo(p, t, ip);
6145 src = ip->regarray.dstsrc_val;
6146 regarray[idx] += src;
6153 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6155 struct thread *t = &p->threads[p->thread_id];
6156 struct instruction *ip = t->ip;
6157 uint64_t *regarray, idx, src;
6159 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6162 regarray = instr_regarray_regarray(p, ip);
6163 idx = instr_regarray_idx_imm(p, ip);
6164 src = instr_regarray_src_nbo(t, ip);
6165 regarray[idx] += src;
6172 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6174 struct thread *t = &p->threads[p->thread_id];
6175 struct instruction *ip = t->ip;
6176 uint64_t *regarray, idx, src;
6178 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6181 regarray = instr_regarray_regarray(p, ip);
6182 idx = instr_regarray_idx_imm(p, ip);
6183 src = instr_regarray_src_hbo(t, ip);
6184 regarray[idx] += src;
6191 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6193 struct thread *t = &p->threads[p->thread_id];
6194 struct instruction *ip = t->ip;
6195 uint64_t *regarray, idx, src;
6197 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6200 regarray = instr_regarray_regarray(p, ip);
6201 idx = instr_regarray_idx_imm(p, ip);
6202 src = ip->regarray.dstsrc_val;
6203 regarray[idx] += src;
6212 static struct metarray *
6213 metarray_find(struct rte_swx_pipeline *p, const char *name);
6216 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6217 struct action *action,
6220 struct instruction *instr,
6221 struct instruction_data *data __rte_unused)
6223 char *metarray = tokens[1], *idx = tokens[2];
6226 uint32_t idx_struct_id, idx_val;
6228 CHECK(n_tokens == 3, EINVAL);
6230 m = metarray_find(p, metarray);
6233 /* METPREFETCH_H, METPREFETCH_M. */
6234 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6236 CHECK(!fidx->var_size, EINVAL);
6238 instr->type = INSTR_METPREFETCH_M;
6240 instr->type = INSTR_METPREFETCH_H;
6242 instr->meter.metarray_id = m->id;
6243 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6244 instr->meter.idx.n_bits = fidx->n_bits;
6245 instr->meter.idx.offset = fidx->offset / 8;
6249 /* METPREFETCH_I. */
6250 idx_val = strtoul(idx, &idx, 0);
6251 CHECK(!idx[0], EINVAL);
6253 instr->type = INSTR_METPREFETCH_I;
6254 instr->meter.metarray_id = m->id;
6255 instr->meter.idx_val = idx_val;
6260 instr_meter_translate(struct rte_swx_pipeline *p,
6261 struct action *action,
6264 struct instruction *instr,
6265 struct instruction_data *data __rte_unused)
6267 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6268 char *color_in = tokens[4], *color_out = tokens[5];
6270 struct field *fidx, *flength, *fcin, *fcout;
6271 uint32_t idx_struct_id, length_struct_id;
6272 uint32_t color_in_struct_id, color_out_struct_id;
6274 CHECK(n_tokens == 6, EINVAL);
6276 m = metarray_find(p, metarray);
6279 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6281 flength = struct_field_parse(p, action, length, &length_struct_id);
6282 CHECK(flength, EINVAL);
6283 CHECK(!flength->var_size, EINVAL);
6285 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6287 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6288 CHECK(fcout, EINVAL);
6289 CHECK(!fcout->var_size, EINVAL);
6291 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6293 CHECK(!fidx->var_size, EINVAL);
6294 CHECK(!fcin->var_size, EINVAL);
6296 instr->type = INSTR_METER_MMM;
6297 if (idx[0] == 'h' && length[0] == 'h')
6298 instr->type = INSTR_METER_HHM;
6299 if (idx[0] == 'h' && length[0] != 'h')
6300 instr->type = INSTR_METER_HMM;
6301 if (idx[0] != 'h' && length[0] == 'h')
6302 instr->type = INSTR_METER_MHM;
6304 instr->meter.metarray_id = m->id;
6306 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6307 instr->meter.idx.n_bits = fidx->n_bits;
6308 instr->meter.idx.offset = fidx->offset / 8;
6310 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6311 instr->meter.length.n_bits = flength->n_bits;
6312 instr->meter.length.offset = flength->offset / 8;
6314 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6315 instr->meter.color_in.n_bits = fcin->n_bits;
6316 instr->meter.color_in.offset = fcin->offset / 8;
6318 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6319 instr->meter.color_out.n_bits = fcout->n_bits;
6320 instr->meter.color_out.offset = fcout->offset / 8;
6325 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6326 if (fidx && !fcin) {
6327 uint32_t color_in_val;
6329 CHECK(!fidx->var_size, EINVAL);
6331 color_in_val = strtoul(color_in, &color_in, 0);
6332 CHECK(!color_in[0], EINVAL);
6334 instr->type = INSTR_METER_MMI;
6335 if (idx[0] == 'h' && length[0] == 'h')
6336 instr->type = INSTR_METER_HHI;
6337 if (idx[0] == 'h' && length[0] != 'h')
6338 instr->type = INSTR_METER_HMI;
6339 if (idx[0] != 'h' && length[0] == 'h')
6340 instr->type = INSTR_METER_MHI;
6342 instr->meter.metarray_id = m->id;
6344 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6345 instr->meter.idx.n_bits = fidx->n_bits;
6346 instr->meter.idx.offset = fidx->offset / 8;
6348 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6349 instr->meter.length.n_bits = flength->n_bits;
6350 instr->meter.length.offset = flength->offset / 8;
6352 instr->meter.color_in_val = color_in_val;
6354 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6355 instr->meter.color_out.n_bits = fcout->n_bits;
6356 instr->meter.color_out.offset = fcout->offset / 8;
6361 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6362 if (!fidx && fcin) {
6365 idx_val = strtoul(idx, &idx, 0);
6366 CHECK(!idx[0], EINVAL);
6368 CHECK(!fcin->var_size, EINVAL);
6370 instr->type = INSTR_METER_IMM;
6371 if (length[0] == 'h')
6372 instr->type = INSTR_METER_IHM;
6374 instr->meter.metarray_id = m->id;
6376 instr->meter.idx_val = idx_val;
6378 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6379 instr->meter.length.n_bits = flength->n_bits;
6380 instr->meter.length.offset = flength->offset / 8;
6382 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6383 instr->meter.color_in.n_bits = fcin->n_bits;
6384 instr->meter.color_in.offset = fcin->offset / 8;
6386 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6387 instr->meter.color_out.n_bits = fcout->n_bits;
6388 instr->meter.color_out.offset = fcout->offset / 8;
6393 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6394 if (!fidx && !fcin) {
6395 uint32_t idx_val, color_in_val;
6397 idx_val = strtoul(idx, &idx, 0);
6398 CHECK(!idx[0], EINVAL);
6400 color_in_val = strtoul(color_in, &color_in, 0);
6401 CHECK(!color_in[0], EINVAL);
6403 instr->type = INSTR_METER_IMI;
6404 if (length[0] == 'h')
6405 instr->type = INSTR_METER_IHI;
6407 instr->meter.metarray_id = m->id;
6409 instr->meter.idx_val = idx_val;
6411 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6412 instr->meter.length.n_bits = flength->n_bits;
6413 instr->meter.length.offset = flength->offset / 8;
6415 instr->meter.color_in_val = color_in_val;
6417 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6418 instr->meter.color_out.n_bits = fcout->n_bits;
6419 instr->meter.color_out.offset = fcout->offset / 8;
6427 static inline struct meter *
6428 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6430 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6432 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6433 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6434 uint64_t idx64 = *idx64_ptr;
6435 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6436 uint64_t idx = idx64 & idx64_mask & r->size_mask;
6438 return &r->metarray[idx];
6441 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6443 static inline struct meter *
6444 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6446 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6448 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6449 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6450 uint64_t idx64 = *idx64_ptr;
6451 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6453 return &r->metarray[idx];
6458 #define instr_meter_idx_nbo instr_meter_idx_hbo
6462 static inline struct meter *
6463 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6465 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6467 uint64_t idx = ip->meter.idx_val & r->size_mask;
6469 return &r->metarray[idx];
6472 static inline uint32_t
6473 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6475 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6476 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6477 uint64_t src64 = *src64_ptr;
6478 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6479 uint64_t src = src64 & src64_mask;
6481 return (uint32_t)src;
6484 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6486 static inline uint32_t
6487 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6489 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6490 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6491 uint64_t src64 = *src64_ptr;
6492 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6494 return (uint32_t)src;
6499 #define instr_meter_length_nbo instr_meter_length_hbo
6503 static inline enum rte_color
6504 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6506 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6507 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6508 uint64_t src64 = *src64_ptr;
6509 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6510 uint64_t src = src64 & src64_mask;
6512 return (enum rte_color)src;
6516 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6518 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6519 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6520 uint64_t dst64 = *dst64_ptr;
6521 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6523 uint64_t src = (uint64_t)color_out;
6525 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6529 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6531 struct thread *t = &p->threads[p->thread_id];
6532 struct instruction *ip = t->ip;
6535 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6538 m = instr_meter_idx_nbo(p, t, ip);
6546 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6548 struct thread *t = &p->threads[p->thread_id];
6549 struct instruction *ip = t->ip;
6552 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6555 m = instr_meter_idx_hbo(p, t, ip);
6563 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6565 struct thread *t = &p->threads[p->thread_id];
6566 struct instruction *ip = t->ip;
6569 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6572 m = instr_meter_idx_imm(p, ip);
6580 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6582 struct thread *t = &p->threads[p->thread_id];
6583 struct instruction *ip = t->ip;
6585 uint64_t time, n_pkts, n_bytes;
6587 enum rte_color color_in, color_out;
6589 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6592 m = instr_meter_idx_nbo(p, t, ip);
6593 rte_prefetch0(m->n_pkts);
6594 time = rte_get_tsc_cycles();
6595 length = instr_meter_length_nbo(t, ip);
6596 color_in = instr_meter_color_in_hbo(t, ip);
6598 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6599 &m->profile->profile,
6604 color_out &= m->color_mask;
6606 n_pkts = m->n_pkts[color_out];
6607 n_bytes = m->n_bytes[color_out];
6609 instr_meter_color_out_hbo_set(t, ip, color_out);
6611 m->n_pkts[color_out] = n_pkts + 1;
6612 m->n_bytes[color_out] = n_bytes + length;
6619 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6621 struct thread *t = &p->threads[p->thread_id];
6622 struct instruction *ip = t->ip;
6624 uint64_t time, n_pkts, n_bytes;
6626 enum rte_color color_in, color_out;
6628 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6631 m = instr_meter_idx_nbo(p, t, ip);
6632 rte_prefetch0(m->n_pkts);
6633 time = rte_get_tsc_cycles();
6634 length = instr_meter_length_nbo(t, ip);
6635 color_in = (enum rte_color)ip->meter.color_in_val;
6637 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6638 &m->profile->profile,
6643 color_out &= m->color_mask;
6645 n_pkts = m->n_pkts[color_out];
6646 n_bytes = m->n_bytes[color_out];
6648 instr_meter_color_out_hbo_set(t, ip, color_out);
6650 m->n_pkts[color_out] = n_pkts + 1;
6651 m->n_bytes[color_out] = n_bytes + length;
6658 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6660 struct thread *t = &p->threads[p->thread_id];
6661 struct instruction *ip = t->ip;
6663 uint64_t time, n_pkts, n_bytes;
6665 enum rte_color color_in, color_out;
6667 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6670 m = instr_meter_idx_nbo(p, t, ip);
6671 rte_prefetch0(m->n_pkts);
6672 time = rte_get_tsc_cycles();
6673 length = instr_meter_length_hbo(t, ip);
6674 color_in = instr_meter_color_in_hbo(t, ip);
6676 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6677 &m->profile->profile,
6682 color_out &= m->color_mask;
6684 n_pkts = m->n_pkts[color_out];
6685 n_bytes = m->n_bytes[color_out];
6687 instr_meter_color_out_hbo_set(t, ip, color_out);
6689 m->n_pkts[color_out] = n_pkts + 1;
6690 m->n_bytes[color_out] = n_bytes + length;
6696 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6698 struct thread *t = &p->threads[p->thread_id];
6699 struct instruction *ip = t->ip;
6701 uint64_t time, n_pkts, n_bytes;
6703 enum rte_color color_in, color_out;
6705 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6708 m = instr_meter_idx_nbo(p, t, ip);
6709 rte_prefetch0(m->n_pkts);
6710 time = rte_get_tsc_cycles();
6711 length = instr_meter_length_hbo(t, ip);
6712 color_in = (enum rte_color)ip->meter.color_in_val;
6714 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6715 &m->profile->profile,
6720 color_out &= m->color_mask;
6722 n_pkts = m->n_pkts[color_out];
6723 n_bytes = m->n_bytes[color_out];
6725 instr_meter_color_out_hbo_set(t, ip, color_out);
6727 m->n_pkts[color_out] = n_pkts + 1;
6728 m->n_bytes[color_out] = n_bytes + length;
6735 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6737 struct thread *t = &p->threads[p->thread_id];
6738 struct instruction *ip = t->ip;
6740 uint64_t time, n_pkts, n_bytes;
6742 enum rte_color color_in, color_out;
6744 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6747 m = instr_meter_idx_hbo(p, t, ip);
6748 rte_prefetch0(m->n_pkts);
6749 time = rte_get_tsc_cycles();
6750 length = instr_meter_length_nbo(t, ip);
6751 color_in = instr_meter_color_in_hbo(t, ip);
6753 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6754 &m->profile->profile,
6759 color_out &= m->color_mask;
6761 n_pkts = m->n_pkts[color_out];
6762 n_bytes = m->n_bytes[color_out];
6764 instr_meter_color_out_hbo_set(t, ip, color_out);
6766 m->n_pkts[color_out] = n_pkts + 1;
6767 m->n_bytes[color_out] = n_bytes + length;
6774 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6776 struct thread *t = &p->threads[p->thread_id];
6777 struct instruction *ip = t->ip;
6779 uint64_t time, n_pkts, n_bytes;
6781 enum rte_color color_in, color_out;
6783 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6786 m = instr_meter_idx_hbo(p, t, ip);
6787 rte_prefetch0(m->n_pkts);
6788 time = rte_get_tsc_cycles();
6789 length = instr_meter_length_nbo(t, ip);
6790 color_in = (enum rte_color)ip->meter.color_in_val;
6792 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6793 &m->profile->profile,
6798 color_out &= m->color_mask;
6800 n_pkts = m->n_pkts[color_out];
6801 n_bytes = m->n_bytes[color_out];
6803 instr_meter_color_out_hbo_set(t, ip, color_out);
6805 m->n_pkts[color_out] = n_pkts + 1;
6806 m->n_bytes[color_out] = n_bytes + length;
6813 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6815 struct thread *t = &p->threads[p->thread_id];
6816 struct instruction *ip = t->ip;
6818 uint64_t time, n_pkts, n_bytes;
6820 enum rte_color color_in, color_out;
6822 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6825 m = instr_meter_idx_hbo(p, t, ip);
6826 rte_prefetch0(m->n_pkts);
6827 time = rte_get_tsc_cycles();
6828 length = instr_meter_length_hbo(t, ip);
6829 color_in = instr_meter_color_in_hbo(t, ip);
6831 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6832 &m->profile->profile,
6837 color_out &= m->color_mask;
6839 n_pkts = m->n_pkts[color_out];
6840 n_bytes = m->n_bytes[color_out];
6842 instr_meter_color_out_hbo_set(t, ip, color_out);
6844 m->n_pkts[color_out] = n_pkts + 1;
6845 m->n_bytes[color_out] = n_bytes + length;
6852 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6854 struct thread *t = &p->threads[p->thread_id];
6855 struct instruction *ip = t->ip;
6857 uint64_t time, n_pkts, n_bytes;
6859 enum rte_color color_in, color_out;
6861 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6864 m = instr_meter_idx_hbo(p, t, ip);
6865 rte_prefetch0(m->n_pkts);
6866 time = rte_get_tsc_cycles();
6867 length = instr_meter_length_hbo(t, ip);
6868 color_in = (enum rte_color)ip->meter.color_in_val;
6870 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6871 &m->profile->profile,
6876 color_out &= m->color_mask;
6878 n_pkts = m->n_pkts[color_out];
6879 n_bytes = m->n_bytes[color_out];
6881 instr_meter_color_out_hbo_set(t, ip, color_out);
6883 m->n_pkts[color_out] = n_pkts + 1;
6884 m->n_bytes[color_out] = n_bytes + length;
6891 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6893 struct thread *t = &p->threads[p->thread_id];
6894 struct instruction *ip = t->ip;
6896 uint64_t time, n_pkts, n_bytes;
6898 enum rte_color color_in, color_out;
6900 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6903 m = instr_meter_idx_imm(p, ip);
6904 rte_prefetch0(m->n_pkts);
6905 time = rte_get_tsc_cycles();
6906 length = instr_meter_length_nbo(t, ip);
6907 color_in = instr_meter_color_in_hbo(t, ip);
6909 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6910 &m->profile->profile,
6915 color_out &= m->color_mask;
6917 n_pkts = m->n_pkts[color_out];
6918 n_bytes = m->n_bytes[color_out];
6920 instr_meter_color_out_hbo_set(t, ip, color_out);
6922 m->n_pkts[color_out] = n_pkts + 1;
6923 m->n_bytes[color_out] = n_bytes + length;
6930 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6932 struct thread *t = &p->threads[p->thread_id];
6933 struct instruction *ip = t->ip;
6935 uint64_t time, n_pkts, n_bytes;
6937 enum rte_color color_in, color_out;
6939 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6942 m = instr_meter_idx_imm(p, ip);
6943 rte_prefetch0(m->n_pkts);
6944 time = rte_get_tsc_cycles();
6945 length = instr_meter_length_nbo(t, ip);
6946 color_in = (enum rte_color)ip->meter.color_in_val;
6948 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6949 &m->profile->profile,
6954 color_out &= m->color_mask;
6956 n_pkts = m->n_pkts[color_out];
6957 n_bytes = m->n_bytes[color_out];
6959 instr_meter_color_out_hbo_set(t, ip, color_out);
6961 m->n_pkts[color_out] = n_pkts + 1;
6962 m->n_bytes[color_out] = n_bytes + length;
6969 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6971 struct thread *t = &p->threads[p->thread_id];
6972 struct instruction *ip = t->ip;
6974 uint64_t time, n_pkts, n_bytes;
6976 enum rte_color color_in, color_out;
6978 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6981 m = instr_meter_idx_imm(p, ip);
6982 rte_prefetch0(m->n_pkts);
6983 time = rte_get_tsc_cycles();
6984 length = instr_meter_length_hbo(t, ip);
6985 color_in = instr_meter_color_in_hbo(t, ip);
6987 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6988 &m->profile->profile,
6993 color_out &= m->color_mask;
6995 n_pkts = m->n_pkts[color_out];
6996 n_bytes = m->n_bytes[color_out];
6998 instr_meter_color_out_hbo_set(t, ip, color_out);
7000 m->n_pkts[color_out] = n_pkts + 1;
7001 m->n_bytes[color_out] = n_bytes + length;
7007 instr_meter_imi_exec(struct rte_swx_pipeline *p)
7009 struct thread *t = &p->threads[p->thread_id];
7010 struct instruction *ip = t->ip;
7012 uint64_t time, n_pkts, n_bytes;
7014 enum rte_color color_in, color_out;
7016 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
7019 m = instr_meter_idx_imm(p, ip);
7020 rte_prefetch0(m->n_pkts);
7021 time = rte_get_tsc_cycles();
7022 length = instr_meter_length_hbo(t, ip);
7023 color_in = (enum rte_color)ip->meter.color_in_val;
7025 color_out = rte_meter_trtcm_color_aware_check(&m->m,
7026 &m->profile->profile,
7031 color_out &= m->color_mask;
7033 n_pkts = m->n_pkts[color_out];
7034 n_bytes = m->n_bytes[color_out];
7036 instr_meter_color_out_hbo_set(t, ip, color_out);
7038 m->n_pkts[color_out] = n_pkts + 1;
7039 m->n_bytes[color_out] = n_bytes + length;
7048 static struct action *
7049 action_find(struct rte_swx_pipeline *p, const char *name);
7052 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
7053 struct action *action __rte_unused,
7056 struct instruction *instr,
7057 struct instruction_data *data)
7059 CHECK(n_tokens == 2, EINVAL);
7061 strcpy(data->jmp_label, tokens[1]);
7063 instr->type = INSTR_JMP;
7064 instr->jmp.ip = NULL; /* Resolved later. */
7069 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
7070 struct action *action __rte_unused,
7073 struct instruction *instr,
7074 struct instruction_data *data)
7078 CHECK(n_tokens == 3, EINVAL);
7080 strcpy(data->jmp_label, tokens[1]);
7082 h = header_parse(p, tokens[2]);
7085 instr->type = INSTR_JMP_VALID;
7086 instr->jmp.ip = NULL; /* Resolved later. */
7087 instr->jmp.header_id = h->id;
7092 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
7093 struct action *action __rte_unused,
7096 struct instruction *instr,
7097 struct instruction_data *data)
7101 CHECK(n_tokens == 3, EINVAL);
7103 strcpy(data->jmp_label, tokens[1]);
7105 h = header_parse(p, tokens[2]);
7108 instr->type = INSTR_JMP_INVALID;
7109 instr->jmp.ip = NULL; /* Resolved later. */
7110 instr->jmp.header_id = h->id;
7115 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
7116 struct action *action,
7119 struct instruction *instr,
7120 struct instruction_data *data)
7122 CHECK(!action, EINVAL);
7123 CHECK(n_tokens == 2, EINVAL);
7125 strcpy(data->jmp_label, tokens[1]);
7127 instr->type = INSTR_JMP_HIT;
7128 instr->jmp.ip = NULL; /* Resolved later. */
7133 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
7134 struct action *action,
7137 struct instruction *instr,
7138 struct instruction_data *data)
7140 CHECK(!action, EINVAL);
7141 CHECK(n_tokens == 2, EINVAL);
7143 strcpy(data->jmp_label, tokens[1]);
7145 instr->type = INSTR_JMP_MISS;
7146 instr->jmp.ip = NULL; /* Resolved later. */
7151 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
7152 struct action *action,
7155 struct instruction *instr,
7156 struct instruction_data *data)
7160 CHECK(!action, EINVAL);
7161 CHECK(n_tokens == 3, EINVAL);
7163 strcpy(data->jmp_label, tokens[1]);
7165 a = action_find(p, tokens[2]);
7168 instr->type = INSTR_JMP_ACTION_HIT;
7169 instr->jmp.ip = NULL; /* Resolved later. */
7170 instr->jmp.action_id = a->id;
7175 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7176 struct action *action,
7179 struct instruction *instr,
7180 struct instruction_data *data)
7184 CHECK(!action, EINVAL);
7185 CHECK(n_tokens == 3, EINVAL);
7187 strcpy(data->jmp_label, tokens[1]);
7189 a = action_find(p, tokens[2]);
7192 instr->type = INSTR_JMP_ACTION_MISS;
7193 instr->jmp.ip = NULL; /* Resolved later. */
7194 instr->jmp.action_id = a->id;
7199 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7200 struct action *action,
7203 struct instruction *instr,
7204 struct instruction_data *data)
7206 char *a = tokens[2], *b = tokens[3];
7207 struct field *fa, *fb;
7209 uint32_t a_struct_id, b_struct_id;
7211 CHECK(n_tokens == 4, EINVAL);
7213 strcpy(data->jmp_label, tokens[1]);
7215 fa = struct_field_parse(p, action, a, &a_struct_id);
7217 CHECK(!fa->var_size, EINVAL);
7219 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7220 fb = struct_field_parse(p, action, b, &b_struct_id);
7222 CHECK(!fb->var_size, EINVAL);
7224 instr->type = INSTR_JMP_EQ;
7225 if (a[0] != 'h' && b[0] == 'h')
7226 instr->type = INSTR_JMP_EQ_MH;
7227 if (a[0] == 'h' && b[0] != 'h')
7228 instr->type = INSTR_JMP_EQ_HM;
7229 if (a[0] == 'h' && b[0] == 'h')
7230 instr->type = INSTR_JMP_EQ_HH;
7231 instr->jmp.ip = NULL; /* Resolved later. */
7233 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7234 instr->jmp.a.n_bits = fa->n_bits;
7235 instr->jmp.a.offset = fa->offset / 8;
7236 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7237 instr->jmp.b.n_bits = fb->n_bits;
7238 instr->jmp.b.offset = fb->offset / 8;
7243 b_val = strtoull(b, &b, 0);
7244 CHECK(!b[0], EINVAL);
7247 b_val = hton64(b_val) >> (64 - fa->n_bits);
7249 instr->type = INSTR_JMP_EQ_I;
7250 instr->jmp.ip = NULL; /* Resolved later. */
7251 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7252 instr->jmp.a.n_bits = fa->n_bits;
7253 instr->jmp.a.offset = fa->offset / 8;
7254 instr->jmp.b_val = b_val;
7259 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7260 struct action *action,
7263 struct instruction *instr,
7264 struct instruction_data *data)
7266 char *a = tokens[2], *b = tokens[3];
7267 struct field *fa, *fb;
7269 uint32_t a_struct_id, b_struct_id;
7271 CHECK(n_tokens == 4, EINVAL);
7273 strcpy(data->jmp_label, tokens[1]);
7275 fa = struct_field_parse(p, action, a, &a_struct_id);
7277 CHECK(!fa->var_size, EINVAL);
7279 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7280 fb = struct_field_parse(p, action, b, &b_struct_id);
7282 CHECK(!fb->var_size, EINVAL);
7284 instr->type = INSTR_JMP_NEQ;
7285 if (a[0] != 'h' && b[0] == 'h')
7286 instr->type = INSTR_JMP_NEQ_MH;
7287 if (a[0] == 'h' && b[0] != 'h')
7288 instr->type = INSTR_JMP_NEQ_HM;
7289 if (a[0] == 'h' && b[0] == 'h')
7290 instr->type = INSTR_JMP_NEQ_HH;
7291 instr->jmp.ip = NULL; /* Resolved later. */
7293 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7294 instr->jmp.a.n_bits = fa->n_bits;
7295 instr->jmp.a.offset = fa->offset / 8;
7296 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7297 instr->jmp.b.n_bits = fb->n_bits;
7298 instr->jmp.b.offset = fb->offset / 8;
7303 b_val = strtoull(b, &b, 0);
7304 CHECK(!b[0], EINVAL);
7307 b_val = hton64(b_val) >> (64 - fa->n_bits);
7309 instr->type = INSTR_JMP_NEQ_I;
7310 instr->jmp.ip = NULL; /* Resolved later. */
7311 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7312 instr->jmp.a.n_bits = fa->n_bits;
7313 instr->jmp.a.offset = fa->offset / 8;
7314 instr->jmp.b_val = b_val;
7319 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7320 struct action *action,
7323 struct instruction *instr,
7324 struct instruction_data *data)
7326 char *a = tokens[2], *b = tokens[3];
7327 struct field *fa, *fb;
7329 uint32_t a_struct_id, b_struct_id;
7331 CHECK(n_tokens == 4, EINVAL);
7333 strcpy(data->jmp_label, tokens[1]);
7335 fa = struct_field_parse(p, action, a, &a_struct_id);
7337 CHECK(!fa->var_size, EINVAL);
7339 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7340 fb = struct_field_parse(p, action, b, &b_struct_id);
7342 CHECK(!fb->var_size, EINVAL);
7344 instr->type = INSTR_JMP_LT;
7345 if (a[0] == 'h' && b[0] != 'h')
7346 instr->type = INSTR_JMP_LT_HM;
7347 if (a[0] != 'h' && b[0] == 'h')
7348 instr->type = INSTR_JMP_LT_MH;
7349 if (a[0] == 'h' && b[0] == 'h')
7350 instr->type = INSTR_JMP_LT_HH;
7351 instr->jmp.ip = NULL; /* Resolved later. */
7353 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7354 instr->jmp.a.n_bits = fa->n_bits;
7355 instr->jmp.a.offset = fa->offset / 8;
7356 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7357 instr->jmp.b.n_bits = fb->n_bits;
7358 instr->jmp.b.offset = fb->offset / 8;
7362 /* JMP_LT_MI, JMP_LT_HI. */
7363 b_val = strtoull(b, &b, 0);
7364 CHECK(!b[0], EINVAL);
7366 instr->type = INSTR_JMP_LT_MI;
7368 instr->type = INSTR_JMP_LT_HI;
7369 instr->jmp.ip = NULL; /* Resolved later. */
7371 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7372 instr->jmp.a.n_bits = fa->n_bits;
7373 instr->jmp.a.offset = fa->offset / 8;
7374 instr->jmp.b_val = b_val;
7379 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7380 struct action *action,
7383 struct instruction *instr,
7384 struct instruction_data *data)
7386 char *a = tokens[2], *b = tokens[3];
7387 struct field *fa, *fb;
7389 uint32_t a_struct_id, b_struct_id;
7391 CHECK(n_tokens == 4, EINVAL);
7393 strcpy(data->jmp_label, tokens[1]);
7395 fa = struct_field_parse(p, action, a, &a_struct_id);
7397 CHECK(!fa->var_size, EINVAL);
7399 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7400 fb = struct_field_parse(p, action, b, &b_struct_id);
7402 CHECK(!fb->var_size, EINVAL);
7404 instr->type = INSTR_JMP_GT;
7405 if (a[0] == 'h' && b[0] != 'h')
7406 instr->type = INSTR_JMP_GT_HM;
7407 if (a[0] != 'h' && b[0] == 'h')
7408 instr->type = INSTR_JMP_GT_MH;
7409 if (a[0] == 'h' && b[0] == 'h')
7410 instr->type = INSTR_JMP_GT_HH;
7411 instr->jmp.ip = NULL; /* Resolved later. */
7413 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7414 instr->jmp.a.n_bits = fa->n_bits;
7415 instr->jmp.a.offset = fa->offset / 8;
7416 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7417 instr->jmp.b.n_bits = fb->n_bits;
7418 instr->jmp.b.offset = fb->offset / 8;
7422 /* JMP_GT_MI, JMP_GT_HI. */
7423 b_val = strtoull(b, &b, 0);
7424 CHECK(!b[0], EINVAL);
7426 instr->type = INSTR_JMP_GT_MI;
7428 instr->type = INSTR_JMP_GT_HI;
7429 instr->jmp.ip = NULL; /* Resolved later. */
7431 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7432 instr->jmp.a.n_bits = fa->n_bits;
7433 instr->jmp.a.offset = fa->offset / 8;
7434 instr->jmp.b_val = b_val;
7439 instr_jmp_exec(struct rte_swx_pipeline *p)
7441 struct thread *t = &p->threads[p->thread_id];
7442 struct instruction *ip = t->ip;
7444 TRACE("[Thread %2u] jmp\n", p->thread_id);
7446 thread_ip_set(t, ip->jmp.ip);
7450 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7452 struct thread *t = &p->threads[p->thread_id];
7453 struct instruction *ip = t->ip;
7454 uint32_t header_id = ip->jmp.header_id;
7456 TRACE("[Thread %2u] jmpv\n", p->thread_id);
7458 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7462 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7464 struct thread *t = &p->threads[p->thread_id];
7465 struct instruction *ip = t->ip;
7466 uint32_t header_id = ip->jmp.header_id;
7468 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7470 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7474 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7476 struct thread *t = &p->threads[p->thread_id];
7477 struct instruction *ip = t->ip;
7478 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7480 TRACE("[Thread %2u] jmph\n", p->thread_id);
7482 t->ip = ip_next[t->hit];
7486 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7488 struct thread *t = &p->threads[p->thread_id];
7489 struct instruction *ip = t->ip;
7490 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7492 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7494 t->ip = ip_next[t->hit];
7498 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7500 struct thread *t = &p->threads[p->thread_id];
7501 struct instruction *ip = t->ip;
7503 TRACE("[Thread %2u] jmpa\n", p->thread_id);
7505 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7509 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7511 struct thread *t = &p->threads[p->thread_id];
7512 struct instruction *ip = t->ip;
7514 TRACE("[Thread %2u] jmpna\n", p->thread_id);
7516 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7520 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7522 struct thread *t = &p->threads[p->thread_id];
7523 struct instruction *ip = t->ip;
7525 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7531 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7533 struct thread *t = &p->threads[p->thread_id];
7534 struct instruction *ip = t->ip;
7536 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7538 JMP_CMP_MH(t, ip, ==);
7542 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7544 struct thread *t = &p->threads[p->thread_id];
7545 struct instruction *ip = t->ip;
7547 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7549 JMP_CMP_HM(t, ip, ==);
7553 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7555 struct thread *t = &p->threads[p->thread_id];
7556 struct instruction *ip = t->ip;
7558 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7560 JMP_CMP_HH_FAST(t, ip, ==);
7564 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7566 struct thread *t = &p->threads[p->thread_id];
7567 struct instruction *ip = t->ip;
7569 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7571 JMP_CMP_I(t, ip, ==);
7575 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7577 struct thread *t = &p->threads[p->thread_id];
7578 struct instruction *ip = t->ip;
7580 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7586 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7588 struct thread *t = &p->threads[p->thread_id];
7589 struct instruction *ip = t->ip;
7591 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7593 JMP_CMP_MH(t, ip, !=);
7597 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7599 struct thread *t = &p->threads[p->thread_id];
7600 struct instruction *ip = t->ip;
7602 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7604 JMP_CMP_HM(t, ip, !=);
7608 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7610 struct thread *t = &p->threads[p->thread_id];
7611 struct instruction *ip = t->ip;
7613 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7615 JMP_CMP_HH_FAST(t, ip, !=);
7619 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7621 struct thread *t = &p->threads[p->thread_id];
7622 struct instruction *ip = t->ip;
7624 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7626 JMP_CMP_I(t, ip, !=);
7630 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7632 struct thread *t = &p->threads[p->thread_id];
7633 struct instruction *ip = t->ip;
7635 TRACE("[Thread %2u] jmplt\n", p->thread_id);
7641 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7643 struct thread *t = &p->threads[p->thread_id];
7644 struct instruction *ip = t->ip;
7646 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7648 JMP_CMP_MH(t, ip, <);
7652 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7654 struct thread *t = &p->threads[p->thread_id];
7655 struct instruction *ip = t->ip;
7657 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7659 JMP_CMP_HM(t, ip, <);
7663 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7665 struct thread *t = &p->threads[p->thread_id];
7666 struct instruction *ip = t->ip;
7668 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7670 JMP_CMP_HH(t, ip, <);
7674 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7676 struct thread *t = &p->threads[p->thread_id];
7677 struct instruction *ip = t->ip;
7679 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7681 JMP_CMP_MI(t, ip, <);
7685 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7687 struct thread *t = &p->threads[p->thread_id];
7688 struct instruction *ip = t->ip;
7690 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7692 JMP_CMP_HI(t, ip, <);
7696 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7698 struct thread *t = &p->threads[p->thread_id];
7699 struct instruction *ip = t->ip;
7701 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7707 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7709 struct thread *t = &p->threads[p->thread_id];
7710 struct instruction *ip = t->ip;
7712 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7714 JMP_CMP_MH(t, ip, >);
7718 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7720 struct thread *t = &p->threads[p->thread_id];
7721 struct instruction *ip = t->ip;
7723 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7725 JMP_CMP_HM(t, ip, >);
7729 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7731 struct thread *t = &p->threads[p->thread_id];
7732 struct instruction *ip = t->ip;
7734 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7736 JMP_CMP_HH(t, ip, >);
7740 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7742 struct thread *t = &p->threads[p->thread_id];
7743 struct instruction *ip = t->ip;
7745 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7747 JMP_CMP_MI(t, ip, >);
7751 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7753 struct thread *t = &p->threads[p->thread_id];
7754 struct instruction *ip = t->ip;
7756 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7758 JMP_CMP_HI(t, ip, >);
7765 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7766 struct action *action,
7767 char **tokens __rte_unused,
7769 struct instruction *instr,
7770 struct instruction_data *data __rte_unused)
7772 CHECK(action, EINVAL);
7773 CHECK(n_tokens == 1, EINVAL);
7775 instr->type = INSTR_RETURN;
7780 instr_return_exec(struct rte_swx_pipeline *p)
7782 struct thread *t = &p->threads[p->thread_id];
7784 TRACE("[Thread %2u] return\n", p->thread_id);
7790 instr_translate(struct rte_swx_pipeline *p,
7791 struct action *action,
7793 struct instruction *instr,
7794 struct instruction_data *data)
7796 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7797 int n_tokens = 0, tpos = 0;
7799 /* Parse the instruction string into tokens. */
7803 token = strtok_r(string, " \t\v", &string);
7807 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7808 CHECK_NAME(token, EINVAL);
7810 tokens[n_tokens] = token;
7814 CHECK(n_tokens, EINVAL);
7816 /* Handle the optional instruction label. */
7817 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7818 strcpy(data->label, tokens[0]);
7821 CHECK(n_tokens - tpos, EINVAL);
7824 /* Identify the instruction type. */
7825 if (!strcmp(tokens[tpos], "rx"))
7826 return instr_rx_translate(p,
7833 if (!strcmp(tokens[tpos], "tx"))
7834 return instr_tx_translate(p,
7841 if (!strcmp(tokens[tpos], "drop"))
7842 return instr_drop_translate(p,
7849 if (!strcmp(tokens[tpos], "extract"))
7850 return instr_hdr_extract_translate(p,
7857 if (!strcmp(tokens[tpos], "emit"))
7858 return instr_hdr_emit_translate(p,
7865 if (!strcmp(tokens[tpos], "validate"))
7866 return instr_hdr_validate_translate(p,
7873 if (!strcmp(tokens[tpos], "invalidate"))
7874 return instr_hdr_invalidate_translate(p,
7881 if (!strcmp(tokens[tpos], "mov"))
7882 return instr_mov_translate(p,
7889 if (!strcmp(tokens[tpos], "add"))
7890 return instr_alu_add_translate(p,
7897 if (!strcmp(tokens[tpos], "sub"))
7898 return instr_alu_sub_translate(p,
7905 if (!strcmp(tokens[tpos], "ckadd"))
7906 return instr_alu_ckadd_translate(p,
7913 if (!strcmp(tokens[tpos], "cksub"))
7914 return instr_alu_cksub_translate(p,
7921 if (!strcmp(tokens[tpos], "and"))
7922 return instr_alu_and_translate(p,
7929 if (!strcmp(tokens[tpos], "or"))
7930 return instr_alu_or_translate(p,
7937 if (!strcmp(tokens[tpos], "xor"))
7938 return instr_alu_xor_translate(p,
7945 if (!strcmp(tokens[tpos], "shl"))
7946 return instr_alu_shl_translate(p,
7953 if (!strcmp(tokens[tpos], "shr"))
7954 return instr_alu_shr_translate(p,
7961 if (!strcmp(tokens[tpos], "regprefetch"))
7962 return instr_regprefetch_translate(p,
7969 if (!strcmp(tokens[tpos], "regrd"))
7970 return instr_regrd_translate(p,
7977 if (!strcmp(tokens[tpos], "regwr"))
7978 return instr_regwr_translate(p,
7985 if (!strcmp(tokens[tpos], "regadd"))
7986 return instr_regadd_translate(p,
7993 if (!strcmp(tokens[tpos], "metprefetch"))
7994 return instr_metprefetch_translate(p,
8001 if (!strcmp(tokens[tpos], "meter"))
8002 return instr_meter_translate(p,
8009 if (!strcmp(tokens[tpos], "table"))
8010 return instr_table_translate(p,
8017 if (!strcmp(tokens[tpos], "extern"))
8018 return instr_extern_translate(p,
8025 if (!strcmp(tokens[tpos], "jmp"))
8026 return instr_jmp_translate(p,
8033 if (!strcmp(tokens[tpos], "jmpv"))
8034 return instr_jmp_valid_translate(p,
8041 if (!strcmp(tokens[tpos], "jmpnv"))
8042 return instr_jmp_invalid_translate(p,
8049 if (!strcmp(tokens[tpos], "jmph"))
8050 return instr_jmp_hit_translate(p,
8057 if (!strcmp(tokens[tpos], "jmpnh"))
8058 return instr_jmp_miss_translate(p,
8065 if (!strcmp(tokens[tpos], "jmpa"))
8066 return instr_jmp_action_hit_translate(p,
8073 if (!strcmp(tokens[tpos], "jmpna"))
8074 return instr_jmp_action_miss_translate(p,
8081 if (!strcmp(tokens[tpos], "jmpeq"))
8082 return instr_jmp_eq_translate(p,
8089 if (!strcmp(tokens[tpos], "jmpneq"))
8090 return instr_jmp_neq_translate(p,
8097 if (!strcmp(tokens[tpos], "jmplt"))
8098 return instr_jmp_lt_translate(p,
8105 if (!strcmp(tokens[tpos], "jmpgt"))
8106 return instr_jmp_gt_translate(p,
8113 if (!strcmp(tokens[tpos], "return"))
8114 return instr_return_translate(p,
8124 static struct instruction_data *
8125 label_find(struct instruction_data *data, uint32_t n, const char *label)
8129 for (i = 0; i < n; i++)
8130 if (!strcmp(label, data[i].label))
8137 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
8139 uint32_t count = 0, i;
8144 for (i = 0; i < n; i++)
8145 if (!strcmp(label, data[i].jmp_label))
8152 instr_label_check(struct instruction_data *instruction_data,
8153 uint32_t n_instructions)
8157 /* Check that all instruction labels are unique. */
8158 for (i = 0; i < n_instructions; i++) {
8159 struct instruction_data *data = &instruction_data[i];
8160 char *label = data->label;
8166 for (j = i + 1; j < n_instructions; j++)
8167 CHECK(strcmp(label, data[j].label), EINVAL);
8170 /* Get users for each instruction label. */
8171 for (i = 0; i < n_instructions; i++) {
8172 struct instruction_data *data = &instruction_data[i];
8173 char *label = data->label;
8175 data->n_users = label_is_used(instruction_data,
8184 instr_jmp_resolve(struct instruction *instructions,
8185 struct instruction_data *instruction_data,
8186 uint32_t n_instructions)
8190 for (i = 0; i < n_instructions; i++) {
8191 struct instruction *instr = &instructions[i];
8192 struct instruction_data *data = &instruction_data[i];
8193 struct instruction_data *found;
8195 if (!instruction_is_jmp(instr))
8198 found = label_find(instruction_data,
8201 CHECK(found, EINVAL);
8203 instr->jmp.ip = &instructions[found - instruction_data];
8210 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8212 struct instruction *instr,
8213 struct instruction_data *data __rte_unused,
8214 uint32_t n_instructions)
8217 enum instruction_type type;
8220 /* Check that the first instruction is rx. */
8221 CHECK(instr[0].type == INSTR_RX, EINVAL);
8223 /* Check that there is at least one tx instruction. */
8224 for (i = 0; i < n_instructions; i++) {
8225 type = instr[i].type;
8227 if (instruction_is_tx(type))
8230 CHECK(i < n_instructions, EINVAL);
8232 /* Check that the last instruction is either tx or unconditional
8235 type = instr[n_instructions - 1].type;
8236 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8240 enum instruction_type type;
8243 /* Check that there is at least one return or tx instruction. */
8244 for (i = 0; i < n_instructions; i++) {
8245 type = instr[i].type;
8247 if ((type == INSTR_RETURN) || instruction_is_tx(type))
8250 CHECK(i < n_instructions, EINVAL);
8257 instr_compact(struct instruction *instructions,
8258 struct instruction_data *instruction_data,
8259 uint32_t n_instructions)
8261 uint32_t i, pos = 0;
8263 /* Eliminate the invalid instructions that have been optimized out. */
8264 for (i = 0; i < n_instructions; i++) {
8265 struct instruction *instr = &instructions[i];
8266 struct instruction_data *data = &instruction_data[i];
8272 memcpy(&instructions[pos], instr, sizeof(*instr));
8273 memcpy(&instruction_data[pos], data, sizeof(*data));
8283 instr_pattern_extract_many_search(struct instruction *instr,
8284 struct instruction_data *data,
8286 uint32_t *n_pattern_instr)
8290 for (i = 0; i < n_instr; i++) {
8291 if (data[i].invalid)
8294 if (instr[i].type != INSTR_HDR_EXTRACT)
8297 if (i == RTE_DIM(instr->io.hdr.header_id))
8300 if (i && data[i].n_users)
8307 *n_pattern_instr = i;
8312 instr_pattern_extract_many_replace(struct instruction *instr,
8313 struct instruction_data *data,
8318 for (i = 1; i < n_instr; i++) {
8320 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8321 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8322 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8324 data[i].invalid = 1;
8329 instr_pattern_extract_many_optimize(struct instruction *instructions,
8330 struct instruction_data *instruction_data,
8331 uint32_t n_instructions)
8335 for (i = 0; i < n_instructions; ) {
8336 struct instruction *instr = &instructions[i];
8337 struct instruction_data *data = &instruction_data[i];
8338 uint32_t n_instr = 0;
8342 detected = instr_pattern_extract_many_search(instr,
8347 instr_pattern_extract_many_replace(instr,
8354 /* No pattern starting at the current instruction. */
8358 /* Eliminate the invalid instructions that have been optimized out. */
8359 n_instructions = instr_compact(instructions,
8363 return n_instructions;
8367 instr_pattern_emit_many_tx_search(struct instruction *instr,
8368 struct instruction_data *data,
8370 uint32_t *n_pattern_instr)
8374 for (i = 0; i < n_instr; i++) {
8375 if (data[i].invalid)
8378 if (instr[i].type != INSTR_HDR_EMIT)
8381 if (i == RTE_DIM(instr->io.hdr.header_id))
8384 if (i && data[i].n_users)
8391 if (!instruction_is_tx(instr[i].type))
8394 if (data[i].n_users)
8399 *n_pattern_instr = i;
8404 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8405 struct instruction_data *data,
8410 /* Any emit instruction in addition to the first one. */
8411 for (i = 1; i < n_instr - 1; i++) {
8413 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8414 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8415 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8417 data[i].invalid = 1;
8420 /* The TX instruction is the last one in the pattern. */
8422 instr[0].io.io.offset = instr[i].io.io.offset;
8423 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8424 data[i].invalid = 1;
8428 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8429 struct instruction_data *instruction_data,
8430 uint32_t n_instructions)
8434 for (i = 0; i < n_instructions; ) {
8435 struct instruction *instr = &instructions[i];
8436 struct instruction_data *data = &instruction_data[i];
8437 uint32_t n_instr = 0;
8440 /* Emit many + TX. */
8441 detected = instr_pattern_emit_many_tx_search(instr,
8446 instr_pattern_emit_many_tx_replace(instr,
8453 /* No pattern starting at the current instruction. */
8457 /* Eliminate the invalid instructions that have been optimized out. */
8458 n_instructions = instr_compact(instructions,
8462 return n_instructions;
8466 action_arg_src_mov_count(struct action *a,
8468 struct instruction *instructions,
8469 struct instruction_data *instruction_data,
8470 uint32_t n_instructions);
8473 instr_pattern_mov_all_validate_search(struct rte_swx_pipeline *p,
8475 struct instruction *instr,
8476 struct instruction_data *data,
8478 struct instruction *instructions,
8479 struct instruction_data *instruction_data,
8480 uint32_t n_instructions,
8481 uint32_t *n_pattern_instr)
8484 uint32_t src_field_id, i, j;
8486 /* Prerequisites. */
8490 /* First instruction: MOV_HM. */
8491 if (data[0].invalid || (instr[0].type != INSTR_MOV_HM))
8494 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8495 if (!h || h->st->var_size)
8498 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8499 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8502 if (src_field_id == a->st->n_fields)
8505 if (instr[0].mov.dst.offset ||
8506 (instr[0].mov.dst.n_bits != h->st->fields[0].n_bits) ||
8507 instr[0].mov.src.struct_id ||
8508 (instr[0].mov.src.n_bits != a->st->fields[src_field_id].n_bits) ||
8509 (instr[0].mov.dst.n_bits != instr[0].mov.src.n_bits))
8512 if ((n_instr < h->st->n_fields + 1) ||
8513 (a->st->n_fields < src_field_id + h->st->n_fields + 1))
8516 /* Subsequent instructions: MOV_HM. */
8517 for (i = 1; i < h->st->n_fields; i++)
8518 if (data[i].invalid ||
8520 (instr[i].type != INSTR_MOV_HM) ||
8521 (instr[i].mov.dst.struct_id != h->struct_id) ||
8522 (instr[i].mov.dst.offset != h->st->fields[i].offset / 8) ||
8523 (instr[i].mov.dst.n_bits != h->st->fields[i].n_bits) ||
8524 instr[i].mov.src.struct_id ||
8525 (instr[i].mov.src.offset != a->st->fields[src_field_id + i].offset / 8) ||
8526 (instr[i].mov.src.n_bits != a->st->fields[src_field_id + i].n_bits) ||
8527 (instr[i].mov.dst.n_bits != instr[i].mov.src.n_bits))
8530 /* Last instruction: HDR_VALIDATE. */
8531 if ((instr[i].type != INSTR_HDR_VALIDATE) ||
8532 (instr[i].valid.header_id != h->id))
8535 /* Check that none of the action args that are used as source for this
8536 * DMA transfer are not used as source in any other mov instruction.
8538 for (j = src_field_id; j < src_field_id + h->st->n_fields; j++) {
8541 n_users = action_arg_src_mov_count(a,
8550 *n_pattern_instr = 1 + i;
8555 instr_pattern_mov_all_validate_replace(struct rte_swx_pipeline *p,
8557 struct instruction *instr,
8558 struct instruction_data *data,
8562 uint32_t src_field_id, src_offset, i;
8564 /* Read from the instructions before they are modified. */
8565 h = header_find_by_struct_id(p, instr[0].mov.dst.struct_id);
8569 for (src_field_id = 0; src_field_id < a->st->n_fields; src_field_id++)
8570 if (instr[0].mov.src.offset == a->st->fields[src_field_id].offset / 8)
8573 if (src_field_id == a->st->n_fields)
8576 src_offset = instr[0].mov.src.offset;
8578 /* Modify the instructions. */
8579 instr[0].type = INSTR_DMA_HT;
8580 instr[0].dma.dst.header_id[0] = h->id;
8581 instr[0].dma.dst.struct_id[0] = h->struct_id;
8582 instr[0].dma.src.offset[0] = (uint8_t)src_offset;
8583 instr[0].dma.n_bytes[0] = h->st->n_bits / 8;
8585 for (i = 1; i < n_instr; i++)
8586 data[i].invalid = 1;
8588 /* Update the endianness of the action arguments to header endianness. */
8589 for (i = 0; i < h->st->n_fields; i++)
8590 a->args_endianness[src_field_id + i] = 1;
8594 instr_pattern_mov_all_validate_optimize(struct rte_swx_pipeline *p,
8596 struct instruction *instructions,
8597 struct instruction_data *instruction_data,
8598 uint32_t n_instructions)
8603 return n_instructions;
8605 for (i = 0; i < n_instructions; ) {
8606 struct instruction *instr = &instructions[i];
8607 struct instruction_data *data = &instruction_data[i];
8608 uint32_t n_instr = 0;
8611 /* Mov all + validate. */
8612 detected = instr_pattern_mov_all_validate_search(p,
8622 instr_pattern_mov_all_validate_replace(p, a, instr, data, n_instr);
8627 /* No pattern starting at the current instruction. */
8631 /* Eliminate the invalid instructions that have been optimized out. */
8632 n_instructions = instr_compact(instructions,
8636 return n_instructions;
8640 instr_pattern_dma_many_search(struct instruction *instr,
8641 struct instruction_data *data,
8643 uint32_t *n_pattern_instr)
8647 for (i = 0; i < n_instr; i++) {
8648 if (data[i].invalid)
8651 if (instr[i].type != INSTR_DMA_HT)
8654 if (i == RTE_DIM(instr->dma.dst.header_id))
8657 if (i && data[i].n_users)
8664 *n_pattern_instr = i;
8669 instr_pattern_dma_many_replace(struct instruction *instr,
8670 struct instruction_data *data,
8675 for (i = 1; i < n_instr; i++) {
8677 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8678 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8679 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8680 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8682 data[i].invalid = 1;
8687 instr_pattern_dma_many_optimize(struct instruction *instructions,
8688 struct instruction_data *instruction_data,
8689 uint32_t n_instructions)
8693 for (i = 0; i < n_instructions; ) {
8694 struct instruction *instr = &instructions[i];
8695 struct instruction_data *data = &instruction_data[i];
8696 uint32_t n_instr = 0;
8700 detected = instr_pattern_dma_many_search(instr,
8705 instr_pattern_dma_many_replace(instr, data, n_instr);
8710 /* No pattern starting at the current instruction. */
8714 /* Eliminate the invalid instructions that have been optimized out. */
8715 n_instructions = instr_compact(instructions,
8719 return n_instructions;
8723 instr_optimize(struct rte_swx_pipeline *p,
8725 struct instruction *instructions,
8726 struct instruction_data *instruction_data,
8727 uint32_t n_instructions)
8730 n_instructions = instr_pattern_extract_many_optimize(instructions,
8734 /* Emit many + TX. */
8735 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8739 /* Mov all + validate. */
8740 n_instructions = instr_pattern_mov_all_validate_optimize(p,
8747 n_instructions = instr_pattern_dma_many_optimize(instructions,
8751 return n_instructions;
8755 instruction_config(struct rte_swx_pipeline *p,
8757 const char **instructions,
8758 uint32_t n_instructions)
8760 struct instruction *instr = NULL;
8761 struct instruction_data *data = NULL;
8765 CHECK(n_instructions, EINVAL);
8766 CHECK(instructions, EINVAL);
8767 for (i = 0; i < n_instructions; i++)
8768 CHECK_INSTRUCTION(instructions[i], EINVAL);
8770 /* Memory allocation. */
8771 instr = calloc(n_instructions, sizeof(struct instruction));
8777 data = calloc(n_instructions, sizeof(struct instruction_data));
8783 for (i = 0; i < n_instructions; i++) {
8784 char *string = strdup(instructions[i]);
8790 err = instr_translate(p, a, string, &instr[i], &data[i]);
8799 err = instr_label_check(data, n_instructions);
8803 err = instr_verify(p, a, instr, data, n_instructions);
8807 n_instructions = instr_optimize(p, a, instr, data, n_instructions);
8809 err = instr_jmp_resolve(instr, data, n_instructions);
8814 a->instructions = instr;
8815 a->n_instructions = n_instructions;
8817 p->instructions = instr;
8818 p->n_instructions = n_instructions;
8830 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8832 static instr_exec_t instruction_table[] = {
8833 [INSTR_RX] = instr_rx_exec,
8834 [INSTR_TX] = instr_tx_exec,
8835 [INSTR_TX_I] = instr_tx_i_exec,
8837 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8838 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8839 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8840 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8841 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8842 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8843 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8844 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8846 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8847 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8848 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8849 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8850 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8851 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8852 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8853 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8854 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8856 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8857 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8859 [INSTR_MOV] = instr_mov_exec,
8860 [INSTR_MOV_MH] = instr_mov_mh_exec,
8861 [INSTR_MOV_HM] = instr_mov_hm_exec,
8862 [INSTR_MOV_HH] = instr_mov_hh_exec,
8863 [INSTR_MOV_I] = instr_mov_i_exec,
8865 [INSTR_DMA_HT] = instr_dma_ht_exec,
8866 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8867 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8868 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8869 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8870 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8871 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8872 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8874 [INSTR_ALU_ADD] = instr_alu_add_exec,
8875 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8876 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8877 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8878 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8879 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8881 [INSTR_ALU_SUB] = instr_alu_sub_exec,
8882 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8883 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8884 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8885 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8886 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8888 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8889 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8890 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8891 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8893 [INSTR_ALU_AND] = instr_alu_and_exec,
8894 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
8895 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
8896 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
8897 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8899 [INSTR_ALU_OR] = instr_alu_or_exec,
8900 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
8901 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
8902 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
8903 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8905 [INSTR_ALU_XOR] = instr_alu_xor_exec,
8906 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
8907 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
8908 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
8909 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8911 [INSTR_ALU_SHL] = instr_alu_shl_exec,
8912 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8913 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8914 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8915 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8916 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8918 [INSTR_ALU_SHR] = instr_alu_shr_exec,
8919 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8920 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8921 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8922 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8923 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8925 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8926 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8927 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8929 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8930 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8931 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8932 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8933 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8934 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8936 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8937 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8938 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8939 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8940 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8941 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8942 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8943 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8944 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8946 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8947 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8948 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8949 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8950 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8951 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8952 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8953 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8954 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8956 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8957 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8958 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8960 [INSTR_METER_HHM] = instr_meter_hhm_exec,
8961 [INSTR_METER_HHI] = instr_meter_hhi_exec,
8962 [INSTR_METER_HMM] = instr_meter_hmm_exec,
8963 [INSTR_METER_HMI] = instr_meter_hmi_exec,
8964 [INSTR_METER_MHM] = instr_meter_mhm_exec,
8965 [INSTR_METER_MHI] = instr_meter_mhi_exec,
8966 [INSTR_METER_MMM] = instr_meter_mmm_exec,
8967 [INSTR_METER_MMI] = instr_meter_mmi_exec,
8968 [INSTR_METER_IHM] = instr_meter_ihm_exec,
8969 [INSTR_METER_IHI] = instr_meter_ihi_exec,
8970 [INSTR_METER_IMM] = instr_meter_imm_exec,
8971 [INSTR_METER_IMI] = instr_meter_imi_exec,
8973 [INSTR_TABLE] = instr_table_exec,
8974 [INSTR_SELECTOR] = instr_selector_exec,
8975 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8976 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8978 [INSTR_JMP] = instr_jmp_exec,
8979 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8980 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8981 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8982 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8983 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8984 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8986 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8987 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
8988 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
8989 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
8990 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8992 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8993 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
8994 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
8995 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
8996 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8998 [INSTR_JMP_LT] = instr_jmp_lt_exec,
8999 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
9000 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
9001 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
9002 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
9003 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
9005 [INSTR_JMP_GT] = instr_jmp_gt_exec,
9006 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
9007 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
9008 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
9009 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
9010 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
9012 [INSTR_RETURN] = instr_return_exec,
9016 instr_exec(struct rte_swx_pipeline *p)
9018 struct thread *t = &p->threads[p->thread_id];
9019 struct instruction *ip = t->ip;
9020 instr_exec_t instr = instruction_table[ip->type];
9028 static struct action *
9029 action_find(struct rte_swx_pipeline *p, const char *name)
9031 struct action *elem;
9036 TAILQ_FOREACH(elem, &p->actions, node)
9037 if (strcmp(elem->name, name) == 0)
9043 static struct action *
9044 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9046 struct action *action = NULL;
9048 TAILQ_FOREACH(action, &p->actions, node)
9049 if (action->id == id)
9055 static struct field *
9056 action_field_find(struct action *a, const char *name)
9058 return a->st ? struct_type_field_find(a->st, name) : NULL;
9061 static struct field *
9062 action_field_parse(struct action *action, const char *name)
9064 if (name[0] != 't' || name[1] != '.')
9067 return action_field_find(action, &name[2]);
9071 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
9073 const char *args_struct_type_name,
9074 const char **instructions,
9075 uint32_t n_instructions)
9077 struct struct_type *args_struct_type = NULL;
9083 CHECK_NAME(name, EINVAL);
9084 CHECK(!action_find(p, name), EEXIST);
9086 if (args_struct_type_name) {
9087 CHECK_NAME(args_struct_type_name, EINVAL);
9088 args_struct_type = struct_type_find(p, args_struct_type_name);
9089 CHECK(args_struct_type, EINVAL);
9090 CHECK(!args_struct_type->var_size, EINVAL);
9093 /* Node allocation. */
9094 a = calloc(1, sizeof(struct action));
9096 if (args_struct_type) {
9097 a->args_endianness = calloc(args_struct_type->n_fields, sizeof(int));
9098 if (!a->args_endianness) {
9104 /* Node initialization. */
9105 strcpy(a->name, name);
9106 a->st = args_struct_type;
9107 a->id = p->n_actions;
9109 /* Instruction translation. */
9110 err = instruction_config(p, a, instructions, n_instructions);
9112 free(a->args_endianness);
9117 /* Node add to tailq. */
9118 TAILQ_INSERT_TAIL(&p->actions, a, node);
9125 action_build(struct rte_swx_pipeline *p)
9127 struct action *action;
9129 p->action_instructions = calloc(p->n_actions,
9130 sizeof(struct instruction *));
9131 CHECK(p->action_instructions, ENOMEM);
9133 TAILQ_FOREACH(action, &p->actions, node)
9134 p->action_instructions[action->id] = action->instructions;
9140 action_build_free(struct rte_swx_pipeline *p)
9142 free(p->action_instructions);
9143 p->action_instructions = NULL;
9147 action_free(struct rte_swx_pipeline *p)
9149 action_build_free(p);
9152 struct action *action;
9154 action = TAILQ_FIRST(&p->actions);
9158 TAILQ_REMOVE(&p->actions, action, node);
9159 free(action->instructions);
9165 action_arg_src_mov_count(struct action *a,
9167 struct instruction *instructions,
9168 struct instruction_data *instruction_data,
9169 uint32_t n_instructions)
9171 uint32_t offset, n_users = 0, i;
9174 (arg_id >= a->st->n_fields) ||
9176 !instruction_data ||
9180 offset = a->st->fields[arg_id].offset / 8;
9182 for (i = 0; i < n_instructions; i++) {
9183 struct instruction *instr = &instructions[i];
9184 struct instruction_data *data = &instruction_data[i];
9186 if (data->invalid ||
9187 ((instr->type != INSTR_MOV) && (instr->type != INSTR_MOV_HM)) ||
9188 instr->mov.src.struct_id ||
9189 (instr->mov.src.offset != offset))
9201 static struct table_type *
9202 table_type_find(struct rte_swx_pipeline *p, const char *name)
9204 struct table_type *elem;
9206 TAILQ_FOREACH(elem, &p->table_types, node)
9207 if (strcmp(elem->name, name) == 0)
9213 static struct table_type *
9214 table_type_resolve(struct rte_swx_pipeline *p,
9215 const char *recommended_type_name,
9216 enum rte_swx_table_match_type match_type)
9218 struct table_type *elem;
9220 /* Only consider the recommended type if the match type is correct. */
9221 if (recommended_type_name)
9222 TAILQ_FOREACH(elem, &p->table_types, node)
9223 if (!strcmp(elem->name, recommended_type_name) &&
9224 (elem->match_type == match_type))
9227 /* Ignore the recommended type and get the first element with this match
9230 TAILQ_FOREACH(elem, &p->table_types, node)
9231 if (elem->match_type == match_type)
9237 static struct table *
9238 table_find(struct rte_swx_pipeline *p, const char *name)
9242 TAILQ_FOREACH(elem, &p->tables, node)
9243 if (strcmp(elem->name, name) == 0)
9249 static struct table *
9250 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9252 struct table *table = NULL;
9254 TAILQ_FOREACH(table, &p->tables, node)
9255 if (table->id == id)
9262 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
9264 enum rte_swx_table_match_type match_type,
9265 struct rte_swx_table_ops *ops)
9267 struct table_type *elem;
9271 CHECK_NAME(name, EINVAL);
9272 CHECK(!table_type_find(p, name), EEXIST);
9275 CHECK(ops->create, EINVAL);
9276 CHECK(ops->lkp, EINVAL);
9277 CHECK(ops->free, EINVAL);
9279 /* Node allocation. */
9280 elem = calloc(1, sizeof(struct table_type));
9281 CHECK(elem, ENOMEM);
9283 /* Node initialization. */
9284 strcpy(elem->name, name);
9285 elem->match_type = match_type;
9286 memcpy(&elem->ops, ops, sizeof(*ops));
9288 /* Node add to tailq. */
9289 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
9295 table_match_type_resolve(struct rte_swx_match_field_params *fields,
9297 enum rte_swx_table_match_type *match_type)
9299 uint32_t n_fields_em = 0, n_fields_lpm = 0, i;
9301 for (i = 0; i < n_fields; i++) {
9302 struct rte_swx_match_field_params *f = &fields[i];
9304 if (f->match_type == RTE_SWX_TABLE_MATCH_EXACT)
9307 if (f->match_type == RTE_SWX_TABLE_MATCH_LPM)
9311 if ((n_fields_lpm > 1) ||
9312 (n_fields_lpm && (n_fields_em != n_fields - 1)))
9315 *match_type = (n_fields_em == n_fields) ?
9316 RTE_SWX_TABLE_MATCH_EXACT :
9317 RTE_SWX_TABLE_MATCH_WILDCARD;
9323 table_match_fields_check(struct rte_swx_pipeline *p,
9324 struct rte_swx_pipeline_table_params *params,
9325 struct header **header)
9327 struct header *h0 = NULL;
9328 struct field *hf, *mf;
9329 uint32_t *offset = NULL, i;
9332 /* Return if no match fields. */
9333 if (!params->n_fields) {
9334 if (params->fields) {
9345 /* Memory allocation. */
9346 offset = calloc(params->n_fields, sizeof(uint32_t));
9352 /* Check that all the match fields belong to either the same header or
9355 hf = header_field_parse(p, params->fields[0].name, &h0);
9356 mf = metadata_field_parse(p, params->fields[0].name);
9357 if ((!hf && !mf) || (hf && hf->var_size)) {
9362 offset[0] = h0 ? hf->offset : mf->offset;
9364 for (i = 1; i < params->n_fields; i++)
9368 hf = header_field_parse(p, params->fields[i].name, &h);
9369 if (!hf || (h->id != h0->id) || hf->var_size) {
9374 offset[i] = hf->offset;
9376 mf = metadata_field_parse(p, params->fields[i].name);
9382 offset[i] = mf->offset;
9385 /* Check that there are no duplicated match fields. */
9386 for (i = 0; i < params->n_fields; i++) {
9389 for (j = 0; j < i; j++)
9390 if (offset[j] == offset[i]) {
9406 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9408 struct rte_swx_pipeline_table_params *params,
9409 const char *recommended_table_type_name,
9413 struct table_type *type;
9415 struct action *default_action;
9416 struct header *header = NULL;
9417 uint32_t action_data_size_max = 0, i;
9422 CHECK_NAME(name, EINVAL);
9423 CHECK(!table_find(p, name), EEXIST);
9424 CHECK(!selector_find(p, name), EEXIST);
9426 CHECK(params, EINVAL);
9429 status = table_match_fields_check(p, params, &header);
9433 /* Action checks. */
9434 CHECK(params->n_actions, EINVAL);
9435 CHECK(params->action_names, EINVAL);
9436 for (i = 0; i < params->n_actions; i++) {
9437 const char *action_name = params->action_names[i];
9439 uint32_t action_data_size;
9441 CHECK_NAME(action_name, EINVAL);
9443 a = action_find(p, action_name);
9446 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9447 if (action_data_size > action_data_size_max)
9448 action_data_size_max = action_data_size;
9451 CHECK_NAME(params->default_action_name, EINVAL);
9452 for (i = 0; i < p->n_actions; i++)
9453 if (!strcmp(params->action_names[i],
9454 params->default_action_name))
9456 CHECK(i < params->n_actions, EINVAL);
9457 default_action = action_find(p, params->default_action_name);
9458 CHECK((default_action->st && params->default_action_data) ||
9459 !params->default_action_data, EINVAL);
9461 /* Table type checks. */
9462 if (recommended_table_type_name)
9463 CHECK_NAME(recommended_table_type_name, EINVAL);
9465 if (params->n_fields) {
9466 enum rte_swx_table_match_type match_type;
9468 status = table_match_type_resolve(params->fields, params->n_fields, &match_type);
9472 type = table_type_resolve(p, recommended_table_type_name, match_type);
9473 CHECK(type, EINVAL);
9478 /* Memory allocation. */
9479 t = calloc(1, sizeof(struct table));
9482 t->fields = calloc(params->n_fields, sizeof(struct match_field));
9488 t->actions = calloc(params->n_actions, sizeof(struct action *));
9495 if (action_data_size_max) {
9496 t->default_action_data = calloc(1, action_data_size_max);
9497 if (!t->default_action_data) {
9505 /* Node initialization. */
9506 strcpy(t->name, name);
9507 if (args && args[0])
9508 strcpy(t->args, args);
9511 for (i = 0; i < params->n_fields; i++) {
9512 struct rte_swx_match_field_params *field = ¶ms->fields[i];
9513 struct match_field *f = &t->fields[i];
9515 f->match_type = field->match_type;
9517 header_field_parse(p, field->name, NULL) :
9518 metadata_field_parse(p, field->name);
9520 t->n_fields = params->n_fields;
9523 for (i = 0; i < params->n_actions; i++)
9524 t->actions[i] = action_find(p, params->action_names[i]);
9525 t->default_action = default_action;
9526 if (default_action->st)
9527 memcpy(t->default_action_data,
9528 params->default_action_data,
9529 default_action->st->n_bits / 8);
9530 t->n_actions = params->n_actions;
9531 t->default_action_is_const = params->default_action_is_const;
9532 t->action_data_size_max = action_data_size_max;
9535 t->id = p->n_tables;
9537 /* Node add to tailq. */
9538 TAILQ_INSERT_TAIL(&p->tables, t, node);
9544 static struct rte_swx_table_params *
9545 table_params_get(struct table *table)
9547 struct rte_swx_table_params *params;
9548 struct field *first, *last;
9550 uint32_t key_size, key_offset, action_data_size, i;
9552 /* Memory allocation. */
9553 params = calloc(1, sizeof(struct rte_swx_table_params));
9557 /* Find first (smallest offset) and last (biggest offset) match fields. */
9558 first = table->fields[0].field;
9559 last = table->fields[0].field;
9561 for (i = 0; i < table->n_fields; i++) {
9562 struct field *f = table->fields[i].field;
9564 if (f->offset < first->offset)
9567 if (f->offset > last->offset)
9571 /* Key offset and size. */
9572 key_offset = first->offset / 8;
9573 key_size = (last->offset + last->n_bits - first->offset) / 8;
9575 /* Memory allocation. */
9576 key_mask = calloc(1, key_size);
9583 for (i = 0; i < table->n_fields; i++) {
9584 struct field *f = table->fields[i].field;
9585 uint32_t start = (f->offset - first->offset) / 8;
9586 size_t size = f->n_bits / 8;
9588 memset(&key_mask[start], 0xFF, size);
9591 /* Action data size. */
9592 action_data_size = 0;
9593 for (i = 0; i < table->n_actions; i++) {
9594 struct action *action = table->actions[i];
9595 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9597 if (ads > action_data_size)
9598 action_data_size = ads;
9602 params->match_type = table->type->match_type;
9603 params->key_size = key_size;
9604 params->key_offset = key_offset;
9605 params->key_mask0 = key_mask;
9606 params->action_data_size = action_data_size;
9607 params->n_keys_max = table->size;
9613 table_params_free(struct rte_swx_table_params *params)
9618 free(params->key_mask0);
9623 table_stub_lkp(void *table __rte_unused,
9624 void *mailbox __rte_unused,
9625 uint8_t **key __rte_unused,
9626 uint64_t *action_id __rte_unused,
9627 uint8_t **action_data __rte_unused,
9631 return 1; /* DONE. */
9635 table_build(struct rte_swx_pipeline *p)
9639 /* Per pipeline: table statistics. */
9640 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9641 CHECK(p->table_stats, ENOMEM);
9643 for (i = 0; i < p->n_tables; i++) {
9644 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9645 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9648 /* Per thread: table runt-time. */
9649 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9650 struct thread *t = &p->threads[i];
9651 struct table *table;
9653 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9654 CHECK(t->tables, ENOMEM);
9656 TAILQ_FOREACH(table, &p->tables, node) {
9657 struct table_runtime *r = &t->tables[table->id];
9662 size = table->type->ops.mailbox_size_get();
9665 r->func = table->type->ops.lkp;
9669 r->mailbox = calloc(1, size);
9670 CHECK(r->mailbox, ENOMEM);
9674 r->key = table->header ?
9675 &t->structs[table->header->struct_id] :
9676 &t->structs[p->metadata_struct_id];
9678 r->func = table_stub_lkp;
9687 table_build_free(struct rte_swx_pipeline *p)
9691 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9692 struct thread *t = &p->threads[i];
9698 for (j = 0; j < p->n_tables; j++) {
9699 struct table_runtime *r = &t->tables[j];
9708 if (p->table_stats) {
9709 for (i = 0; i < p->n_tables; i++)
9710 free(p->table_stats[i].n_pkts_action);
9712 free(p->table_stats);
9717 table_free(struct rte_swx_pipeline *p)
9719 table_build_free(p);
9725 elem = TAILQ_FIRST(&p->tables);
9729 TAILQ_REMOVE(&p->tables, elem, node);
9731 free(elem->actions);
9732 free(elem->default_action_data);
9738 struct table_type *elem;
9740 elem = TAILQ_FIRST(&p->table_types);
9744 TAILQ_REMOVE(&p->table_types, elem, node);
9752 static struct selector *
9753 selector_find(struct rte_swx_pipeline *p, const char *name)
9757 TAILQ_FOREACH(s, &p->selectors, node)
9758 if (strcmp(s->name, name) == 0)
9764 static struct selector *
9765 selector_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9767 struct selector *s = NULL;
9769 TAILQ_FOREACH(s, &p->selectors, node)
9777 selector_fields_check(struct rte_swx_pipeline *p,
9778 struct rte_swx_pipeline_selector_params *params,
9779 struct header **header)
9781 struct header *h0 = NULL;
9782 struct field *hf, *mf;
9785 /* Return if no selector fields. */
9786 if (!params->n_selector_fields || !params->selector_field_names)
9789 /* Check that all the selector fields either belong to the same header
9790 * or are all meta-data fields.
9792 hf = header_field_parse(p, params->selector_field_names[0], &h0);
9793 mf = metadata_field_parse(p, params->selector_field_names[0]);
9797 for (i = 1; i < params->n_selector_fields; i++)
9801 hf = header_field_parse(p, params->selector_field_names[i], &h);
9802 if (!hf || (h->id != h0->id))
9805 mf = metadata_field_parse(p, params->selector_field_names[i]);
9810 /* Check that there are no duplicated match fields. */
9811 for (i = 0; i < params->n_selector_fields; i++) {
9812 const char *field_name = params->selector_field_names[i];
9815 for (j = i + 1; j < params->n_selector_fields; j++)
9816 if (!strcmp(params->selector_field_names[j], field_name))
9828 rte_swx_pipeline_selector_config(struct rte_swx_pipeline *p,
9830 struct rte_swx_pipeline_selector_params *params)
9833 struct header *selector_header = NULL;
9834 struct field *group_id_field, *member_id_field;
9840 CHECK_NAME(name, EINVAL);
9841 CHECK(!table_find(p, name), EEXIST);
9842 CHECK(!selector_find(p, name), EEXIST);
9844 CHECK(params, EINVAL);
9846 CHECK_NAME(params->group_id_field_name, EINVAL);
9847 group_id_field = metadata_field_parse(p, params->group_id_field_name);
9848 CHECK(group_id_field, EINVAL);
9850 for (i = 0; i < params->n_selector_fields; i++) {
9851 const char *field_name = params->selector_field_names[i];
9853 CHECK_NAME(field_name, EINVAL);
9855 status = selector_fields_check(p, params, &selector_header);
9859 CHECK_NAME(params->member_id_field_name, EINVAL);
9860 member_id_field = metadata_field_parse(p, params->member_id_field_name);
9861 CHECK(member_id_field, EINVAL);
9863 CHECK(params->n_groups_max, EINVAL);
9865 CHECK(params->n_members_per_group_max, EINVAL);
9867 /* Memory allocation. */
9868 s = calloc(1, sizeof(struct selector));
9874 s->selector_fields = calloc(params->n_selector_fields, sizeof(struct field *));
9875 if (!s->selector_fields) {
9880 /* Node initialization. */
9881 strcpy(s->name, name);
9883 s->group_id_field = group_id_field;
9885 for (i = 0; i < params->n_selector_fields; i++) {
9886 const char *field_name = params->selector_field_names[i];
9888 s->selector_fields[i] = selector_header ?
9889 header_field_parse(p, field_name, NULL) :
9890 metadata_field_parse(p, field_name);
9893 s->n_selector_fields = params->n_selector_fields;
9895 s->selector_header = selector_header;
9897 s->member_id_field = member_id_field;
9899 s->n_groups_max = params->n_groups_max;
9901 s->n_members_per_group_max = params->n_members_per_group_max;
9903 s->id = p->n_selectors;
9905 /* Node add to tailq. */
9906 TAILQ_INSERT_TAIL(&p->selectors, s, node);
9915 free(s->selector_fields);
9923 selector_params_free(struct rte_swx_table_selector_params *params)
9928 free(params->selector_mask);
9933 static struct rte_swx_table_selector_params *
9934 selector_table_params_get(struct selector *s)
9936 struct rte_swx_table_selector_params *params = NULL;
9937 struct field *first, *last;
9940 /* Memory allocation. */
9941 params = calloc(1, sizeof(struct rte_swx_table_selector_params));
9946 params->group_id_offset = s->group_id_field->offset / 8;
9948 /* Find first (smallest offset) and last (biggest offset) selector fields. */
9949 first = s->selector_fields[0];
9950 last = s->selector_fields[0];
9952 for (i = 0; i < s->n_selector_fields; i++) {
9953 struct field *f = s->selector_fields[i];
9955 if (f->offset < first->offset)
9958 if (f->offset > last->offset)
9962 /* Selector offset and size. */
9963 params->selector_offset = first->offset / 8;
9964 params->selector_size = (last->offset + last->n_bits - first->offset) / 8;
9966 /* Memory allocation. */
9967 params->selector_mask = calloc(1, params->selector_size);
9968 if (!params->selector_mask)
9971 /* Selector mask. */
9972 for (i = 0; i < s->n_selector_fields; i++) {
9973 struct field *f = s->selector_fields[i];
9974 uint32_t start = (f->offset - first->offset) / 8;
9975 size_t size = f->n_bits / 8;
9977 memset(¶ms->selector_mask[start], 0xFF, size);
9981 params->member_id_offset = s->member_id_field->offset / 8;
9983 /* Maximum number of groups. */
9984 params->n_groups_max = s->n_groups_max;
9986 /* Maximum number of members per group. */
9987 params->n_members_per_group_max = s->n_members_per_group_max;
9992 selector_params_free(params);
9997 selector_build_free(struct rte_swx_pipeline *p)
10001 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10002 struct thread *t = &p->threads[i];
10008 for (j = 0; j < p->n_selectors; j++) {
10009 struct selector_runtime *r = &t->selectors[j];
10014 free(t->selectors);
10015 t->selectors = NULL;
10018 free(p->selector_stats);
10019 p->selector_stats = NULL;
10023 selector_build(struct rte_swx_pipeline *p)
10028 /* Per pipeline: selector statistics. */
10029 p->selector_stats = calloc(p->n_selectors, sizeof(struct selector_statistics));
10030 if (!p->selector_stats) {
10035 /* Per thread: selector run-time. */
10036 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10037 struct thread *t = &p->threads[i];
10038 struct selector *s;
10040 t->selectors = calloc(p->n_selectors, sizeof(struct selector_runtime));
10041 if (!t->selectors) {
10046 TAILQ_FOREACH(s, &p->selectors, node) {
10047 struct selector_runtime *r = &t->selectors[s->id];
10051 size = rte_swx_table_selector_mailbox_size_get();
10053 r->mailbox = calloc(1, size);
10060 /* r->group_id_buffer. */
10061 r->group_id_buffer = &t->structs[p->metadata_struct_id];
10063 /* r->selector_buffer. */
10064 r->selector_buffer = s->selector_header ?
10065 &t->structs[s->selector_header->struct_id] :
10066 &t->structs[p->metadata_struct_id];
10068 /* r->member_id_buffer. */
10069 r->member_id_buffer = &t->structs[p->metadata_struct_id];
10076 selector_build_free(p);
10081 selector_free(struct rte_swx_pipeline *p)
10083 selector_build_free(p);
10085 /* Selector tables. */
10087 struct selector *elem;
10089 elem = TAILQ_FIRST(&p->selectors);
10093 TAILQ_REMOVE(&p->selectors, elem, node);
10094 free(elem->selector_fields);
10103 table_state_build(struct rte_swx_pipeline *p)
10105 struct table *table;
10106 struct selector *s;
10108 p->table_state = calloc(p->n_tables + p->n_selectors,
10109 sizeof(struct rte_swx_table_state));
10110 CHECK(p->table_state, ENOMEM);
10112 TAILQ_FOREACH(table, &p->tables, node) {
10113 struct rte_swx_table_state *ts = &p->table_state[table->id];
10116 struct rte_swx_table_params *params;
10119 params = table_params_get(table);
10120 CHECK(params, ENOMEM);
10122 ts->obj = table->type->ops.create(params,
10127 table_params_free(params);
10128 CHECK(ts->obj, ENODEV);
10131 /* ts->default_action_data. */
10132 if (table->action_data_size_max) {
10133 ts->default_action_data =
10134 malloc(table->action_data_size_max);
10135 CHECK(ts->default_action_data, ENOMEM);
10137 memcpy(ts->default_action_data,
10138 table->default_action_data,
10139 table->action_data_size_max);
10142 /* ts->default_action_id. */
10143 ts->default_action_id = table->default_action->id;
10146 TAILQ_FOREACH(s, &p->selectors, node) {
10147 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + s->id];
10148 struct rte_swx_table_selector_params *params;
10151 params = selector_table_params_get(s);
10152 CHECK(params, ENOMEM);
10154 ts->obj = rte_swx_table_selector_create(params, NULL, p->numa_node);
10156 selector_params_free(params);
10157 CHECK(ts->obj, ENODEV);
10164 table_state_build_free(struct rte_swx_pipeline *p)
10168 if (!p->table_state)
10171 for (i = 0; i < p->n_tables; i++) {
10172 struct rte_swx_table_state *ts = &p->table_state[i];
10173 struct table *table = table_find_by_id(p, i);
10176 if (table->type && ts->obj)
10177 table->type->ops.free(ts->obj);
10179 /* ts->default_action_data. */
10180 free(ts->default_action_data);
10183 for (i = 0; i < p->n_selectors; i++) {
10184 struct rte_swx_table_state *ts = &p->table_state[p->n_tables + i];
10188 rte_swx_table_selector_free(ts->obj);
10191 free(p->table_state);
10192 p->table_state = NULL;
10196 table_state_free(struct rte_swx_pipeline *p)
10198 table_state_build_free(p);
10204 static struct regarray *
10205 regarray_find(struct rte_swx_pipeline *p, const char *name)
10207 struct regarray *elem;
10209 TAILQ_FOREACH(elem, &p->regarrays, node)
10210 if (!strcmp(elem->name, name))
10216 static struct regarray *
10217 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10219 struct regarray *elem = NULL;
10221 TAILQ_FOREACH(elem, &p->regarrays, node)
10222 if (elem->id == id)
10229 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
10234 struct regarray *r;
10238 CHECK_NAME(name, EINVAL);
10239 CHECK(!regarray_find(p, name), EEXIST);
10241 CHECK(size, EINVAL);
10242 size = rte_align32pow2(size);
10244 /* Memory allocation. */
10245 r = calloc(1, sizeof(struct regarray));
10248 /* Node initialization. */
10249 strcpy(r->name, name);
10250 r->init_val = init_val;
10252 r->id = p->n_regarrays;
10254 /* Node add to tailq. */
10255 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
10262 regarray_build(struct rte_swx_pipeline *p)
10264 struct regarray *regarray;
10266 if (!p->n_regarrays)
10269 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
10270 CHECK(p->regarray_runtime, ENOMEM);
10272 TAILQ_FOREACH(regarray, &p->regarrays, node) {
10273 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
10276 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
10277 RTE_CACHE_LINE_SIZE,
10279 CHECK(r->regarray, ENOMEM);
10281 if (regarray->init_val)
10282 for (i = 0; i < regarray->size; i++)
10283 r->regarray[i] = regarray->init_val;
10285 r->size_mask = regarray->size - 1;
10292 regarray_build_free(struct rte_swx_pipeline *p)
10296 if (!p->regarray_runtime)
10299 for (i = 0; i < p->n_regarrays; i++) {
10300 struct regarray *regarray = regarray_find_by_id(p, i);
10301 struct regarray_runtime *r = &p->regarray_runtime[i];
10303 env_free(r->regarray, regarray->size * sizeof(uint64_t));
10306 free(p->regarray_runtime);
10307 p->regarray_runtime = NULL;
10311 regarray_free(struct rte_swx_pipeline *p)
10313 regarray_build_free(p);
10316 struct regarray *elem;
10318 elem = TAILQ_FIRST(&p->regarrays);
10322 TAILQ_REMOVE(&p->regarrays, elem, node);
10330 static struct meter_profile *
10331 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
10333 struct meter_profile *elem;
10335 TAILQ_FOREACH(elem, &p->meter_profiles, node)
10336 if (!strcmp(elem->name, name))
10342 static struct metarray *
10343 metarray_find(struct rte_swx_pipeline *p, const char *name)
10345 struct metarray *elem;
10347 TAILQ_FOREACH(elem, &p->metarrays, node)
10348 if (!strcmp(elem->name, name))
10354 static struct metarray *
10355 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
10357 struct metarray *elem = NULL;
10359 TAILQ_FOREACH(elem, &p->metarrays, node)
10360 if (elem->id == id)
10367 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
10371 struct metarray *m;
10375 CHECK_NAME(name, EINVAL);
10376 CHECK(!metarray_find(p, name), EEXIST);
10378 CHECK(size, EINVAL);
10379 size = rte_align32pow2(size);
10381 /* Memory allocation. */
10382 m = calloc(1, sizeof(struct metarray));
10385 /* Node initialization. */
10386 strcpy(m->name, name);
10388 m->id = p->n_metarrays;
10390 /* Node add to tailq. */
10391 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
10397 struct meter_profile meter_profile_default = {
10406 .cir_bytes_per_period = 1,
10408 .pir_bytes_per_period = 1,
10415 meter_init(struct meter *m)
10417 memset(m, 0, sizeof(struct meter));
10418 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
10419 m->profile = &meter_profile_default;
10420 m->color_mask = RTE_COLOR_GREEN;
10422 meter_profile_default.n_users++;
10426 metarray_build(struct rte_swx_pipeline *p)
10428 struct metarray *m;
10430 if (!p->n_metarrays)
10433 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
10434 CHECK(p->metarray_runtime, ENOMEM);
10436 TAILQ_FOREACH(m, &p->metarrays, node) {
10437 struct metarray_runtime *r = &p->metarray_runtime[m->id];
10440 r->metarray = env_malloc(m->size * sizeof(struct meter),
10441 RTE_CACHE_LINE_SIZE,
10443 CHECK(r->metarray, ENOMEM);
10445 for (i = 0; i < m->size; i++)
10446 meter_init(&r->metarray[i]);
10448 r->size_mask = m->size - 1;
10455 metarray_build_free(struct rte_swx_pipeline *p)
10459 if (!p->metarray_runtime)
10462 for (i = 0; i < p->n_metarrays; i++) {
10463 struct metarray *m = metarray_find_by_id(p, i);
10464 struct metarray_runtime *r = &p->metarray_runtime[i];
10466 env_free(r->metarray, m->size * sizeof(struct meter));
10469 free(p->metarray_runtime);
10470 p->metarray_runtime = NULL;
10474 metarray_free(struct rte_swx_pipeline *p)
10476 metarray_build_free(p);
10478 /* Meter arrays. */
10480 struct metarray *elem;
10482 elem = TAILQ_FIRST(&p->metarrays);
10486 TAILQ_REMOVE(&p->metarrays, elem, node);
10490 /* Meter profiles. */
10492 struct meter_profile *elem;
10494 elem = TAILQ_FIRST(&p->meter_profiles);
10498 TAILQ_REMOVE(&p->meter_profiles, elem, node);
10507 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
10509 struct rte_swx_pipeline *pipeline;
10511 /* Check input parameters. */
10514 /* Memory allocation. */
10515 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
10516 CHECK(pipeline, ENOMEM);
10518 /* Initialization. */
10519 TAILQ_INIT(&pipeline->struct_types);
10520 TAILQ_INIT(&pipeline->port_in_types);
10521 TAILQ_INIT(&pipeline->ports_in);
10522 TAILQ_INIT(&pipeline->port_out_types);
10523 TAILQ_INIT(&pipeline->ports_out);
10524 TAILQ_INIT(&pipeline->extern_types);
10525 TAILQ_INIT(&pipeline->extern_objs);
10526 TAILQ_INIT(&pipeline->extern_funcs);
10527 TAILQ_INIT(&pipeline->headers);
10528 TAILQ_INIT(&pipeline->actions);
10529 TAILQ_INIT(&pipeline->table_types);
10530 TAILQ_INIT(&pipeline->tables);
10531 TAILQ_INIT(&pipeline->selectors);
10532 TAILQ_INIT(&pipeline->regarrays);
10533 TAILQ_INIT(&pipeline->meter_profiles);
10534 TAILQ_INIT(&pipeline->metarrays);
10536 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
10537 pipeline->numa_node = numa_node;
10544 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
10549 free(p->instructions);
10553 table_state_free(p);
10559 extern_func_free(p);
10560 extern_obj_free(p);
10569 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
10570 const char **instructions,
10571 uint32_t n_instructions)
10576 err = instruction_config(p, NULL, instructions, n_instructions);
10580 /* Thread instruction pointer reset. */
10581 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
10582 struct thread *t = &p->threads[i];
10584 thread_ip_reset(p, t);
10591 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
10596 CHECK(p->build_done == 0, EEXIST);
10598 status = port_in_build(p);
10602 status = port_out_build(p);
10606 status = struct_build(p);
10610 status = extern_obj_build(p);
10614 status = extern_func_build(p);
10618 status = header_build(p);
10622 status = metadata_build(p);
10626 status = action_build(p);
10630 status = table_build(p);
10634 status = selector_build(p);
10638 status = table_state_build(p);
10642 status = regarray_build(p);
10646 status = metarray_build(p);
10654 metarray_build_free(p);
10655 regarray_build_free(p);
10656 table_state_build_free(p);
10657 selector_build_free(p);
10658 table_build_free(p);
10659 action_build_free(p);
10660 metadata_build_free(p);
10661 header_build_free(p);
10662 extern_func_build_free(p);
10663 extern_obj_build_free(p);
10664 port_out_build_free(p);
10665 port_in_build_free(p);
10666 struct_build_free(p);
10672 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
10676 for (i = 0; i < n_instructions; i++)
10681 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
10685 for (i = 0; i < p->n_ports_out; i++) {
10686 struct port_out_runtime *port = &p->out[i];
10689 port->flush(port->obj);
10697 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
10698 struct rte_swx_ctl_pipeline_info *pipeline)
10700 struct action *action;
10701 struct table *table;
10702 uint32_t n_actions = 0, n_tables = 0;
10704 if (!p || !pipeline)
10707 TAILQ_FOREACH(action, &p->actions, node)
10710 TAILQ_FOREACH(table, &p->tables, node)
10713 pipeline->n_ports_in = p->n_ports_in;
10714 pipeline->n_ports_out = p->n_ports_out;
10715 pipeline->n_actions = n_actions;
10716 pipeline->n_tables = n_tables;
10717 pipeline->n_selectors = p->n_selectors;
10718 pipeline->n_regarrays = p->n_regarrays;
10719 pipeline->n_metarrays = p->n_metarrays;
10725 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
10727 if (!p || !numa_node)
10730 *numa_node = p->numa_node;
10735 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
10736 uint32_t action_id,
10737 struct rte_swx_ctl_action_info *action)
10739 struct action *a = NULL;
10741 if (!p || (action_id >= p->n_actions) || !action)
10744 a = action_find_by_id(p, action_id);
10748 strcpy(action->name, a->name);
10749 action->n_args = a->st ? a->st->n_fields : 0;
10754 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10755 uint32_t action_id,
10756 uint32_t action_arg_id,
10757 struct rte_swx_ctl_action_arg_info *action_arg)
10759 struct action *a = NULL;
10760 struct field *arg = NULL;
10762 if (!p || (action_id >= p->n_actions) || !action_arg)
10765 a = action_find_by_id(p, action_id);
10766 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10769 arg = &a->st->fields[action_arg_id];
10770 strcpy(action_arg->name, arg->name);
10771 action_arg->n_bits = arg->n_bits;
10772 action_arg->is_network_byte_order = a->args_endianness[action_arg_id];
10778 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10780 struct rte_swx_ctl_table_info *table)
10782 struct table *t = NULL;
10787 t = table_find_by_id(p, table_id);
10791 strcpy(table->name, t->name);
10792 strcpy(table->args, t->args);
10793 table->n_match_fields = t->n_fields;
10794 table->n_actions = t->n_actions;
10795 table->default_action_is_const = t->default_action_is_const;
10796 table->size = t->size;
10801 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10803 uint32_t match_field_id,
10804 struct rte_swx_ctl_table_match_field_info *match_field)
10807 struct match_field *f;
10809 if (!p || (table_id >= p->n_tables) || !match_field)
10812 t = table_find_by_id(p, table_id);
10813 if (!t || (match_field_id >= t->n_fields))
10816 f = &t->fields[match_field_id];
10817 match_field->match_type = f->match_type;
10818 match_field->is_header = t->header ? 1 : 0;
10819 match_field->n_bits = f->field->n_bits;
10820 match_field->offset = f->field->offset;
10826 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10828 uint32_t table_action_id,
10829 struct rte_swx_ctl_table_action_info *table_action)
10833 if (!p || (table_id >= p->n_tables) || !table_action)
10836 t = table_find_by_id(p, table_id);
10837 if (!t || (table_action_id >= t->n_actions))
10840 table_action->action_id = t->actions[table_action_id]->id;
10846 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10848 struct rte_swx_table_ops *table_ops,
10853 if (!p || (table_id >= p->n_tables))
10856 t = table_find_by_id(p, table_id);
10862 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10872 rte_swx_ctl_selector_info_get(struct rte_swx_pipeline *p,
10873 uint32_t selector_id,
10874 struct rte_swx_ctl_selector_info *selector)
10876 struct selector *s = NULL;
10878 if (!p || !selector)
10881 s = selector_find_by_id(p, selector_id);
10885 strcpy(selector->name, s->name);
10887 selector->n_selector_fields = s->n_selector_fields;
10888 selector->n_groups_max = s->n_groups_max;
10889 selector->n_members_per_group_max = s->n_members_per_group_max;
10895 rte_swx_ctl_selector_group_id_field_info_get(struct rte_swx_pipeline *p,
10896 uint32_t selector_id,
10897 struct rte_swx_ctl_table_match_field_info *field)
10899 struct selector *s;
10901 if (!p || (selector_id >= p->n_selectors) || !field)
10904 s = selector_find_by_id(p, selector_id);
10908 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10909 field->is_header = 0;
10910 field->n_bits = s->group_id_field->n_bits;
10911 field->offset = s->group_id_field->offset;
10917 rte_swx_ctl_selector_field_info_get(struct rte_swx_pipeline *p,
10918 uint32_t selector_id,
10919 uint32_t selector_field_id,
10920 struct rte_swx_ctl_table_match_field_info *field)
10922 struct selector *s;
10925 if (!p || (selector_id >= p->n_selectors) || !field)
10928 s = selector_find_by_id(p, selector_id);
10929 if (!s || (selector_field_id >= s->n_selector_fields))
10932 f = s->selector_fields[selector_field_id];
10933 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10934 field->is_header = s->selector_header ? 1 : 0;
10935 field->n_bits = f->n_bits;
10936 field->offset = f->offset;
10942 rte_swx_ctl_selector_member_id_field_info_get(struct rte_swx_pipeline *p,
10943 uint32_t selector_id,
10944 struct rte_swx_ctl_table_match_field_info *field)
10946 struct selector *s;
10948 if (!p || (selector_id >= p->n_selectors) || !field)
10951 s = selector_find_by_id(p, selector_id);
10955 field->match_type = RTE_SWX_TABLE_MATCH_EXACT;
10956 field->is_header = 0;
10957 field->n_bits = s->member_id_field->n_bits;
10958 field->offset = s->member_id_field->offset;
10964 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10965 struct rte_swx_table_state **table_state)
10967 if (!p || !table_state || !p->build_done)
10970 *table_state = p->table_state;
10975 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10976 struct rte_swx_table_state *table_state)
10978 if (!p || !table_state || !p->build_done)
10981 p->table_state = table_state;
10986 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10988 struct rte_swx_port_in_stats *stats)
10990 struct port_in *port;
10995 port = port_in_find(p, port_id);
10999 port->type->ops.stats_read(port->obj, stats);
11004 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
11006 struct rte_swx_port_out_stats *stats)
11008 struct port_out *port;
11013 port = port_out_find(p, port_id);
11017 port->type->ops.stats_read(port->obj, stats);
11022 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
11023 const char *table_name,
11024 struct rte_swx_table_stats *stats)
11026 struct table *table;
11027 struct table_statistics *table_stats;
11029 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
11032 table = table_find(p, table_name);
11036 table_stats = &p->table_stats[table->id];
11038 memcpy(stats->n_pkts_action,
11039 table_stats->n_pkts_action,
11040 p->n_actions * sizeof(uint64_t));
11042 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
11043 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
11049 rte_swx_ctl_pipeline_selector_stats_read(struct rte_swx_pipeline *p,
11050 const char *selector_name,
11051 struct rte_swx_pipeline_selector_stats *stats)
11053 struct selector *s;
11055 if (!p || !selector_name || !selector_name[0] || !stats)
11058 s = selector_find(p, selector_name);
11062 stats->n_pkts = p->selector_stats[s->id].n_pkts;
11068 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
11069 uint32_t regarray_id,
11070 struct rte_swx_ctl_regarray_info *regarray)
11072 struct regarray *r;
11074 if (!p || !regarray)
11077 r = regarray_find_by_id(p, regarray_id);
11081 strcpy(regarray->name, r->name);
11082 regarray->size = r->size;
11087 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
11088 const char *regarray_name,
11089 uint32_t regarray_index,
11092 struct regarray *regarray;
11093 struct regarray_runtime *r;
11095 if (!p || !regarray_name || !value)
11098 regarray = regarray_find(p, regarray_name);
11099 if (!regarray || (regarray_index >= regarray->size))
11102 r = &p->regarray_runtime[regarray->id];
11103 *value = r->regarray[regarray_index];
11108 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
11109 const char *regarray_name,
11110 uint32_t regarray_index,
11113 struct regarray *regarray;
11114 struct regarray_runtime *r;
11116 if (!p || !regarray_name)
11119 regarray = regarray_find(p, regarray_name);
11120 if (!regarray || (regarray_index >= regarray->size))
11123 r = &p->regarray_runtime[regarray->id];
11124 r->regarray[regarray_index] = value;
11129 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
11130 uint32_t metarray_id,
11131 struct rte_swx_ctl_metarray_info *metarray)
11133 struct metarray *m;
11135 if (!p || !metarray)
11138 m = metarray_find_by_id(p, metarray_id);
11142 strcpy(metarray->name, m->name);
11143 metarray->size = m->size;
11148 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
11150 struct rte_meter_trtcm_params *params)
11152 struct meter_profile *mp;
11156 CHECK_NAME(name, EINVAL);
11157 CHECK(params, EINVAL);
11158 CHECK(!meter_profile_find(p, name), EEXIST);
11160 /* Node allocation. */
11161 mp = calloc(1, sizeof(struct meter_profile));
11164 /* Node initialization. */
11165 strcpy(mp->name, name);
11166 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
11167 status = rte_meter_trtcm_profile_config(&mp->profile, params);
11173 /* Node add to tailq. */
11174 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
11180 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
11183 struct meter_profile *mp;
11186 CHECK_NAME(name, EINVAL);
11188 mp = meter_profile_find(p, name);
11190 CHECK(!mp->n_users, EBUSY);
11192 /* Remove node from tailq. */
11193 TAILQ_REMOVE(&p->meter_profiles, mp, node);
11200 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
11201 const char *metarray_name,
11202 uint32_t metarray_index)
11204 struct meter_profile *mp_old;
11205 struct metarray *metarray;
11206 struct metarray_runtime *metarray_runtime;
11210 CHECK_NAME(metarray_name, EINVAL);
11212 metarray = metarray_find(p, metarray_name);
11213 CHECK(metarray, EINVAL);
11214 CHECK(metarray_index < metarray->size, EINVAL);
11216 metarray_runtime = &p->metarray_runtime[metarray->id];
11217 m = &metarray_runtime->metarray[metarray_index];
11218 mp_old = m->profile;
11228 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
11229 const char *metarray_name,
11230 uint32_t metarray_index,
11231 const char *profile_name)
11233 struct meter_profile *mp, *mp_old;
11234 struct metarray *metarray;
11235 struct metarray_runtime *metarray_runtime;
11239 CHECK_NAME(metarray_name, EINVAL);
11241 metarray = metarray_find(p, metarray_name);
11242 CHECK(metarray, EINVAL);
11243 CHECK(metarray_index < metarray->size, EINVAL);
11245 mp = meter_profile_find(p, profile_name);
11248 metarray_runtime = &p->metarray_runtime[metarray->id];
11249 m = &metarray_runtime->metarray[metarray_index];
11250 mp_old = m->profile;
11252 memset(m, 0, sizeof(struct meter));
11253 rte_meter_trtcm_config(&m->m, &mp->profile);
11255 m->color_mask = RTE_COLORS;
11264 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
11265 const char *metarray_name,
11266 uint32_t metarray_index,
11267 struct rte_swx_ctl_meter_stats *stats)
11269 struct metarray *metarray;
11270 struct metarray_runtime *metarray_runtime;
11274 CHECK_NAME(metarray_name, EINVAL);
11276 metarray = metarray_find(p, metarray_name);
11277 CHECK(metarray, EINVAL);
11278 CHECK(metarray_index < metarray->size, EINVAL);
11280 CHECK(stats, EINVAL);
11282 metarray_runtime = &p->metarray_runtime[metarray->id];
11283 m = &metarray_runtime->metarray[metarray_index];
11285 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
11286 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));