1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <arpa/inet.h>
12 #include <rte_common.h>
13 #include <rte_prefetch.h>
14 #include <rte_byteorder.h>
15 #include <rte_cycles.h>
16 #include <rte_meter.h>
18 #include "rte_swx_pipeline.h"
19 #include "rte_swx_ctl.h"
21 #define CHECK(condition, err_code) \
27 #define CHECK_NAME(name, err_code) \
30 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
33 #define CHECK_INSTRUCTION(instr, err_code) \
36 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
37 RTE_SWX_INSTRUCTION_SIZE), \
45 #define TRACE(...) printf(__VA_ARGS__)
53 #define ntoh64(x) rte_be_to_cpu_64(x)
54 #define hton64(x) rte_cpu_to_be_64(x)
56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
58 #include <rte_malloc.h>
61 env_malloc(size_t size, size_t alignment, int numa_node)
63 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
67 env_free(void *start, size_t size __rte_unused)
77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
81 if (numa_available() == -1)
84 start = numa_alloc_onnode(size, numa_node);
88 memset(start, 0, size);
93 env_free(void *start, size_t size)
95 if (numa_available() == -1)
98 numa_free(start, size);
107 char name[RTE_SWX_NAME_SIZE];
113 TAILQ_ENTRY(struct_type) node;
114 char name[RTE_SWX_NAME_SIZE];
115 struct field *fields;
120 TAILQ_HEAD(struct_type_tailq, struct_type);
125 struct port_in_type {
126 TAILQ_ENTRY(port_in_type) node;
127 char name[RTE_SWX_NAME_SIZE];
128 struct rte_swx_port_in_ops ops;
131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
134 TAILQ_ENTRY(port_in) node;
135 struct port_in_type *type;
140 TAILQ_HEAD(port_in_tailq, port_in);
142 struct port_in_runtime {
143 rte_swx_port_in_pkt_rx_t pkt_rx;
150 struct port_out_type {
151 TAILQ_ENTRY(port_out_type) node;
152 char name[RTE_SWX_NAME_SIZE];
153 struct rte_swx_port_out_ops ops;
156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
159 TAILQ_ENTRY(port_out) node;
160 struct port_out_type *type;
165 TAILQ_HEAD(port_out_tailq, port_out);
167 struct port_out_runtime {
168 rte_swx_port_out_pkt_tx_t pkt_tx;
169 rte_swx_port_out_flush_t flush;
176 struct extern_type_member_func {
177 TAILQ_ENTRY(extern_type_member_func) node;
178 char name[RTE_SWX_NAME_SIZE];
179 rte_swx_extern_type_member_func_t func;
183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
186 TAILQ_ENTRY(extern_type) node;
187 char name[RTE_SWX_NAME_SIZE];
188 struct struct_type *mailbox_struct_type;
189 rte_swx_extern_type_constructor_t constructor;
190 rte_swx_extern_type_destructor_t destructor;
191 struct extern_type_member_func_tailq funcs;
195 TAILQ_HEAD(extern_type_tailq, extern_type);
198 TAILQ_ENTRY(extern_obj) node;
199 char name[RTE_SWX_NAME_SIZE];
200 struct extern_type *type;
206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
212 struct extern_obj_runtime {
215 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
222 TAILQ_ENTRY(extern_func) node;
223 char name[RTE_SWX_NAME_SIZE];
224 struct struct_type *mailbox_struct_type;
225 rte_swx_extern_func_t func;
230 TAILQ_HEAD(extern_func_tailq, extern_func);
232 struct extern_func_runtime {
234 rte_swx_extern_func_t func;
241 TAILQ_ENTRY(header) node;
242 char name[RTE_SWX_NAME_SIZE];
243 struct struct_type *st;
248 TAILQ_HEAD(header_tailq, header);
250 struct header_runtime {
254 struct header_out_runtime {
264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
265 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
266 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
267 * when transferred to packet meta-data and in NBO when transferred to packet
271 /* Notation conventions:
272 * -Header field: H = h.header.field (dst/src)
273 * -Meta-data field: M = m.field (dst/src)
274 * -Extern object mailbox field: E = e.field (dst/src)
275 * -Extern function mailbox field: F = f.field (dst/src)
276 * -Table action data field: T = t.field (src only)
277 * -Immediate value: I = 32-bit unsigned value (src only)
280 enum instruction_type {
287 /* extract h.header */
308 /* validate h.header */
311 /* invalidate h.header */
312 INSTR_HDR_INVALIDATE,
316 * dst = HMEF, src = HMEFTI
318 INSTR_MOV, /* dst = MEF, src = MEFT */
319 INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
320 INSTR_MOV_I, /* dst = HMEF, src = I */
322 /* dma h.header t.field
323 * memcpy(h.header, t.field, sizeof(h.header))
336 * dst = HMEF, src = HMEFTI
338 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
339 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
340 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
341 INSTR_ALU_ADD_HH, /* dst = H, src = H */
342 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
343 INSTR_ALU_ADD_HI, /* dst = H, src = I */
347 * dst = HMEF, src = HMEFTI
349 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
350 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
351 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
352 INSTR_ALU_SUB_HH, /* dst = H, src = H */
353 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
354 INSTR_ALU_SUB_HI, /* dst = H, src = I */
357 * dst = dst '+ src[0:1] '+ src[2:3] + ...
358 * dst = H, src = {H, h.header}
360 INSTR_ALU_CKADD_FIELD, /* src = H */
361 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
362 INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
368 INSTR_ALU_CKSUB_FIELD,
372 * dst = HMEF, src = HMEFTI
374 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
375 INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
376 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
380 * dst = HMEF, src = HMEFTI
382 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
383 INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
384 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
388 * dst = HMEF, src = HMEFTI
390 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
391 INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
392 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
396 * dst = HMEF, src = HMEFTI
398 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
399 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
400 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
401 INSTR_ALU_SHL_HH, /* dst = H, src = H */
402 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
403 INSTR_ALU_SHL_HI, /* dst = H, src = I */
407 * dst = HMEF, src = HMEFTI
409 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
410 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
411 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
412 INSTR_ALU_SHR_HH, /* dst = H, src = H */
413 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
414 INSTR_ALU_SHR_HI, /* dst = H, src = I */
416 /* regprefetch REGARRAY index
417 * prefetch REGARRAY[index]
420 INSTR_REGPREFETCH_RH, /* index = H */
421 INSTR_REGPREFETCH_RM, /* index = MEFT */
422 INSTR_REGPREFETCH_RI, /* index = I */
424 /* regrd dst REGARRAY index
425 * dst = REGARRAY[index]
426 * dst = HMEF, index = HMEFTI
428 INSTR_REGRD_HRH, /* dst = H, index = H */
429 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
430 INSTR_REGRD_HRI, /* dst = H, index = I */
431 INSTR_REGRD_MRH, /* dst = MEF, index = H */
432 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
433 INSTR_REGRD_MRI, /* dst = MEF, index = I */
435 /* regwr REGARRAY index src
436 * REGARRAY[index] = src
437 * index = HMEFTI, src = HMEFTI
439 INSTR_REGWR_RHH, /* index = H, src = H */
440 INSTR_REGWR_RHM, /* index = H, src = MEFT */
441 INSTR_REGWR_RHI, /* index = H, src = I */
442 INSTR_REGWR_RMH, /* index = MEFT, src = H */
443 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
444 INSTR_REGWR_RMI, /* index = MEFT, src = I */
445 INSTR_REGWR_RIH, /* index = I, src = H */
446 INSTR_REGWR_RIM, /* index = I, src = MEFT */
447 INSTR_REGWR_RII, /* index = I, src = I */
449 /* regadd REGARRAY index src
450 * REGARRAY[index] += src
451 * index = HMEFTI, src = HMEFTI
453 INSTR_REGADD_RHH, /* index = H, src = H */
454 INSTR_REGADD_RHM, /* index = H, src = MEFT */
455 INSTR_REGADD_RHI, /* index = H, src = I */
456 INSTR_REGADD_RMH, /* index = MEFT, src = H */
457 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
458 INSTR_REGADD_RMI, /* index = MEFT, src = I */
459 INSTR_REGADD_RIH, /* index = I, src = H */
460 INSTR_REGADD_RIM, /* index = I, src = MEFT */
461 INSTR_REGADD_RII, /* index = I, src = I */
463 /* metprefetch METARRAY index
464 * prefetch METARRAY[index]
467 INSTR_METPREFETCH_H, /* index = H */
468 INSTR_METPREFETCH_M, /* index = MEFT */
469 INSTR_METPREFETCH_I, /* index = I */
471 /* meter METARRAY index length color_in color_out
472 * color_out = meter(METARRAY[index], length, color_in)
473 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
475 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
476 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
477 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
478 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
479 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
480 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
481 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
482 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
483 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
484 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
485 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
486 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
491 /* extern e.obj.func */
502 /* jmpv LABEL h.header
503 * Jump if header is valid
507 /* jmpnv LABEL h.header
508 * Jump if header is invalid
513 * Jump if table lookup hit
518 * Jump if table lookup miss
525 INSTR_JMP_ACTION_HIT,
527 /* jmpna LABEL ACTION
528 * Jump if action not run
530 INSTR_JMP_ACTION_MISS,
533 * Jump is a is equal to b
534 * a = HMEFT, b = HMEFTI
536 INSTR_JMP_EQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
537 INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
538 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
541 * Jump is a is not equal to b
542 * a = HMEFT, b = HMEFTI
544 INSTR_JMP_NEQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
545 INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
546 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
549 * Jump if a is less than b
550 * a = HMEFT, b = HMEFTI
552 INSTR_JMP_LT, /* a = MEF, b = MEF */
553 INSTR_JMP_LT_MH, /* a = MEF, b = H */
554 INSTR_JMP_LT_HM, /* a = H, b = MEF */
555 INSTR_JMP_LT_HH, /* a = H, b = H */
556 INSTR_JMP_LT_MI, /* a = MEF, b = I */
557 INSTR_JMP_LT_HI, /* a = H, b = I */
560 * Jump if a is greater than b
561 * a = HMEFT, b = HMEFTI
563 INSTR_JMP_GT, /* a = MEF, b = MEF */
564 INSTR_JMP_GT_MH, /* a = MEF, b = H */
565 INSTR_JMP_GT_HM, /* a = H, b = MEF */
566 INSTR_JMP_GT_HH, /* a = H, b = H */
567 INSTR_JMP_GT_MI, /* a = MEF, b = I */
568 INSTR_JMP_GT_HI, /* a = H, b = I */
576 struct instr_operand {
591 uint8_t header_id[8];
592 uint8_t struct_id[8];
597 struct instr_hdr_validity {
605 struct instr_extern_obj {
610 struct instr_extern_func {
614 struct instr_dst_src {
615 struct instr_operand dst;
617 struct instr_operand src;
622 struct instr_regarray {
627 struct instr_operand idx;
632 struct instr_operand dstsrc;
642 struct instr_operand idx;
646 struct instr_operand length;
649 struct instr_operand color_in;
650 uint32_t color_in_val;
653 struct instr_operand color_out;
658 uint8_t header_id[8];
659 uint8_t struct_id[8];
670 struct instruction *ip;
673 struct instr_operand a;
679 struct instr_operand b;
685 enum instruction_type type;
688 struct instr_hdr_validity valid;
689 struct instr_dst_src mov;
690 struct instr_regarray regarray;
691 struct instr_meter meter;
692 struct instr_dma dma;
693 struct instr_dst_src alu;
694 struct instr_table table;
695 struct instr_extern_obj ext_obj;
696 struct instr_extern_func ext_func;
697 struct instr_jmp jmp;
701 struct instruction_data {
702 char label[RTE_SWX_NAME_SIZE];
703 char jmp_label[RTE_SWX_NAME_SIZE];
704 uint32_t n_users; /* user = jmp instruction to this instruction. */
712 TAILQ_ENTRY(action) node;
713 char name[RTE_SWX_NAME_SIZE];
714 struct struct_type *st;
715 struct instruction *instructions;
716 uint32_t n_instructions;
720 TAILQ_HEAD(action_tailq, action);
726 TAILQ_ENTRY(table_type) node;
727 char name[RTE_SWX_NAME_SIZE];
728 enum rte_swx_table_match_type match_type;
729 struct rte_swx_table_ops ops;
732 TAILQ_HEAD(table_type_tailq, table_type);
735 enum rte_swx_table_match_type match_type;
740 TAILQ_ENTRY(table) node;
741 char name[RTE_SWX_NAME_SIZE];
742 char args[RTE_SWX_NAME_SIZE];
743 struct table_type *type; /* NULL when n_fields == 0. */
746 struct match_field *fields;
748 int is_header; /* Only valid when n_fields > 0. */
749 struct header *header; /* Only valid when n_fields > 0. */
752 struct action **actions;
753 struct action *default_action;
754 uint8_t *default_action_data;
756 int default_action_is_const;
757 uint32_t action_data_size_max;
763 TAILQ_HEAD(table_tailq, table);
765 struct table_runtime {
766 rte_swx_table_lookup_t func;
775 TAILQ_ENTRY(regarray) node;
776 char name[RTE_SWX_NAME_SIZE];
782 TAILQ_HEAD(regarray_tailq, regarray);
784 struct regarray_runtime {
792 struct meter_profile {
793 TAILQ_ENTRY(meter_profile) node;
794 char name[RTE_SWX_NAME_SIZE];
795 struct rte_meter_trtcm_params params;
796 struct rte_meter_trtcm_profile profile;
800 TAILQ_HEAD(meter_profile_tailq, meter_profile);
803 TAILQ_ENTRY(metarray) node;
804 char name[RTE_SWX_NAME_SIZE];
809 TAILQ_HEAD(metarray_tailq, metarray);
812 struct rte_meter_trtcm m;
813 struct meter_profile *profile;
814 enum rte_color color_mask;
817 uint64_t n_pkts[RTE_COLORS];
818 uint64_t n_bytes[RTE_COLORS];
821 struct metarray_runtime {
822 struct meter *metarray;
831 struct rte_swx_pkt pkt;
837 /* Packet headers. */
838 struct header_runtime *headers; /* Extracted or generated headers. */
839 struct header_out_runtime *headers_out; /* Emitted headers. */
840 uint8_t *header_storage;
841 uint8_t *header_out_storage;
842 uint64_t valid_headers;
843 uint32_t n_headers_out;
845 /* Packet meta-data. */
849 struct table_runtime *tables;
850 struct rte_swx_table_state *table_state;
852 int hit; /* 0 = Miss, 1 = Hit. */
854 /* Extern objects and functions. */
855 struct extern_obj_runtime *extern_objs;
856 struct extern_func_runtime *extern_funcs;
859 struct instruction *ip;
860 struct instruction *ret;
863 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
864 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
865 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
867 #define HEADER_VALID(thread, header_id) \
868 MASK64_BIT_GET((thread)->valid_headers, header_id)
870 #define ALU(thread, ip, operator) \
872 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
873 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
874 uint64_t dst64 = *dst64_ptr; \
875 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
876 uint64_t dst = dst64 & dst64_mask; \
878 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
879 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
880 uint64_t src64 = *src64_ptr; \
881 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
882 uint64_t src = src64 & src64_mask; \
884 uint64_t result = dst operator src; \
886 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
889 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
891 #define ALU_S(thread, ip, operator) \
893 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
894 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
895 uint64_t dst64 = *dst64_ptr; \
896 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
897 uint64_t dst = dst64 & dst64_mask; \
899 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
900 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
901 uint64_t src64 = *src64_ptr; \
902 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
904 uint64_t result = dst operator src; \
906 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
911 #define ALU_HM(thread, ip, operator) \
913 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
914 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
915 uint64_t dst64 = *dst64_ptr; \
916 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
917 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
919 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
920 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
921 uint64_t src64 = *src64_ptr; \
922 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
923 uint64_t src = src64 & src64_mask; \
925 uint64_t result = dst operator src; \
926 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
928 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
931 #define ALU_HH(thread, ip, operator) \
933 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
934 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
935 uint64_t dst64 = *dst64_ptr; \
936 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
937 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
939 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
940 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
941 uint64_t src64 = *src64_ptr; \
942 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
944 uint64_t result = dst operator src; \
945 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
947 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
959 #define ALU_I(thread, ip, operator) \
961 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
962 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
963 uint64_t dst64 = *dst64_ptr; \
964 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
965 uint64_t dst = dst64 & dst64_mask; \
967 uint64_t src = (ip)->alu.src_val; \
969 uint64_t result = dst operator src; \
971 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
976 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
978 #define ALU_HI(thread, ip, operator) \
980 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
981 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
982 uint64_t dst64 = *dst64_ptr; \
983 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
984 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
986 uint64_t src = (ip)->alu.src_val; \
988 uint64_t result = dst operator src; \
989 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
991 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1000 #define MOV(thread, ip) \
1002 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1003 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1004 uint64_t dst64 = *dst64_ptr; \
1005 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1007 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1008 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1009 uint64_t src64 = *src64_ptr; \
1010 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1011 uint64_t src = src64 & src64_mask; \
1013 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1016 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1018 #define MOV_S(thread, ip) \
1020 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1021 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1022 uint64_t dst64 = *dst64_ptr; \
1023 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1025 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1026 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1027 uint64_t src64 = *src64_ptr; \
1028 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1030 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1039 #define MOV_I(thread, ip) \
1041 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1042 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1043 uint64_t dst64 = *dst64_ptr; \
1044 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1046 uint64_t src = (ip)->mov.src_val; \
1048 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1051 #define JMP_CMP(thread, ip, operator) \
1053 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1054 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1055 uint64_t a64 = *a64_ptr; \
1056 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1057 uint64_t a = a64 & a64_mask; \
1059 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1060 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1061 uint64_t b64 = *b64_ptr; \
1062 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1063 uint64_t b = b64 & b64_mask; \
1065 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1068 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1070 #define JMP_CMP_S(thread, ip, operator) \
1072 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1073 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1074 uint64_t a64 = *a64_ptr; \
1075 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1076 uint64_t a = a64 & a64_mask; \
1078 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1079 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1080 uint64_t b64 = *b64_ptr; \
1081 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1083 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1086 #define JMP_CMP_MH JMP_CMP_S
1088 #define JMP_CMP_HM(thread, ip, operator) \
1090 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1091 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1092 uint64_t a64 = *a64_ptr; \
1093 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1095 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1096 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1097 uint64_t b64 = *b64_ptr; \
1098 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1099 uint64_t b = b64 & b64_mask; \
1101 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1104 #define JMP_CMP_HH(thread, ip, operator) \
1106 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1107 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1108 uint64_t a64 = *a64_ptr; \
1109 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1111 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1112 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1113 uint64_t b64 = *b64_ptr; \
1114 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1116 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1121 #define JMP_CMP_S JMP_CMP
1122 #define JMP_CMP_MH JMP_CMP
1123 #define JMP_CMP_HM JMP_CMP
1124 #define JMP_CMP_HH JMP_CMP
1128 #define JMP_CMP_I(thread, ip, operator) \
1130 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1131 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1132 uint64_t a64 = *a64_ptr; \
1133 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1134 uint64_t a = a64 & a64_mask; \
1136 uint64_t b = (ip)->jmp.b_val; \
1138 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1141 #define JMP_CMP_MI JMP_CMP_I
1143 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1145 #define JMP_CMP_HI(thread, ip, operator) \
1147 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1148 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1149 uint64_t a64 = *a64_ptr; \
1150 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1152 uint64_t b = (ip)->jmp.b_val; \
1154 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1159 #define JMP_CMP_HI JMP_CMP_I
1163 #define METADATA_READ(thread, offset, n_bits) \
1165 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1166 uint64_t m64 = *m64_ptr; \
1167 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1171 #define METADATA_WRITE(thread, offset, n_bits, value) \
1173 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1174 uint64_t m64 = *m64_ptr; \
1175 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1177 uint64_t m_new = value; \
1179 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1182 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1183 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1186 struct rte_swx_pipeline {
1187 struct struct_type_tailq struct_types;
1188 struct port_in_type_tailq port_in_types;
1189 struct port_in_tailq ports_in;
1190 struct port_out_type_tailq port_out_types;
1191 struct port_out_tailq ports_out;
1192 struct extern_type_tailq extern_types;
1193 struct extern_obj_tailq extern_objs;
1194 struct extern_func_tailq extern_funcs;
1195 struct header_tailq headers;
1196 struct struct_type *metadata_st;
1197 uint32_t metadata_struct_id;
1198 struct action_tailq actions;
1199 struct table_type_tailq table_types;
1200 struct table_tailq tables;
1201 struct regarray_tailq regarrays;
1202 struct meter_profile_tailq meter_profiles;
1203 struct metarray_tailq metarrays;
1205 struct port_in_runtime *in;
1206 struct port_out_runtime *out;
1207 struct instruction **action_instructions;
1208 struct rte_swx_table_state *table_state;
1209 struct regarray_runtime *regarray_runtime;
1210 struct metarray_runtime *metarray_runtime;
1211 struct instruction *instructions;
1212 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1215 uint32_t n_ports_in;
1216 uint32_t n_ports_out;
1217 uint32_t n_extern_objs;
1218 uint32_t n_extern_funcs;
1221 uint32_t n_regarrays;
1222 uint32_t n_metarrays;
1226 uint32_t n_instructions;
1234 static struct struct_type *
1235 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1237 struct struct_type *elem;
1239 TAILQ_FOREACH(elem, &p->struct_types, node)
1240 if (strcmp(elem->name, name) == 0)
1246 static struct field *
1247 struct_type_field_find(struct struct_type *st, const char *name)
1251 for (i = 0; i < st->n_fields; i++) {
1252 struct field *f = &st->fields[i];
1254 if (strcmp(f->name, name) == 0)
1262 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1264 struct rte_swx_field_params *fields,
1267 struct struct_type *st;
1271 CHECK_NAME(name, EINVAL);
1272 CHECK(fields, EINVAL);
1273 CHECK(n_fields, EINVAL);
1275 for (i = 0; i < n_fields; i++) {
1276 struct rte_swx_field_params *f = &fields[i];
1279 CHECK_NAME(f->name, EINVAL);
1280 CHECK(f->n_bits, EINVAL);
1281 CHECK(f->n_bits <= 64, EINVAL);
1282 CHECK((f->n_bits & 7) == 0, EINVAL);
1284 for (j = 0; j < i; j++) {
1285 struct rte_swx_field_params *f_prev = &fields[j];
1287 CHECK(strcmp(f->name, f_prev->name), EINVAL);
1291 CHECK(!struct_type_find(p, name), EEXIST);
1293 /* Node allocation. */
1294 st = calloc(1, sizeof(struct struct_type));
1297 st->fields = calloc(n_fields, sizeof(struct field));
1303 /* Node initialization. */
1304 strcpy(st->name, name);
1305 for (i = 0; i < n_fields; i++) {
1306 struct field *dst = &st->fields[i];
1307 struct rte_swx_field_params *src = &fields[i];
1309 strcpy(dst->name, src->name);
1310 dst->n_bits = src->n_bits;
1311 dst->offset = st->n_bits;
1313 st->n_bits += src->n_bits;
1315 st->n_fields = n_fields;
1317 /* Node add to tailq. */
1318 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1324 struct_build(struct rte_swx_pipeline *p)
1328 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1329 struct thread *t = &p->threads[i];
1331 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1332 CHECK(t->structs, ENOMEM);
1339 struct_build_free(struct rte_swx_pipeline *p)
1343 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1344 struct thread *t = &p->threads[i];
1352 struct_free(struct rte_swx_pipeline *p)
1354 struct_build_free(p);
1358 struct struct_type *elem;
1360 elem = TAILQ_FIRST(&p->struct_types);
1364 TAILQ_REMOVE(&p->struct_types, elem, node);
1373 static struct port_in_type *
1374 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1376 struct port_in_type *elem;
1381 TAILQ_FOREACH(elem, &p->port_in_types, node)
1382 if (strcmp(elem->name, name) == 0)
1389 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1391 struct rte_swx_port_in_ops *ops)
1393 struct port_in_type *elem;
1396 CHECK_NAME(name, EINVAL);
1398 CHECK(ops->create, EINVAL);
1399 CHECK(ops->free, EINVAL);
1400 CHECK(ops->pkt_rx, EINVAL);
1401 CHECK(ops->stats_read, EINVAL);
1403 CHECK(!port_in_type_find(p, name), EEXIST);
1405 /* Node allocation. */
1406 elem = calloc(1, sizeof(struct port_in_type));
1407 CHECK(elem, ENOMEM);
1409 /* Node initialization. */
1410 strcpy(elem->name, name);
1411 memcpy(&elem->ops, ops, sizeof(*ops));
1413 /* Node add to tailq. */
1414 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1419 static struct port_in *
1420 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1422 struct port_in *port;
1424 TAILQ_FOREACH(port, &p->ports_in, node)
1425 if (port->id == port_id)
1432 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1434 const char *port_type_name,
1437 struct port_in_type *type = NULL;
1438 struct port_in *port = NULL;
1443 CHECK(!port_in_find(p, port_id), EINVAL);
1445 CHECK_NAME(port_type_name, EINVAL);
1446 type = port_in_type_find(p, port_type_name);
1447 CHECK(type, EINVAL);
1449 obj = type->ops.create(args);
1452 /* Node allocation. */
1453 port = calloc(1, sizeof(struct port_in));
1454 CHECK(port, ENOMEM);
1456 /* Node initialization. */
1461 /* Node add to tailq. */
1462 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1463 if (p->n_ports_in < port_id + 1)
1464 p->n_ports_in = port_id + 1;
1470 port_in_build(struct rte_swx_pipeline *p)
1472 struct port_in *port;
1475 CHECK(p->n_ports_in, EINVAL);
1476 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1478 for (i = 0; i < p->n_ports_in; i++)
1479 CHECK(port_in_find(p, i), EINVAL);
1481 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1482 CHECK(p->in, ENOMEM);
1484 TAILQ_FOREACH(port, &p->ports_in, node) {
1485 struct port_in_runtime *in = &p->in[port->id];
1487 in->pkt_rx = port->type->ops.pkt_rx;
1488 in->obj = port->obj;
1495 port_in_build_free(struct rte_swx_pipeline *p)
1502 port_in_free(struct rte_swx_pipeline *p)
1504 port_in_build_free(p);
1508 struct port_in *port;
1510 port = TAILQ_FIRST(&p->ports_in);
1514 TAILQ_REMOVE(&p->ports_in, port, node);
1515 port->type->ops.free(port->obj);
1519 /* Input port types. */
1521 struct port_in_type *elem;
1523 elem = TAILQ_FIRST(&p->port_in_types);
1527 TAILQ_REMOVE(&p->port_in_types, elem, node);
1535 static struct port_out_type *
1536 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1538 struct port_out_type *elem;
1543 TAILQ_FOREACH(elem, &p->port_out_types, node)
1544 if (!strcmp(elem->name, name))
1551 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1553 struct rte_swx_port_out_ops *ops)
1555 struct port_out_type *elem;
1558 CHECK_NAME(name, EINVAL);
1560 CHECK(ops->create, EINVAL);
1561 CHECK(ops->free, EINVAL);
1562 CHECK(ops->pkt_tx, EINVAL);
1563 CHECK(ops->stats_read, EINVAL);
1565 CHECK(!port_out_type_find(p, name), EEXIST);
1567 /* Node allocation. */
1568 elem = calloc(1, sizeof(struct port_out_type));
1569 CHECK(elem, ENOMEM);
1571 /* Node initialization. */
1572 strcpy(elem->name, name);
1573 memcpy(&elem->ops, ops, sizeof(*ops));
1575 /* Node add to tailq. */
1576 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1581 static struct port_out *
1582 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1584 struct port_out *port;
1586 TAILQ_FOREACH(port, &p->ports_out, node)
1587 if (port->id == port_id)
1594 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1596 const char *port_type_name,
1599 struct port_out_type *type = NULL;
1600 struct port_out *port = NULL;
1605 CHECK(!port_out_find(p, port_id), EINVAL);
1607 CHECK_NAME(port_type_name, EINVAL);
1608 type = port_out_type_find(p, port_type_name);
1609 CHECK(type, EINVAL);
1611 obj = type->ops.create(args);
1614 /* Node allocation. */
1615 port = calloc(1, sizeof(struct port_out));
1616 CHECK(port, ENOMEM);
1618 /* Node initialization. */
1623 /* Node add to tailq. */
1624 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1625 if (p->n_ports_out < port_id + 1)
1626 p->n_ports_out = port_id + 1;
1632 port_out_build(struct rte_swx_pipeline *p)
1634 struct port_out *port;
1637 CHECK(p->n_ports_out, EINVAL);
1639 for (i = 0; i < p->n_ports_out; i++)
1640 CHECK(port_out_find(p, i), EINVAL);
1642 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1643 CHECK(p->out, ENOMEM);
1645 TAILQ_FOREACH(port, &p->ports_out, node) {
1646 struct port_out_runtime *out = &p->out[port->id];
1648 out->pkt_tx = port->type->ops.pkt_tx;
1649 out->flush = port->type->ops.flush;
1650 out->obj = port->obj;
1657 port_out_build_free(struct rte_swx_pipeline *p)
1664 port_out_free(struct rte_swx_pipeline *p)
1666 port_out_build_free(p);
1670 struct port_out *port;
1672 port = TAILQ_FIRST(&p->ports_out);
1676 TAILQ_REMOVE(&p->ports_out, port, node);
1677 port->type->ops.free(port->obj);
1681 /* Output port types. */
1683 struct port_out_type *elem;
1685 elem = TAILQ_FIRST(&p->port_out_types);
1689 TAILQ_REMOVE(&p->port_out_types, elem, node);
1697 static struct extern_type *
1698 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1700 struct extern_type *elem;
1702 TAILQ_FOREACH(elem, &p->extern_types, node)
1703 if (strcmp(elem->name, name) == 0)
1709 static struct extern_type_member_func *
1710 extern_type_member_func_find(struct extern_type *type, const char *name)
1712 struct extern_type_member_func *elem;
1714 TAILQ_FOREACH(elem, &type->funcs, node)
1715 if (strcmp(elem->name, name) == 0)
1721 static struct extern_obj *
1722 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1724 struct extern_obj *elem;
1726 TAILQ_FOREACH(elem, &p->extern_objs, node)
1727 if (strcmp(elem->name, name) == 0)
1733 static struct extern_type_member_func *
1734 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1736 struct extern_obj **obj)
1738 struct extern_obj *object;
1739 struct extern_type_member_func *func;
1740 char *object_name, *func_name;
1742 if (name[0] != 'e' || name[1] != '.')
1745 object_name = strdup(&name[2]);
1749 func_name = strchr(object_name, '.');
1758 object = extern_obj_find(p, object_name);
1764 func = extern_type_member_func_find(object->type, func_name);
1777 static struct field *
1778 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1780 struct extern_obj **object)
1782 struct extern_obj *obj;
1784 char *obj_name, *field_name;
1786 if ((name[0] != 'e') || (name[1] != '.'))
1789 obj_name = strdup(&name[2]);
1793 field_name = strchr(obj_name, '.');
1802 obj = extern_obj_find(p, obj_name);
1808 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1822 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1824 const char *mailbox_struct_type_name,
1825 rte_swx_extern_type_constructor_t constructor,
1826 rte_swx_extern_type_destructor_t destructor)
1828 struct extern_type *elem;
1829 struct struct_type *mailbox_struct_type;
1833 CHECK_NAME(name, EINVAL);
1834 CHECK(!extern_type_find(p, name), EEXIST);
1836 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1837 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1838 CHECK(mailbox_struct_type, EINVAL);
1840 CHECK(constructor, EINVAL);
1841 CHECK(destructor, EINVAL);
1843 /* Node allocation. */
1844 elem = calloc(1, sizeof(struct extern_type));
1845 CHECK(elem, ENOMEM);
1847 /* Node initialization. */
1848 strcpy(elem->name, name);
1849 elem->mailbox_struct_type = mailbox_struct_type;
1850 elem->constructor = constructor;
1851 elem->destructor = destructor;
1852 TAILQ_INIT(&elem->funcs);
1854 /* Node add to tailq. */
1855 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1861 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1862 const char *extern_type_name,
1864 rte_swx_extern_type_member_func_t member_func)
1866 struct extern_type *type;
1867 struct extern_type_member_func *type_member;
1871 CHECK_NAME(extern_type_name, EINVAL);
1872 type = extern_type_find(p, extern_type_name);
1873 CHECK(type, EINVAL);
1874 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1876 CHECK_NAME(name, EINVAL);
1877 CHECK(!extern_type_member_func_find(type, name), EEXIST);
1879 CHECK(member_func, EINVAL);
1881 /* Node allocation. */
1882 type_member = calloc(1, sizeof(struct extern_type_member_func));
1883 CHECK(type_member, ENOMEM);
1885 /* Node initialization. */
1886 strcpy(type_member->name, name);
1887 type_member->func = member_func;
1888 type_member->id = type->n_funcs;
1890 /* Node add to tailq. */
1891 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1898 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1899 const char *extern_type_name,
1903 struct extern_type *type;
1904 struct extern_obj *obj;
1909 CHECK_NAME(extern_type_name, EINVAL);
1910 type = extern_type_find(p, extern_type_name);
1911 CHECK(type, EINVAL);
1913 CHECK_NAME(name, EINVAL);
1914 CHECK(!extern_obj_find(p, name), EEXIST);
1916 /* Node allocation. */
1917 obj = calloc(1, sizeof(struct extern_obj));
1920 /* Object construction. */
1921 obj_handle = type->constructor(args);
1927 /* Node initialization. */
1928 strcpy(obj->name, name);
1930 obj->obj = obj_handle;
1931 obj->struct_id = p->n_structs;
1932 obj->id = p->n_extern_objs;
1934 /* Node add to tailq. */
1935 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1943 extern_obj_build(struct rte_swx_pipeline *p)
1947 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1948 struct thread *t = &p->threads[i];
1949 struct extern_obj *obj;
1951 t->extern_objs = calloc(p->n_extern_objs,
1952 sizeof(struct extern_obj_runtime));
1953 CHECK(t->extern_objs, ENOMEM);
1955 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1956 struct extern_obj_runtime *r =
1957 &t->extern_objs[obj->id];
1958 struct extern_type_member_func *func;
1959 uint32_t mailbox_size =
1960 obj->type->mailbox_struct_type->n_bits / 8;
1964 r->mailbox = calloc(1, mailbox_size);
1965 CHECK(r->mailbox, ENOMEM);
1967 TAILQ_FOREACH(func, &obj->type->funcs, node)
1968 r->funcs[func->id] = func->func;
1970 t->structs[obj->struct_id] = r->mailbox;
1978 extern_obj_build_free(struct rte_swx_pipeline *p)
1982 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1983 struct thread *t = &p->threads[i];
1986 if (!t->extern_objs)
1989 for (j = 0; j < p->n_extern_objs; j++) {
1990 struct extern_obj_runtime *r = &t->extern_objs[j];
1995 free(t->extern_objs);
1996 t->extern_objs = NULL;
2001 extern_obj_free(struct rte_swx_pipeline *p)
2003 extern_obj_build_free(p);
2005 /* Extern objects. */
2007 struct extern_obj *elem;
2009 elem = TAILQ_FIRST(&p->extern_objs);
2013 TAILQ_REMOVE(&p->extern_objs, elem, node);
2015 elem->type->destructor(elem->obj);
2021 struct extern_type *elem;
2023 elem = TAILQ_FIRST(&p->extern_types);
2027 TAILQ_REMOVE(&p->extern_types, elem, node);
2030 struct extern_type_member_func *func;
2032 func = TAILQ_FIRST(&elem->funcs);
2036 TAILQ_REMOVE(&elem->funcs, func, node);
2047 static struct extern_func *
2048 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2050 struct extern_func *elem;
2052 TAILQ_FOREACH(elem, &p->extern_funcs, node)
2053 if (strcmp(elem->name, name) == 0)
2059 static struct extern_func *
2060 extern_func_parse(struct rte_swx_pipeline *p,
2063 if (name[0] != 'f' || name[1] != '.')
2066 return extern_func_find(p, &name[2]);
2069 static struct field *
2070 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2072 struct extern_func **function)
2074 struct extern_func *func;
2076 char *func_name, *field_name;
2078 if ((name[0] != 'f') || (name[1] != '.'))
2081 func_name = strdup(&name[2]);
2085 field_name = strchr(func_name, '.');
2094 func = extern_func_find(p, func_name);
2100 f = struct_type_field_find(func->mailbox_struct_type, field_name);
2114 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2116 const char *mailbox_struct_type_name,
2117 rte_swx_extern_func_t func)
2119 struct extern_func *f;
2120 struct struct_type *mailbox_struct_type;
2124 CHECK_NAME(name, EINVAL);
2125 CHECK(!extern_func_find(p, name), EEXIST);
2127 CHECK_NAME(mailbox_struct_type_name, EINVAL);
2128 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2129 CHECK(mailbox_struct_type, EINVAL);
2131 CHECK(func, EINVAL);
2133 /* Node allocation. */
2134 f = calloc(1, sizeof(struct extern_func));
2135 CHECK(func, ENOMEM);
2137 /* Node initialization. */
2138 strcpy(f->name, name);
2139 f->mailbox_struct_type = mailbox_struct_type;
2141 f->struct_id = p->n_structs;
2142 f->id = p->n_extern_funcs;
2144 /* Node add to tailq. */
2145 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2146 p->n_extern_funcs++;
2153 extern_func_build(struct rte_swx_pipeline *p)
2157 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2158 struct thread *t = &p->threads[i];
2159 struct extern_func *func;
2161 /* Memory allocation. */
2162 t->extern_funcs = calloc(p->n_extern_funcs,
2163 sizeof(struct extern_func_runtime));
2164 CHECK(t->extern_funcs, ENOMEM);
2166 /* Extern function. */
2167 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2168 struct extern_func_runtime *r =
2169 &t->extern_funcs[func->id];
2170 uint32_t mailbox_size =
2171 func->mailbox_struct_type->n_bits / 8;
2173 r->func = func->func;
2175 r->mailbox = calloc(1, mailbox_size);
2176 CHECK(r->mailbox, ENOMEM);
2178 t->structs[func->struct_id] = r->mailbox;
2186 extern_func_build_free(struct rte_swx_pipeline *p)
2190 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2191 struct thread *t = &p->threads[i];
2194 if (!t->extern_funcs)
2197 for (j = 0; j < p->n_extern_funcs; j++) {
2198 struct extern_func_runtime *r = &t->extern_funcs[j];
2203 free(t->extern_funcs);
2204 t->extern_funcs = NULL;
2209 extern_func_free(struct rte_swx_pipeline *p)
2211 extern_func_build_free(p);
2214 struct extern_func *elem;
2216 elem = TAILQ_FIRST(&p->extern_funcs);
2220 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2228 static struct header *
2229 header_find(struct rte_swx_pipeline *p, const char *name)
2231 struct header *elem;
2233 TAILQ_FOREACH(elem, &p->headers, node)
2234 if (strcmp(elem->name, name) == 0)
2240 static struct header *
2241 header_parse(struct rte_swx_pipeline *p,
2244 if (name[0] != 'h' || name[1] != '.')
2247 return header_find(p, &name[2]);
2250 static struct field *
2251 header_field_parse(struct rte_swx_pipeline *p,
2253 struct header **header)
2257 char *header_name, *field_name;
2259 if ((name[0] != 'h') || (name[1] != '.'))
2262 header_name = strdup(&name[2]);
2266 field_name = strchr(header_name, '.');
2275 h = header_find(p, header_name);
2281 f = struct_type_field_find(h->st, field_name);
2295 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2297 const char *struct_type_name)
2299 struct struct_type *st;
2301 size_t n_headers_max;
2304 CHECK_NAME(name, EINVAL);
2305 CHECK_NAME(struct_type_name, EINVAL);
2307 CHECK(!header_find(p, name), EEXIST);
2309 st = struct_type_find(p, struct_type_name);
2312 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2313 CHECK(p->n_headers < n_headers_max, ENOSPC);
2315 /* Node allocation. */
2316 h = calloc(1, sizeof(struct header));
2319 /* Node initialization. */
2320 strcpy(h->name, name);
2322 h->struct_id = p->n_structs;
2323 h->id = p->n_headers;
2325 /* Node add to tailq. */
2326 TAILQ_INSERT_TAIL(&p->headers, h, node);
2334 header_build(struct rte_swx_pipeline *p)
2337 uint32_t n_bytes = 0, i;
2339 TAILQ_FOREACH(h, &p->headers, node) {
2340 n_bytes += h->st->n_bits / 8;
2343 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2344 struct thread *t = &p->threads[i];
2345 uint32_t offset = 0;
2347 t->headers = calloc(p->n_headers,
2348 sizeof(struct header_runtime));
2349 CHECK(t->headers, ENOMEM);
2351 t->headers_out = calloc(p->n_headers,
2352 sizeof(struct header_out_runtime));
2353 CHECK(t->headers_out, ENOMEM);
2355 t->header_storage = calloc(1, n_bytes);
2356 CHECK(t->header_storage, ENOMEM);
2358 t->header_out_storage = calloc(1, n_bytes);
2359 CHECK(t->header_out_storage, ENOMEM);
2361 TAILQ_FOREACH(h, &p->headers, node) {
2362 uint8_t *header_storage;
2364 header_storage = &t->header_storage[offset];
2365 offset += h->st->n_bits / 8;
2367 t->headers[h->id].ptr0 = header_storage;
2368 t->structs[h->struct_id] = header_storage;
2376 header_build_free(struct rte_swx_pipeline *p)
2380 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2381 struct thread *t = &p->threads[i];
2383 free(t->headers_out);
2384 t->headers_out = NULL;
2389 free(t->header_out_storage);
2390 t->header_out_storage = NULL;
2392 free(t->header_storage);
2393 t->header_storage = NULL;
2398 header_free(struct rte_swx_pipeline *p)
2400 header_build_free(p);
2403 struct header *elem;
2405 elem = TAILQ_FIRST(&p->headers);
2409 TAILQ_REMOVE(&p->headers, elem, node);
2417 static struct field *
2418 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2420 if (!p->metadata_st)
2423 if (name[0] != 'm' || name[1] != '.')
2426 return struct_type_field_find(p->metadata_st, &name[2]);
2430 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2431 const char *struct_type_name)
2433 struct struct_type *st = NULL;
2437 CHECK_NAME(struct_type_name, EINVAL);
2438 st = struct_type_find(p, struct_type_name);
2440 CHECK(!p->metadata_st, EINVAL);
2442 p->metadata_st = st;
2443 p->metadata_struct_id = p->n_structs;
2451 metadata_build(struct rte_swx_pipeline *p)
2453 uint32_t n_bytes = p->metadata_st->n_bits / 8;
2456 /* Thread-level initialization. */
2457 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2458 struct thread *t = &p->threads[i];
2461 metadata = calloc(1, n_bytes);
2462 CHECK(metadata, ENOMEM);
2464 t->metadata = metadata;
2465 t->structs[p->metadata_struct_id] = metadata;
2472 metadata_build_free(struct rte_swx_pipeline *p)
2476 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2477 struct thread *t = &p->threads[i];
2485 metadata_free(struct rte_swx_pipeline *p)
2487 metadata_build_free(p);
2494 instruction_is_jmp(struct instruction *instr)
2496 switch (instr->type) {
2498 case INSTR_JMP_VALID:
2499 case INSTR_JMP_INVALID:
2501 case INSTR_JMP_MISS:
2502 case INSTR_JMP_ACTION_HIT:
2503 case INSTR_JMP_ACTION_MISS:
2505 case INSTR_JMP_EQ_S:
2506 case INSTR_JMP_EQ_I:
2508 case INSTR_JMP_NEQ_S:
2509 case INSTR_JMP_NEQ_I:
2511 case INSTR_JMP_LT_MH:
2512 case INSTR_JMP_LT_HM:
2513 case INSTR_JMP_LT_HH:
2514 case INSTR_JMP_LT_MI:
2515 case INSTR_JMP_LT_HI:
2517 case INSTR_JMP_GT_MH:
2518 case INSTR_JMP_GT_HM:
2519 case INSTR_JMP_GT_HH:
2520 case INSTR_JMP_GT_MI:
2521 case INSTR_JMP_GT_HI:
2529 static struct field *
2530 action_field_parse(struct action *action, const char *name);
2532 static struct field *
2533 struct_field_parse(struct rte_swx_pipeline *p,
2534 struct action *action,
2536 uint32_t *struct_id)
2543 struct header *header;
2545 f = header_field_parse(p, name, &header);
2549 *struct_id = header->struct_id;
2555 f = metadata_field_parse(p, name);
2559 *struct_id = p->metadata_struct_id;
2568 f = action_field_parse(action, name);
2578 struct extern_obj *obj;
2580 f = extern_obj_mailbox_field_parse(p, name, &obj);
2584 *struct_id = obj->struct_id;
2590 struct extern_func *func;
2592 f = extern_func_mailbox_field_parse(p, name, &func);
2596 *struct_id = func->struct_id;
2606 pipeline_port_inc(struct rte_swx_pipeline *p)
2608 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2612 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2614 t->ip = p->instructions;
2618 thread_ip_set(struct thread *t, struct instruction *ip)
2624 thread_ip_action_call(struct rte_swx_pipeline *p,
2629 t->ip = p->action_instructions[action_id];
2633 thread_ip_inc(struct rte_swx_pipeline *p);
2636 thread_ip_inc(struct rte_swx_pipeline *p)
2638 struct thread *t = &p->threads[p->thread_id];
2644 thread_ip_inc_cond(struct thread *t, int cond)
2650 thread_yield(struct rte_swx_pipeline *p)
2652 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2656 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2658 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2665 instr_rx_translate(struct rte_swx_pipeline *p,
2666 struct action *action,
2669 struct instruction *instr,
2670 struct instruction_data *data __rte_unused)
2674 CHECK(!action, EINVAL);
2675 CHECK(n_tokens == 2, EINVAL);
2677 f = metadata_field_parse(p, tokens[1]);
2680 instr->type = INSTR_RX;
2681 instr->io.io.offset = f->offset / 8;
2682 instr->io.io.n_bits = f->n_bits;
2687 instr_rx_exec(struct rte_swx_pipeline *p);
2690 instr_rx_exec(struct rte_swx_pipeline *p)
2692 struct thread *t = &p->threads[p->thread_id];
2693 struct instruction *ip = t->ip;
2694 struct port_in_runtime *port = &p->in[p->port_id];
2695 struct rte_swx_pkt *pkt = &t->pkt;
2699 pkt_received = port->pkt_rx(port->obj, pkt);
2700 t->ptr = &pkt->pkt[pkt->offset];
2701 rte_prefetch0(t->ptr);
2703 TRACE("[Thread %2u] rx %s from port %u\n",
2705 pkt_received ? "1 pkt" : "0 pkts",
2709 t->valid_headers = 0;
2710 t->n_headers_out = 0;
2713 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2716 t->table_state = p->table_state;
2719 pipeline_port_inc(p);
2720 thread_ip_inc_cond(t, pkt_received);
2728 instr_tx_translate(struct rte_swx_pipeline *p,
2729 struct action *action __rte_unused,
2732 struct instruction *instr,
2733 struct instruction_data *data __rte_unused)
2737 CHECK(n_tokens == 2, EINVAL);
2739 f = metadata_field_parse(p, tokens[1]);
2742 instr->type = INSTR_TX;
2743 instr->io.io.offset = f->offset / 8;
2744 instr->io.io.n_bits = f->n_bits;
2749 emit_handler(struct thread *t)
2751 struct header_out_runtime *h0 = &t->headers_out[0];
2752 struct header_out_runtime *h1 = &t->headers_out[1];
2753 uint32_t offset = 0, i;
2755 /* No header change or header decapsulation. */
2756 if ((t->n_headers_out == 1) &&
2757 (h0->ptr + h0->n_bytes == t->ptr)) {
2758 TRACE("Emit handler: no header change or header decap.\n");
2760 t->pkt.offset -= h0->n_bytes;
2761 t->pkt.length += h0->n_bytes;
2766 /* Header encapsulation (optionally, with prior header decasulation). */
2767 if ((t->n_headers_out == 2) &&
2768 (h1->ptr + h1->n_bytes == t->ptr) &&
2769 (h0->ptr == h0->ptr0)) {
2772 TRACE("Emit handler: header encapsulation.\n");
2774 offset = h0->n_bytes + h1->n_bytes;
2775 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2776 t->pkt.offset -= offset;
2777 t->pkt.length += offset;
2782 /* Header insertion. */
2785 /* Header extraction. */
2788 /* For any other case. */
2789 TRACE("Emit handler: complex case.\n");
2791 for (i = 0; i < t->n_headers_out; i++) {
2792 struct header_out_runtime *h = &t->headers_out[i];
2794 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2795 offset += h->n_bytes;
2799 memcpy(t->ptr - offset, t->header_out_storage, offset);
2800 t->pkt.offset -= offset;
2801 t->pkt.length += offset;
2806 instr_tx_exec(struct rte_swx_pipeline *p);
2809 instr_tx_exec(struct rte_swx_pipeline *p)
2811 struct thread *t = &p->threads[p->thread_id];
2812 struct instruction *ip = t->ip;
2813 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2814 struct port_out_runtime *port = &p->out[port_id];
2815 struct rte_swx_pkt *pkt = &t->pkt;
2817 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2825 port->pkt_tx(port->obj, pkt);
2828 thread_ip_reset(p, t);
2836 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2837 struct action *action,
2840 struct instruction *instr,
2841 struct instruction_data *data __rte_unused)
2845 CHECK(!action, EINVAL);
2846 CHECK(n_tokens == 2, EINVAL);
2848 h = header_parse(p, tokens[1]);
2851 instr->type = INSTR_HDR_EXTRACT;
2852 instr->io.hdr.header_id[0] = h->id;
2853 instr->io.hdr.struct_id[0] = h->struct_id;
2854 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2859 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2862 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2864 struct thread *t = &p->threads[p->thread_id];
2865 struct instruction *ip = t->ip;
2866 uint64_t valid_headers = t->valid_headers;
2867 uint8_t *ptr = t->ptr;
2868 uint32_t offset = t->pkt.offset;
2869 uint32_t length = t->pkt.length;
2872 for (i = 0; i < n_extract; i++) {
2873 uint32_t header_id = ip->io.hdr.header_id[i];
2874 uint32_t struct_id = ip->io.hdr.struct_id[i];
2875 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2877 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2883 t->structs[struct_id] = ptr;
2884 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2893 t->valid_headers = valid_headers;
2896 t->pkt.offset = offset;
2897 t->pkt.length = length;
2902 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2904 __instr_hdr_extract_exec(p, 1);
2911 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2913 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2916 __instr_hdr_extract_exec(p, 2);
2923 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
2925 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
2928 __instr_hdr_extract_exec(p, 3);
2935 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
2937 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
2940 __instr_hdr_extract_exec(p, 4);
2947 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
2949 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
2952 __instr_hdr_extract_exec(p, 5);
2959 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
2961 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
2964 __instr_hdr_extract_exec(p, 6);
2971 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
2973 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
2976 __instr_hdr_extract_exec(p, 7);
2983 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
2985 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
2988 __instr_hdr_extract_exec(p, 8);
2998 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
2999 struct action *action __rte_unused,
3002 struct instruction *instr,
3003 struct instruction_data *data __rte_unused)
3007 CHECK(n_tokens == 2, EINVAL);
3009 h = header_parse(p, tokens[1]);
3012 instr->type = INSTR_HDR_EMIT;
3013 instr->io.hdr.header_id[0] = h->id;
3014 instr->io.hdr.struct_id[0] = h->struct_id;
3015 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3020 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3023 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3025 struct thread *t = &p->threads[p->thread_id];
3026 struct instruction *ip = t->ip;
3027 uint32_t n_headers_out = t->n_headers_out;
3028 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3029 uint8_t *ho_ptr = NULL;
3030 uint32_t ho_nbytes = 0, i;
3032 for (i = 0; i < n_emit; i++) {
3033 uint32_t header_id = ip->io.hdr.header_id[i];
3034 uint32_t struct_id = ip->io.hdr.struct_id[i];
3035 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3037 struct header_runtime *hi = &t->headers[header_id];
3038 uint8_t *hi_ptr = t->structs[struct_id];
3040 TRACE("[Thread %2u]: emit header %u\n",
3046 if (!t->n_headers_out) {
3047 ho = &t->headers_out[0];
3049 ho->ptr0 = hi->ptr0;
3053 ho_nbytes = n_bytes;
3060 ho_nbytes = ho->n_bytes;
3064 if (ho_ptr + ho_nbytes == hi_ptr) {
3065 ho_nbytes += n_bytes;
3067 ho->n_bytes = ho_nbytes;
3070 ho->ptr0 = hi->ptr0;
3074 ho_nbytes = n_bytes;
3080 ho->n_bytes = ho_nbytes;
3081 t->n_headers_out = n_headers_out;
3085 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3087 __instr_hdr_emit_exec(p, 1);
3094 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3096 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3099 __instr_hdr_emit_exec(p, 1);
3104 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3106 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3109 __instr_hdr_emit_exec(p, 2);
3114 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3116 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3119 __instr_hdr_emit_exec(p, 3);
3124 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3126 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3129 __instr_hdr_emit_exec(p, 4);
3134 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3136 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3139 __instr_hdr_emit_exec(p, 5);
3144 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3146 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3149 __instr_hdr_emit_exec(p, 6);
3154 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3156 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3159 __instr_hdr_emit_exec(p, 7);
3164 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3166 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3169 __instr_hdr_emit_exec(p, 8);
3177 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3178 struct action *action __rte_unused,
3181 struct instruction *instr,
3182 struct instruction_data *data __rte_unused)
3186 CHECK(n_tokens == 2, EINVAL);
3188 h = header_parse(p, tokens[1]);
3191 instr->type = INSTR_HDR_VALIDATE;
3192 instr->valid.header_id = h->id;
3197 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3199 struct thread *t = &p->threads[p->thread_id];
3200 struct instruction *ip = t->ip;
3201 uint32_t header_id = ip->valid.header_id;
3203 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3206 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3216 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3217 struct action *action __rte_unused,
3220 struct instruction *instr,
3221 struct instruction_data *data __rte_unused)
3225 CHECK(n_tokens == 2, EINVAL);
3227 h = header_parse(p, tokens[1]);
3230 instr->type = INSTR_HDR_INVALIDATE;
3231 instr->valid.header_id = h->id;
3236 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3238 struct thread *t = &p->threads[p->thread_id];
3239 struct instruction *ip = t->ip;
3240 uint32_t header_id = ip->valid.header_id;
3242 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3245 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3254 static struct table *
3255 table_find(struct rte_swx_pipeline *p, const char *name);
3258 instr_table_translate(struct rte_swx_pipeline *p,
3259 struct action *action,
3262 struct instruction *instr,
3263 struct instruction_data *data __rte_unused)
3267 CHECK(!action, EINVAL);
3268 CHECK(n_tokens == 2, EINVAL);
3270 t = table_find(p, tokens[1]);
3273 instr->type = INSTR_TABLE;
3274 instr->table.table_id = t->id;
3279 instr_table_exec(struct rte_swx_pipeline *p)
3281 struct thread *t = &p->threads[p->thread_id];
3282 struct instruction *ip = t->ip;
3283 uint32_t table_id = ip->table.table_id;
3284 struct rte_swx_table_state *ts = &t->table_state[table_id];
3285 struct table_runtime *table = &t->tables[table_id];
3287 uint8_t *action_data;
3291 done = table->func(ts->obj,
3299 TRACE("[Thread %2u] table %u (not finalized)\n",
3307 action_id = hit ? action_id : ts->default_action_id;
3308 action_data = hit ? action_data : ts->default_action_data;
3310 TRACE("[Thread %2u] table %u (%s, action %u)\n",
3313 hit ? "hit" : "miss",
3314 (uint32_t)action_id);
3316 t->action_id = action_id;
3317 t->structs[0] = action_data;
3321 thread_ip_action_call(p, t, action_id);
3328 instr_extern_translate(struct rte_swx_pipeline *p,
3329 struct action *action __rte_unused,
3332 struct instruction *instr,
3333 struct instruction_data *data __rte_unused)
3335 char *token = tokens[1];
3337 CHECK(n_tokens == 2, EINVAL);
3339 if (token[0] == 'e') {
3340 struct extern_obj *obj;
3341 struct extern_type_member_func *func;
3343 func = extern_obj_member_func_parse(p, token, &obj);
3344 CHECK(func, EINVAL);
3346 instr->type = INSTR_EXTERN_OBJ;
3347 instr->ext_obj.ext_obj_id = obj->id;
3348 instr->ext_obj.func_id = func->id;
3353 if (token[0] == 'f') {
3354 struct extern_func *func;
3356 func = extern_func_parse(p, token);
3357 CHECK(func, EINVAL);
3359 instr->type = INSTR_EXTERN_FUNC;
3360 instr->ext_func.ext_func_id = func->id;
3369 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3371 struct thread *t = &p->threads[p->thread_id];
3372 struct instruction *ip = t->ip;
3373 uint32_t obj_id = ip->ext_obj.ext_obj_id;
3374 uint32_t func_id = ip->ext_obj.func_id;
3375 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3376 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3378 TRACE("[Thread %2u] extern obj %u member func %u\n",
3383 /* Extern object member function execute. */
3384 uint32_t done = func(obj->obj, obj->mailbox);
3387 thread_ip_inc_cond(t, done);
3388 thread_yield_cond(p, done ^ 1);
3392 instr_extern_func_exec(struct rte_swx_pipeline *p)
3394 struct thread *t = &p->threads[p->thread_id];
3395 struct instruction *ip = t->ip;
3396 uint32_t ext_func_id = ip->ext_func.ext_func_id;
3397 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3398 rte_swx_extern_func_t func = ext_func->func;
3400 TRACE("[Thread %2u] extern func %u\n",
3404 /* Extern function execute. */
3405 uint32_t done = func(ext_func->mailbox);
3408 thread_ip_inc_cond(t, done);
3409 thread_yield_cond(p, done ^ 1);
3416 instr_mov_translate(struct rte_swx_pipeline *p,
3417 struct action *action,
3420 struct instruction *instr,
3421 struct instruction_data *data __rte_unused)
3423 char *dst = tokens[1], *src = tokens[2];
3424 struct field *fdst, *fsrc;
3426 uint32_t dst_struct_id, src_struct_id;
3428 CHECK(n_tokens == 3, EINVAL);
3430 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3431 CHECK(fdst, EINVAL);
3434 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3436 instr->type = INSTR_MOV;
3437 if ((dst[0] == 'h' && src[0] != 'h') ||
3438 (dst[0] != 'h' && src[0] == 'h'))
3439 instr->type = INSTR_MOV_S;
3441 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3442 instr->mov.dst.n_bits = fdst->n_bits;
3443 instr->mov.dst.offset = fdst->offset / 8;
3444 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3445 instr->mov.src.n_bits = fsrc->n_bits;
3446 instr->mov.src.offset = fsrc->offset / 8;
3451 src_val = strtoull(src, &src, 0);
3452 CHECK(!src[0], EINVAL);
3455 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3457 instr->type = INSTR_MOV_I;
3458 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3459 instr->mov.dst.n_bits = fdst->n_bits;
3460 instr->mov.dst.offset = fdst->offset / 8;
3461 instr->mov.src_val = src_val;
3466 instr_mov_exec(struct rte_swx_pipeline *p)
3468 struct thread *t = &p->threads[p->thread_id];
3469 struct instruction *ip = t->ip;
3471 TRACE("[Thread %2u] mov\n",
3481 instr_mov_s_exec(struct rte_swx_pipeline *p)
3483 struct thread *t = &p->threads[p->thread_id];
3484 struct instruction *ip = t->ip;
3486 TRACE("[Thread %2u] mov (s)\n",
3496 instr_mov_i_exec(struct rte_swx_pipeline *p)
3498 struct thread *t = &p->threads[p->thread_id];
3499 struct instruction *ip = t->ip;
3501 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3515 instr_dma_translate(struct rte_swx_pipeline *p,
3516 struct action *action,
3519 struct instruction *instr,
3520 struct instruction_data *data __rte_unused)
3522 char *dst = tokens[1];
3523 char *src = tokens[2];
3527 CHECK(action, EINVAL);
3528 CHECK(n_tokens == 3, EINVAL);
3530 h = header_parse(p, dst);
3533 tf = action_field_parse(action, src);
3536 instr->type = INSTR_DMA_HT;
3537 instr->dma.dst.header_id[0] = h->id;
3538 instr->dma.dst.struct_id[0] = h->struct_id;
3539 instr->dma.n_bytes[0] = h->st->n_bits / 8;
3540 instr->dma.src.offset[0] = tf->offset / 8;
3546 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3549 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3551 struct thread *t = &p->threads[p->thread_id];
3552 struct instruction *ip = t->ip;
3553 uint8_t *action_data = t->structs[0];
3554 uint64_t valid_headers = t->valid_headers;
3557 for (i = 0; i < n_dma; i++) {
3558 uint32_t header_id = ip->dma.dst.header_id[i];
3559 uint32_t struct_id = ip->dma.dst.struct_id[i];
3560 uint32_t offset = ip->dma.src.offset[i];
3561 uint32_t n_bytes = ip->dma.n_bytes[i];
3563 struct header_runtime *h = &t->headers[header_id];
3564 uint8_t *h_ptr0 = h->ptr0;
3565 uint8_t *h_ptr = t->structs[struct_id];
3567 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3569 void *src = &action_data[offset];
3571 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3574 memcpy(dst, src, n_bytes);
3575 t->structs[struct_id] = dst;
3576 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3579 t->valid_headers = valid_headers;
3583 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3585 __instr_dma_ht_exec(p, 1);
3592 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3594 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3597 __instr_dma_ht_exec(p, 2);
3604 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3606 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3609 __instr_dma_ht_exec(p, 3);
3616 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3618 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3621 __instr_dma_ht_exec(p, 4);
3628 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3630 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3633 __instr_dma_ht_exec(p, 5);
3640 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3642 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3645 __instr_dma_ht_exec(p, 6);
3652 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3654 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3657 __instr_dma_ht_exec(p, 7);
3664 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3666 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3669 __instr_dma_ht_exec(p, 8);
3679 instr_alu_add_translate(struct rte_swx_pipeline *p,
3680 struct action *action,
3683 struct instruction *instr,
3684 struct instruction_data *data __rte_unused)
3686 char *dst = tokens[1], *src = tokens[2];
3687 struct field *fdst, *fsrc;
3689 uint32_t dst_struct_id, src_struct_id;
3691 CHECK(n_tokens == 3, EINVAL);
3693 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3694 CHECK(fdst, EINVAL);
3696 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3697 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3699 instr->type = INSTR_ALU_ADD;
3700 if (dst[0] == 'h' && src[0] != 'h')
3701 instr->type = INSTR_ALU_ADD_HM;
3702 if (dst[0] != 'h' && src[0] == 'h')
3703 instr->type = INSTR_ALU_ADD_MH;
3704 if (dst[0] == 'h' && src[0] == 'h')
3705 instr->type = INSTR_ALU_ADD_HH;
3707 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3708 instr->alu.dst.n_bits = fdst->n_bits;
3709 instr->alu.dst.offset = fdst->offset / 8;
3710 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3711 instr->alu.src.n_bits = fsrc->n_bits;
3712 instr->alu.src.offset = fsrc->offset / 8;
3716 /* ADD_MI, ADD_HI. */
3717 src_val = strtoull(src, &src, 0);
3718 CHECK(!src[0], EINVAL);
3720 instr->type = INSTR_ALU_ADD_MI;
3722 instr->type = INSTR_ALU_ADD_HI;
3724 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3725 instr->alu.dst.n_bits = fdst->n_bits;
3726 instr->alu.dst.offset = fdst->offset / 8;
3727 instr->alu.src_val = src_val;
3732 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3733 struct action *action,
3736 struct instruction *instr,
3737 struct instruction_data *data __rte_unused)
3739 char *dst = tokens[1], *src = tokens[2];
3740 struct field *fdst, *fsrc;
3742 uint32_t dst_struct_id, src_struct_id;
3744 CHECK(n_tokens == 3, EINVAL);
3746 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3747 CHECK(fdst, EINVAL);
3749 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3750 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3752 instr->type = INSTR_ALU_SUB;
3753 if (dst[0] == 'h' && src[0] != 'h')
3754 instr->type = INSTR_ALU_SUB_HM;
3755 if (dst[0] != 'h' && src[0] == 'h')
3756 instr->type = INSTR_ALU_SUB_MH;
3757 if (dst[0] == 'h' && src[0] == 'h')
3758 instr->type = INSTR_ALU_SUB_HH;
3760 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3761 instr->alu.dst.n_bits = fdst->n_bits;
3762 instr->alu.dst.offset = fdst->offset / 8;
3763 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3764 instr->alu.src.n_bits = fsrc->n_bits;
3765 instr->alu.src.offset = fsrc->offset / 8;
3769 /* SUB_MI, SUB_HI. */
3770 src_val = strtoull(src, &src, 0);
3771 CHECK(!src[0], EINVAL);
3773 instr->type = INSTR_ALU_SUB_MI;
3775 instr->type = INSTR_ALU_SUB_HI;
3777 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3778 instr->alu.dst.n_bits = fdst->n_bits;
3779 instr->alu.dst.offset = fdst->offset / 8;
3780 instr->alu.src_val = src_val;
3785 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3786 struct action *action __rte_unused,
3789 struct instruction *instr,
3790 struct instruction_data *data __rte_unused)
3792 char *dst = tokens[1], *src = tokens[2];
3793 struct header *hdst, *hsrc;
3794 struct field *fdst, *fsrc;
3796 CHECK(n_tokens == 3, EINVAL);
3798 fdst = header_field_parse(p, dst, &hdst);
3799 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3802 fsrc = header_field_parse(p, src, &hsrc);
3804 instr->type = INSTR_ALU_CKADD_FIELD;
3805 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3806 instr->alu.dst.n_bits = fdst->n_bits;
3807 instr->alu.dst.offset = fdst->offset / 8;
3808 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3809 instr->alu.src.n_bits = fsrc->n_bits;
3810 instr->alu.src.offset = fsrc->offset / 8;
3814 /* CKADD_STRUCT, CKADD_STRUCT20. */
3815 hsrc = header_parse(p, src);
3816 CHECK(hsrc, EINVAL);
3818 instr->type = INSTR_ALU_CKADD_STRUCT;
3819 if ((hsrc->st->n_bits / 8) == 20)
3820 instr->type = INSTR_ALU_CKADD_STRUCT20;
3822 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3823 instr->alu.dst.n_bits = fdst->n_bits;
3824 instr->alu.dst.offset = fdst->offset / 8;
3825 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3826 instr->alu.src.n_bits = hsrc->st->n_bits;
3827 instr->alu.src.offset = 0; /* Unused. */
3832 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3833 struct action *action __rte_unused,
3836 struct instruction *instr,
3837 struct instruction_data *data __rte_unused)
3839 char *dst = tokens[1], *src = tokens[2];
3840 struct header *hdst, *hsrc;
3841 struct field *fdst, *fsrc;
3843 CHECK(n_tokens == 3, EINVAL);
3845 fdst = header_field_parse(p, dst, &hdst);
3846 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3848 fsrc = header_field_parse(p, src, &hsrc);
3849 CHECK(fsrc, EINVAL);
3851 instr->type = INSTR_ALU_CKSUB_FIELD;
3852 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3853 instr->alu.dst.n_bits = fdst->n_bits;
3854 instr->alu.dst.offset = fdst->offset / 8;
3855 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3856 instr->alu.src.n_bits = fsrc->n_bits;
3857 instr->alu.src.offset = fsrc->offset / 8;
3862 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3863 struct action *action,
3866 struct instruction *instr,
3867 struct instruction_data *data __rte_unused)
3869 char *dst = tokens[1], *src = tokens[2];
3870 struct field *fdst, *fsrc;
3872 uint32_t dst_struct_id, src_struct_id;
3874 CHECK(n_tokens == 3, EINVAL);
3876 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3877 CHECK(fdst, EINVAL);
3879 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3880 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3882 instr->type = INSTR_ALU_SHL;
3883 if (dst[0] == 'h' && src[0] != 'h')
3884 instr->type = INSTR_ALU_SHL_HM;
3885 if (dst[0] != 'h' && src[0] == 'h')
3886 instr->type = INSTR_ALU_SHL_MH;
3887 if (dst[0] == 'h' && src[0] == 'h')
3888 instr->type = INSTR_ALU_SHL_HH;
3890 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3891 instr->alu.dst.n_bits = fdst->n_bits;
3892 instr->alu.dst.offset = fdst->offset / 8;
3893 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3894 instr->alu.src.n_bits = fsrc->n_bits;
3895 instr->alu.src.offset = fsrc->offset / 8;
3899 /* SHL_MI, SHL_HI. */
3900 src_val = strtoull(src, &src, 0);
3901 CHECK(!src[0], EINVAL);
3903 instr->type = INSTR_ALU_SHL_MI;
3905 instr->type = INSTR_ALU_SHL_HI;
3907 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3908 instr->alu.dst.n_bits = fdst->n_bits;
3909 instr->alu.dst.offset = fdst->offset / 8;
3910 instr->alu.src_val = src_val;
3915 instr_alu_shr_translate(struct rte_swx_pipeline *p,
3916 struct action *action,
3919 struct instruction *instr,
3920 struct instruction_data *data __rte_unused)
3922 char *dst = tokens[1], *src = tokens[2];
3923 struct field *fdst, *fsrc;
3925 uint32_t dst_struct_id, src_struct_id;
3927 CHECK(n_tokens == 3, EINVAL);
3929 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3930 CHECK(fdst, EINVAL);
3932 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
3933 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3935 instr->type = INSTR_ALU_SHR;
3936 if (dst[0] == 'h' && src[0] != 'h')
3937 instr->type = INSTR_ALU_SHR_HM;
3938 if (dst[0] != 'h' && src[0] == 'h')
3939 instr->type = INSTR_ALU_SHR_MH;
3940 if (dst[0] == 'h' && src[0] == 'h')
3941 instr->type = INSTR_ALU_SHR_HH;
3943 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3944 instr->alu.dst.n_bits = fdst->n_bits;
3945 instr->alu.dst.offset = fdst->offset / 8;
3946 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3947 instr->alu.src.n_bits = fsrc->n_bits;
3948 instr->alu.src.offset = fsrc->offset / 8;
3952 /* SHR_MI, SHR_HI. */
3953 src_val = strtoull(src, &src, 0);
3954 CHECK(!src[0], EINVAL);
3956 instr->type = INSTR_ALU_SHR_MI;
3958 instr->type = INSTR_ALU_SHR_HI;
3960 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3961 instr->alu.dst.n_bits = fdst->n_bits;
3962 instr->alu.dst.offset = fdst->offset / 8;
3963 instr->alu.src_val = src_val;
3968 instr_alu_and_translate(struct rte_swx_pipeline *p,
3969 struct action *action,
3972 struct instruction *instr,
3973 struct instruction_data *data __rte_unused)
3975 char *dst = tokens[1], *src = tokens[2];
3976 struct field *fdst, *fsrc;
3978 uint32_t dst_struct_id, src_struct_id;
3980 CHECK(n_tokens == 3, EINVAL);
3982 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3983 CHECK(fdst, EINVAL);
3986 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3988 instr->type = INSTR_ALU_AND;
3989 if ((dst[0] == 'h' && src[0] != 'h') ||
3990 (dst[0] != 'h' && src[0] == 'h'))
3991 instr->type = INSTR_ALU_AND_S;
3993 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3994 instr->alu.dst.n_bits = fdst->n_bits;
3995 instr->alu.dst.offset = fdst->offset / 8;
3996 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3997 instr->alu.src.n_bits = fsrc->n_bits;
3998 instr->alu.src.offset = fsrc->offset / 8;
4003 src_val = strtoull(src, &src, 0);
4004 CHECK(!src[0], EINVAL);
4007 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4009 instr->type = INSTR_ALU_AND_I;
4010 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4011 instr->alu.dst.n_bits = fdst->n_bits;
4012 instr->alu.dst.offset = fdst->offset / 8;
4013 instr->alu.src_val = src_val;
4018 instr_alu_or_translate(struct rte_swx_pipeline *p,
4019 struct action *action,
4022 struct instruction *instr,
4023 struct instruction_data *data __rte_unused)
4025 char *dst = tokens[1], *src = tokens[2];
4026 struct field *fdst, *fsrc;
4028 uint32_t dst_struct_id, src_struct_id;
4030 CHECK(n_tokens == 3, EINVAL);
4032 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4033 CHECK(fdst, EINVAL);
4036 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4038 instr->type = INSTR_ALU_OR;
4039 if ((dst[0] == 'h' && src[0] != 'h') ||
4040 (dst[0] != 'h' && src[0] == 'h'))
4041 instr->type = INSTR_ALU_OR_S;
4043 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4044 instr->alu.dst.n_bits = fdst->n_bits;
4045 instr->alu.dst.offset = fdst->offset / 8;
4046 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4047 instr->alu.src.n_bits = fsrc->n_bits;
4048 instr->alu.src.offset = fsrc->offset / 8;
4053 src_val = strtoull(src, &src, 0);
4054 CHECK(!src[0], EINVAL);
4057 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4059 instr->type = INSTR_ALU_OR_I;
4060 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4061 instr->alu.dst.n_bits = fdst->n_bits;
4062 instr->alu.dst.offset = fdst->offset / 8;
4063 instr->alu.src_val = src_val;
4068 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4069 struct action *action,
4072 struct instruction *instr,
4073 struct instruction_data *data __rte_unused)
4075 char *dst = tokens[1], *src = tokens[2];
4076 struct field *fdst, *fsrc;
4078 uint32_t dst_struct_id, src_struct_id;
4080 CHECK(n_tokens == 3, EINVAL);
4082 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4083 CHECK(fdst, EINVAL);
4086 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4088 instr->type = INSTR_ALU_XOR;
4089 if ((dst[0] == 'h' && src[0] != 'h') ||
4090 (dst[0] != 'h' && src[0] == 'h'))
4091 instr->type = INSTR_ALU_XOR_S;
4093 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4094 instr->alu.dst.n_bits = fdst->n_bits;
4095 instr->alu.dst.offset = fdst->offset / 8;
4096 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4097 instr->alu.src.n_bits = fsrc->n_bits;
4098 instr->alu.src.offset = fsrc->offset / 8;
4103 src_val = strtoull(src, &src, 0);
4104 CHECK(!src[0], EINVAL);
4107 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4109 instr->type = INSTR_ALU_XOR_I;
4110 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4111 instr->alu.dst.n_bits = fdst->n_bits;
4112 instr->alu.dst.offset = fdst->offset / 8;
4113 instr->alu.src_val = src_val;
4118 instr_alu_add_exec(struct rte_swx_pipeline *p)
4120 struct thread *t = &p->threads[p->thread_id];
4121 struct instruction *ip = t->ip;
4123 TRACE("[Thread %2u] add\n", p->thread_id);
4133 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4135 struct thread *t = &p->threads[p->thread_id];
4136 struct instruction *ip = t->ip;
4138 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4148 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4150 struct thread *t = &p->threads[p->thread_id];
4151 struct instruction *ip = t->ip;
4153 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4163 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4165 struct thread *t = &p->threads[p->thread_id];
4166 struct instruction *ip = t->ip;
4168 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4178 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4180 struct thread *t = &p->threads[p->thread_id];
4181 struct instruction *ip = t->ip;
4183 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4193 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4195 struct thread *t = &p->threads[p->thread_id];
4196 struct instruction *ip = t->ip;
4198 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4208 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4210 struct thread *t = &p->threads[p->thread_id];
4211 struct instruction *ip = t->ip;
4213 TRACE("[Thread %2u] sub\n", p->thread_id);
4223 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4225 struct thread *t = &p->threads[p->thread_id];
4226 struct instruction *ip = t->ip;
4228 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4238 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4240 struct thread *t = &p->threads[p->thread_id];
4241 struct instruction *ip = t->ip;
4243 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4253 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4255 struct thread *t = &p->threads[p->thread_id];
4256 struct instruction *ip = t->ip;
4258 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4268 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4270 struct thread *t = &p->threads[p->thread_id];
4271 struct instruction *ip = t->ip;
4273 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4283 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4285 struct thread *t = &p->threads[p->thread_id];
4286 struct instruction *ip = t->ip;
4288 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4298 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4300 struct thread *t = &p->threads[p->thread_id];
4301 struct instruction *ip = t->ip;
4303 TRACE("[Thread %2u] shl\n", p->thread_id);
4313 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4315 struct thread *t = &p->threads[p->thread_id];
4316 struct instruction *ip = t->ip;
4318 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4328 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4330 struct thread *t = &p->threads[p->thread_id];
4331 struct instruction *ip = t->ip;
4333 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4343 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4345 struct thread *t = &p->threads[p->thread_id];
4346 struct instruction *ip = t->ip;
4348 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4358 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4360 struct thread *t = &p->threads[p->thread_id];
4361 struct instruction *ip = t->ip;
4363 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4373 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4375 struct thread *t = &p->threads[p->thread_id];
4376 struct instruction *ip = t->ip;
4378 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4388 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4390 struct thread *t = &p->threads[p->thread_id];
4391 struct instruction *ip = t->ip;
4393 TRACE("[Thread %2u] shr\n", p->thread_id);
4403 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4405 struct thread *t = &p->threads[p->thread_id];
4406 struct instruction *ip = t->ip;
4408 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4418 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4420 struct thread *t = &p->threads[p->thread_id];
4421 struct instruction *ip = t->ip;
4423 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4433 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4435 struct thread *t = &p->threads[p->thread_id];
4436 struct instruction *ip = t->ip;
4438 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4448 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4450 struct thread *t = &p->threads[p->thread_id];
4451 struct instruction *ip = t->ip;
4453 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4463 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4465 struct thread *t = &p->threads[p->thread_id];
4466 struct instruction *ip = t->ip;
4468 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4478 instr_alu_and_exec(struct rte_swx_pipeline *p)
4480 struct thread *t = &p->threads[p->thread_id];
4481 struct instruction *ip = t->ip;
4483 TRACE("[Thread %2u] and\n", p->thread_id);
4493 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4495 struct thread *t = &p->threads[p->thread_id];
4496 struct instruction *ip = t->ip;
4498 TRACE("[Thread %2u] and (s)\n", p->thread_id);
4508 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4510 struct thread *t = &p->threads[p->thread_id];
4511 struct instruction *ip = t->ip;
4513 TRACE("[Thread %2u] and (i)\n", p->thread_id);
4523 instr_alu_or_exec(struct rte_swx_pipeline *p)
4525 struct thread *t = &p->threads[p->thread_id];
4526 struct instruction *ip = t->ip;
4528 TRACE("[Thread %2u] or\n", p->thread_id);
4538 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4540 struct thread *t = &p->threads[p->thread_id];
4541 struct instruction *ip = t->ip;
4543 TRACE("[Thread %2u] or (s)\n", p->thread_id);
4553 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4555 struct thread *t = &p->threads[p->thread_id];
4556 struct instruction *ip = t->ip;
4558 TRACE("[Thread %2u] or (i)\n", p->thread_id);
4568 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4570 struct thread *t = &p->threads[p->thread_id];
4571 struct instruction *ip = t->ip;
4573 TRACE("[Thread %2u] xor\n", p->thread_id);
4583 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4585 struct thread *t = &p->threads[p->thread_id];
4586 struct instruction *ip = t->ip;
4588 TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4598 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4600 struct thread *t = &p->threads[p->thread_id];
4601 struct instruction *ip = t->ip;
4603 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4613 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4615 struct thread *t = &p->threads[p->thread_id];
4616 struct instruction *ip = t->ip;
4617 uint8_t *dst_struct, *src_struct;
4618 uint16_t *dst16_ptr, dst;
4619 uint64_t *src64_ptr, src64, src64_mask, src;
4622 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4625 dst_struct = t->structs[ip->alu.dst.struct_id];
4626 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4629 src_struct = t->structs[ip->alu.src.struct_id];
4630 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4632 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4633 src = src64 & src64_mask;
4638 /* The first input (r) is a 16-bit number. The second and the third
4639 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4640 * three numbers (output r) is a 34-bit number.
4642 r += (src >> 32) + (src & 0xFFFFFFFF);
4644 /* The first input is a 16-bit number. The second input is an 18-bit
4645 * number. In the worst case scenario, the sum of the two numbers is a
4648 r = (r & 0xFFFF) + (r >> 16);
4650 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4651 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4653 r = (r & 0xFFFF) + (r >> 16);
4655 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4656 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4657 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4658 * therefore the output r is always a 16-bit number.
4660 r = (r & 0xFFFF) + (r >> 16);
4665 *dst16_ptr = (uint16_t)r;
4672 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4674 struct thread *t = &p->threads[p->thread_id];
4675 struct instruction *ip = t->ip;
4676 uint8_t *dst_struct, *src_struct;
4677 uint16_t *dst16_ptr, dst;
4678 uint64_t *src64_ptr, src64, src64_mask, src;
4681 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4684 dst_struct = t->structs[ip->alu.dst.struct_id];
4685 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4688 src_struct = t->structs[ip->alu.src.struct_id];
4689 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4691 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4692 src = src64 & src64_mask;
4697 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4698 * the following sequence of operations in 2's complement arithmetic:
4699 * a '- b = (a - b) % 0xFFFF.
4701 * In order to prevent an underflow for the below subtraction, in which
4702 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4703 * minuend), we first add a multiple of the 0xFFFF modulus to the
4704 * minuend. The number we add to the minuend needs to be a 34-bit number
4705 * or higher, so for readability reasons we picked the 36-bit multiple.
4706 * We are effectively turning the 16-bit minuend into a 36-bit number:
4707 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4709 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4711 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4712 * result (the output r) is a 36-bit number.
4714 r -= (src >> 32) + (src & 0xFFFFFFFF);
4716 /* The first input is a 16-bit number. The second input is a 20-bit
4717 * number. Their sum is a 21-bit number.
4719 r = (r & 0xFFFF) + (r >> 16);
4721 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4722 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4724 r = (r & 0xFFFF) + (r >> 16);
4726 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4727 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4728 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4729 * generated, therefore the output r is always a 16-bit number.
4731 r = (r & 0xFFFF) + (r >> 16);
4736 *dst16_ptr = (uint16_t)r;
4743 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4745 struct thread *t = &p->threads[p->thread_id];
4746 struct instruction *ip = t->ip;
4747 uint8_t *dst_struct, *src_struct;
4748 uint16_t *dst16_ptr;
4749 uint32_t *src32_ptr;
4752 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4755 dst_struct = t->structs[ip->alu.dst.struct_id];
4756 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4758 src_struct = t->structs[ip->alu.src.struct_id];
4759 src32_ptr = (uint32_t *)&src_struct[0];
4761 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4762 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4763 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4764 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4765 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4767 /* The first input is a 16-bit number. The second input is a 19-bit
4768 * number. Their sum is a 20-bit number.
4770 r0 = (r0 & 0xFFFF) + (r0 >> 16);
4772 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4773 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4775 r0 = (r0 & 0xFFFF) + (r0 >> 16);
4777 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4778 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4779 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4780 * generated, therefore the output r is always a 16-bit number.
4782 r0 = (r0 & 0xFFFF) + (r0 >> 16);
4785 r0 = r0 ? r0 : 0xFFFF;
4787 *dst16_ptr = (uint16_t)r0;
4794 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4796 struct thread *t = &p->threads[p->thread_id];
4797 struct instruction *ip = t->ip;
4798 uint8_t *dst_struct, *src_struct;
4799 uint16_t *dst16_ptr;
4800 uint32_t *src32_ptr;
4804 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4807 dst_struct = t->structs[ip->alu.dst.struct_id];
4808 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4810 src_struct = t->structs[ip->alu.src.struct_id];
4811 src32_ptr = (uint32_t *)&src_struct[0];
4813 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4814 * Therefore, in the worst case scenario, a 35-bit number is added to a
4815 * 16-bit number (the input r), so the output r is 36-bit number.
4817 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4820 /* The first input is a 16-bit number. The second input is a 20-bit
4821 * number. Their sum is a 21-bit number.
4823 r = (r & 0xFFFF) + (r >> 16);
4825 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4826 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4828 r = (r & 0xFFFF) + (r >> 16);
4830 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4831 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4832 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4833 * generated, therefore the output r is always a 16-bit number.
4835 r = (r & 0xFFFF) + (r >> 16);
4840 *dst16_ptr = (uint16_t)r;
4849 static struct regarray *
4850 regarray_find(struct rte_swx_pipeline *p, const char *name);
4853 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4854 struct action *action,
4857 struct instruction *instr,
4858 struct instruction_data *data __rte_unused)
4860 char *regarray = tokens[1], *idx = tokens[2];
4863 uint32_t idx_struct_id, idx_val;
4865 CHECK(n_tokens == 3, EINVAL);
4867 r = regarray_find(p, regarray);
4870 /* REGPREFETCH_RH, REGPREFETCH_RM. */
4871 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4873 instr->type = INSTR_REGPREFETCH_RM;
4875 instr->type = INSTR_REGPREFETCH_RH;
4877 instr->regarray.regarray_id = r->id;
4878 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4879 instr->regarray.idx.n_bits = fidx->n_bits;
4880 instr->regarray.idx.offset = fidx->offset / 8;
4881 instr->regarray.dstsrc_val = 0; /* Unused. */
4885 /* REGPREFETCH_RI. */
4886 idx_val = strtoul(idx, &idx, 0);
4887 CHECK(!idx[0], EINVAL);
4889 instr->type = INSTR_REGPREFETCH_RI;
4890 instr->regarray.regarray_id = r->id;
4891 instr->regarray.idx_val = idx_val;
4892 instr->regarray.dstsrc_val = 0; /* Unused. */
4897 instr_regrd_translate(struct rte_swx_pipeline *p,
4898 struct action *action,
4901 struct instruction *instr,
4902 struct instruction_data *data __rte_unused)
4904 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4906 struct field *fdst, *fidx;
4907 uint32_t dst_struct_id, idx_struct_id, idx_val;
4909 CHECK(n_tokens == 4, EINVAL);
4911 r = regarray_find(p, regarray);
4914 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4915 CHECK(fdst, EINVAL);
4917 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
4918 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4920 instr->type = INSTR_REGRD_MRM;
4921 if (dst[0] == 'h' && idx[0] != 'h')
4922 instr->type = INSTR_REGRD_HRM;
4923 if (dst[0] != 'h' && idx[0] == 'h')
4924 instr->type = INSTR_REGRD_MRH;
4925 if (dst[0] == 'h' && idx[0] == 'h')
4926 instr->type = INSTR_REGRD_HRH;
4928 instr->regarray.regarray_id = r->id;
4929 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4930 instr->regarray.idx.n_bits = fidx->n_bits;
4931 instr->regarray.idx.offset = fidx->offset / 8;
4932 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4933 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4934 instr->regarray.dstsrc.offset = fdst->offset / 8;
4938 /* REGRD_MRI, REGRD_HRI. */
4939 idx_val = strtoul(idx, &idx, 0);
4940 CHECK(!idx[0], EINVAL);
4942 instr->type = INSTR_REGRD_MRI;
4944 instr->type = INSTR_REGRD_HRI;
4946 instr->regarray.regarray_id = r->id;
4947 instr->regarray.idx_val = idx_val;
4948 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
4949 instr->regarray.dstsrc.n_bits = fdst->n_bits;
4950 instr->regarray.dstsrc.offset = fdst->offset / 8;
4955 instr_regwr_translate(struct rte_swx_pipeline *p,
4956 struct action *action,
4959 struct instruction *instr,
4960 struct instruction_data *data __rte_unused)
4962 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
4964 struct field *fidx, *fsrc;
4966 uint32_t idx_struct_id, idx_val, src_struct_id;
4968 CHECK(n_tokens == 4, EINVAL);
4970 r = regarray_find(p, regarray);
4973 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
4974 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4975 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4977 instr->type = INSTR_REGWR_RMM;
4978 if (idx[0] == 'h' && src[0] != 'h')
4979 instr->type = INSTR_REGWR_RHM;
4980 if (idx[0] != 'h' && src[0] == 'h')
4981 instr->type = INSTR_REGWR_RMH;
4982 if (idx[0] == 'h' && src[0] == 'h')
4983 instr->type = INSTR_REGWR_RHH;
4985 instr->regarray.regarray_id = r->id;
4986 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4987 instr->regarray.idx.n_bits = fidx->n_bits;
4988 instr->regarray.idx.offset = fidx->offset / 8;
4989 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
4990 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
4991 instr->regarray.dstsrc.offset = fsrc->offset / 8;
4995 /* REGWR_RHI, REGWR_RMI. */
4996 if (fidx && !fsrc) {
4997 src_val = strtoull(src, &src, 0);
4998 CHECK(!src[0], EINVAL);
5000 instr->type = INSTR_REGWR_RMI;
5002 instr->type = INSTR_REGWR_RHI;
5004 instr->regarray.regarray_id = r->id;
5005 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5006 instr->regarray.idx.n_bits = fidx->n_bits;
5007 instr->regarray.idx.offset = fidx->offset / 8;
5008 instr->regarray.dstsrc_val = src_val;
5012 /* REGWR_RIH, REGWR_RIM. */
5013 if (!fidx && fsrc) {
5014 idx_val = strtoul(idx, &idx, 0);
5015 CHECK(!idx[0], EINVAL);
5017 instr->type = INSTR_REGWR_RIM;
5019 instr->type = INSTR_REGWR_RIH;
5021 instr->regarray.regarray_id = r->id;
5022 instr->regarray.idx_val = idx_val;
5023 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5024 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5025 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5030 src_val = strtoull(src, &src, 0);
5031 CHECK(!src[0], EINVAL);
5033 idx_val = strtoul(idx, &idx, 0);
5034 CHECK(!idx[0], EINVAL);
5036 instr->type = INSTR_REGWR_RII;
5037 instr->regarray.idx_val = idx_val;
5038 instr->regarray.dstsrc_val = src_val;
5044 instr_regadd_translate(struct rte_swx_pipeline *p,
5045 struct action *action,
5048 struct instruction *instr,
5049 struct instruction_data *data __rte_unused)
5051 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5053 struct field *fidx, *fsrc;
5055 uint32_t idx_struct_id, idx_val, src_struct_id;
5057 CHECK(n_tokens == 4, EINVAL);
5059 r = regarray_find(p, regarray);
5062 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5063 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5064 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5066 instr->type = INSTR_REGADD_RMM;
5067 if (idx[0] == 'h' && src[0] != 'h')
5068 instr->type = INSTR_REGADD_RHM;
5069 if (idx[0] != 'h' && src[0] == 'h')
5070 instr->type = INSTR_REGADD_RMH;
5071 if (idx[0] == 'h' && src[0] == 'h')
5072 instr->type = INSTR_REGADD_RHH;
5074 instr->regarray.regarray_id = r->id;
5075 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5076 instr->regarray.idx.n_bits = fidx->n_bits;
5077 instr->regarray.idx.offset = fidx->offset / 8;
5078 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5079 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5080 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5084 /* REGADD_RHI, REGADD_RMI. */
5085 if (fidx && !fsrc) {
5086 src_val = strtoull(src, &src, 0);
5087 CHECK(!src[0], EINVAL);
5089 instr->type = INSTR_REGADD_RMI;
5091 instr->type = INSTR_REGADD_RHI;
5093 instr->regarray.regarray_id = r->id;
5094 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5095 instr->regarray.idx.n_bits = fidx->n_bits;
5096 instr->regarray.idx.offset = fidx->offset / 8;
5097 instr->regarray.dstsrc_val = src_val;
5101 /* REGADD_RIH, REGADD_RIM. */
5102 if (!fidx && fsrc) {
5103 idx_val = strtoul(idx, &idx, 0);
5104 CHECK(!idx[0], EINVAL);
5106 instr->type = INSTR_REGADD_RIM;
5108 instr->type = INSTR_REGADD_RIH;
5110 instr->regarray.regarray_id = r->id;
5111 instr->regarray.idx_val = idx_val;
5112 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5113 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5114 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5119 src_val = strtoull(src, &src, 0);
5120 CHECK(!src[0], EINVAL);
5122 idx_val = strtoul(idx, &idx, 0);
5123 CHECK(!idx[0], EINVAL);
5125 instr->type = INSTR_REGADD_RII;
5126 instr->regarray.idx_val = idx_val;
5127 instr->regarray.dstsrc_val = src_val;
5131 static inline uint64_t *
5132 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5134 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5138 static inline uint64_t
5139 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5141 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5143 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5144 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5145 uint64_t idx64 = *idx64_ptr;
5146 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5147 uint64_t idx = idx64 & idx64_mask & r->size_mask;
5152 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5154 static inline uint64_t
5155 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5157 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5159 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5160 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5161 uint64_t idx64 = *idx64_ptr;
5162 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5169 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5173 static inline uint64_t
5174 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5176 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5178 uint64_t idx = ip->regarray.idx_val & r->size_mask;
5183 static inline uint64_t
5184 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5186 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5187 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5188 uint64_t src64 = *src64_ptr;
5189 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5190 uint64_t src = src64 & src64_mask;
5195 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5197 static inline uint64_t
5198 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5200 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5201 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5202 uint64_t src64 = *src64_ptr;
5203 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5210 #define instr_regarray_src_nbo instr_regarray_src_hbo
5215 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5217 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5218 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5219 uint64_t dst64 = *dst64_ptr;
5220 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5222 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5226 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5229 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5231 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5232 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5233 uint64_t dst64 = *dst64_ptr;
5234 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5236 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5237 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5242 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5247 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5249 struct thread *t = &p->threads[p->thread_id];
5250 struct instruction *ip = t->ip;
5251 uint64_t *regarray, idx;
5253 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5256 regarray = instr_regarray_regarray(p, ip);
5257 idx = instr_regarray_idx_nbo(p, t, ip);
5258 rte_prefetch0(®array[idx]);
5265 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5267 struct thread *t = &p->threads[p->thread_id];
5268 struct instruction *ip = t->ip;
5269 uint64_t *regarray, idx;
5271 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5274 regarray = instr_regarray_regarray(p, ip);
5275 idx = instr_regarray_idx_hbo(p, t, ip);
5276 rte_prefetch0(®array[idx]);
5283 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5285 struct thread *t = &p->threads[p->thread_id];
5286 struct instruction *ip = t->ip;
5287 uint64_t *regarray, idx;
5289 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5292 regarray = instr_regarray_regarray(p, ip);
5293 idx = instr_regarray_idx_imm(p, ip);
5294 rte_prefetch0(®array[idx]);
5301 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5303 struct thread *t = &p->threads[p->thread_id];
5304 struct instruction *ip = t->ip;
5305 uint64_t *regarray, idx;
5307 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5310 regarray = instr_regarray_regarray(p, ip);
5311 idx = instr_regarray_idx_nbo(p, t, ip);
5312 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5319 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5321 struct thread *t = &p->threads[p->thread_id];
5322 struct instruction *ip = t->ip;
5323 uint64_t *regarray, idx;
5325 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5328 regarray = instr_regarray_regarray(p, ip);
5329 idx = instr_regarray_idx_hbo(p, t, ip);
5330 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5337 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5339 struct thread *t = &p->threads[p->thread_id];
5340 struct instruction *ip = t->ip;
5341 uint64_t *regarray, idx;
5343 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5346 regarray = instr_regarray_regarray(p, ip);
5347 idx = instr_regarray_idx_nbo(p, t, ip);
5348 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5355 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5357 struct thread *t = &p->threads[p->thread_id];
5358 struct instruction *ip = t->ip;
5359 uint64_t *regarray, idx;
5362 regarray = instr_regarray_regarray(p, ip);
5363 idx = instr_regarray_idx_hbo(p, t, ip);
5364 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5371 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5373 struct thread *t = &p->threads[p->thread_id];
5374 struct instruction *ip = t->ip;
5375 uint64_t *regarray, idx;
5377 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5380 regarray = instr_regarray_regarray(p, ip);
5381 idx = instr_regarray_idx_imm(p, ip);
5382 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5389 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5391 struct thread *t = &p->threads[p->thread_id];
5392 struct instruction *ip = t->ip;
5393 uint64_t *regarray, idx;
5395 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5398 regarray = instr_regarray_regarray(p, ip);
5399 idx = instr_regarray_idx_imm(p, ip);
5400 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5407 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5409 struct thread *t = &p->threads[p->thread_id];
5410 struct instruction *ip = t->ip;
5411 uint64_t *regarray, idx, src;
5413 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5416 regarray = instr_regarray_regarray(p, ip);
5417 idx = instr_regarray_idx_nbo(p, t, ip);
5418 src = instr_regarray_src_nbo(t, ip);
5419 regarray[idx] = src;
5426 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5428 struct thread *t = &p->threads[p->thread_id];
5429 struct instruction *ip = t->ip;
5430 uint64_t *regarray, idx, src;
5432 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5435 regarray = instr_regarray_regarray(p, ip);
5436 idx = instr_regarray_idx_nbo(p, t, ip);
5437 src = instr_regarray_src_hbo(t, ip);
5438 regarray[idx] = src;
5445 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5447 struct thread *t = &p->threads[p->thread_id];
5448 struct instruction *ip = t->ip;
5449 uint64_t *regarray, idx, src;
5451 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5454 regarray = instr_regarray_regarray(p, ip);
5455 idx = instr_regarray_idx_hbo(p, t, ip);
5456 src = instr_regarray_src_nbo(t, ip);
5457 regarray[idx] = src;
5464 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5466 struct thread *t = &p->threads[p->thread_id];
5467 struct instruction *ip = t->ip;
5468 uint64_t *regarray, idx, src;
5470 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5473 regarray = instr_regarray_regarray(p, ip);
5474 idx = instr_regarray_idx_hbo(p, t, ip);
5475 src = instr_regarray_src_hbo(t, ip);
5476 regarray[idx] = src;
5483 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5485 struct thread *t = &p->threads[p->thread_id];
5486 struct instruction *ip = t->ip;
5487 uint64_t *regarray, idx, src;
5489 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5492 regarray = instr_regarray_regarray(p, ip);
5493 idx = instr_regarray_idx_nbo(p, t, ip);
5494 src = ip->regarray.dstsrc_val;
5495 regarray[idx] = src;
5502 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5504 struct thread *t = &p->threads[p->thread_id];
5505 struct instruction *ip = t->ip;
5506 uint64_t *regarray, idx, src;
5508 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5511 regarray = instr_regarray_regarray(p, ip);
5512 idx = instr_regarray_idx_hbo(p, t, ip);
5513 src = ip->regarray.dstsrc_val;
5514 regarray[idx] = src;
5521 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5523 struct thread *t = &p->threads[p->thread_id];
5524 struct instruction *ip = t->ip;
5525 uint64_t *regarray, idx, src;
5527 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5530 regarray = instr_regarray_regarray(p, ip);
5531 idx = instr_regarray_idx_imm(p, ip);
5532 src = instr_regarray_src_nbo(t, ip);
5533 regarray[idx] = src;
5540 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5542 struct thread *t = &p->threads[p->thread_id];
5543 struct instruction *ip = t->ip;
5544 uint64_t *regarray, idx, src;
5546 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5549 regarray = instr_regarray_regarray(p, ip);
5550 idx = instr_regarray_idx_imm(p, ip);
5551 src = instr_regarray_src_hbo(t, ip);
5552 regarray[idx] = src;
5559 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5561 struct thread *t = &p->threads[p->thread_id];
5562 struct instruction *ip = t->ip;
5563 uint64_t *regarray, idx, src;
5565 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5568 regarray = instr_regarray_regarray(p, ip);
5569 idx = instr_regarray_idx_imm(p, ip);
5570 src = ip->regarray.dstsrc_val;
5571 regarray[idx] = src;
5578 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5580 struct thread *t = &p->threads[p->thread_id];
5581 struct instruction *ip = t->ip;
5582 uint64_t *regarray, idx, src;
5584 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5587 regarray = instr_regarray_regarray(p, ip);
5588 idx = instr_regarray_idx_nbo(p, t, ip);
5589 src = instr_regarray_src_nbo(t, ip);
5590 regarray[idx] += src;
5597 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5599 struct thread *t = &p->threads[p->thread_id];
5600 struct instruction *ip = t->ip;
5601 uint64_t *regarray, idx, src;
5603 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5606 regarray = instr_regarray_regarray(p, ip);
5607 idx = instr_regarray_idx_nbo(p, t, ip);
5608 src = instr_regarray_src_hbo(t, ip);
5609 regarray[idx] += src;
5616 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5618 struct thread *t = &p->threads[p->thread_id];
5619 struct instruction *ip = t->ip;
5620 uint64_t *regarray, idx, src;
5622 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5625 regarray = instr_regarray_regarray(p, ip);
5626 idx = instr_regarray_idx_hbo(p, t, ip);
5627 src = instr_regarray_src_nbo(t, ip);
5628 regarray[idx] += src;
5635 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5637 struct thread *t = &p->threads[p->thread_id];
5638 struct instruction *ip = t->ip;
5639 uint64_t *regarray, idx, src;
5641 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5644 regarray = instr_regarray_regarray(p, ip);
5645 idx = instr_regarray_idx_hbo(p, t, ip);
5646 src = instr_regarray_src_hbo(t, ip);
5647 regarray[idx] += src;
5654 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5656 struct thread *t = &p->threads[p->thread_id];
5657 struct instruction *ip = t->ip;
5658 uint64_t *regarray, idx, src;
5660 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5663 regarray = instr_regarray_regarray(p, ip);
5664 idx = instr_regarray_idx_nbo(p, t, ip);
5665 src = ip->regarray.dstsrc_val;
5666 regarray[idx] += src;
5673 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5675 struct thread *t = &p->threads[p->thread_id];
5676 struct instruction *ip = t->ip;
5677 uint64_t *regarray, idx, src;
5679 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5682 regarray = instr_regarray_regarray(p, ip);
5683 idx = instr_regarray_idx_hbo(p, t, ip);
5684 src = ip->regarray.dstsrc_val;
5685 regarray[idx] += src;
5692 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5694 struct thread *t = &p->threads[p->thread_id];
5695 struct instruction *ip = t->ip;
5696 uint64_t *regarray, idx, src;
5698 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5701 regarray = instr_regarray_regarray(p, ip);
5702 idx = instr_regarray_idx_imm(p, ip);
5703 src = instr_regarray_src_nbo(t, ip);
5704 regarray[idx] += src;
5711 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5713 struct thread *t = &p->threads[p->thread_id];
5714 struct instruction *ip = t->ip;
5715 uint64_t *regarray, idx, src;
5717 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5720 regarray = instr_regarray_regarray(p, ip);
5721 idx = instr_regarray_idx_imm(p, ip);
5722 src = instr_regarray_src_hbo(t, ip);
5723 regarray[idx] += src;
5730 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5732 struct thread *t = &p->threads[p->thread_id];
5733 struct instruction *ip = t->ip;
5734 uint64_t *regarray, idx, src;
5736 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5739 regarray = instr_regarray_regarray(p, ip);
5740 idx = instr_regarray_idx_imm(p, ip);
5741 src = ip->regarray.dstsrc_val;
5742 regarray[idx] += src;
5751 static struct metarray *
5752 metarray_find(struct rte_swx_pipeline *p, const char *name);
5755 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5756 struct action *action,
5759 struct instruction *instr,
5760 struct instruction_data *data __rte_unused)
5762 char *metarray = tokens[1], *idx = tokens[2];
5765 uint32_t idx_struct_id, idx_val;
5767 CHECK(n_tokens == 3, EINVAL);
5769 m = metarray_find(p, metarray);
5772 /* METPREFETCH_H, METPREFETCH_M. */
5773 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5775 instr->type = INSTR_METPREFETCH_M;
5777 instr->type = INSTR_METPREFETCH_H;
5779 instr->meter.metarray_id = m->id;
5780 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5781 instr->meter.idx.n_bits = fidx->n_bits;
5782 instr->meter.idx.offset = fidx->offset / 8;
5786 /* METPREFETCH_I. */
5787 idx_val = strtoul(idx, &idx, 0);
5788 CHECK(!idx[0], EINVAL);
5790 instr->type = INSTR_METPREFETCH_I;
5791 instr->meter.metarray_id = m->id;
5792 instr->meter.idx_val = idx_val;
5797 instr_meter_translate(struct rte_swx_pipeline *p,
5798 struct action *action,
5801 struct instruction *instr,
5802 struct instruction_data *data __rte_unused)
5804 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5805 char *color_in = tokens[4], *color_out = tokens[5];
5807 struct field *fidx, *flength, *fcin, *fcout;
5808 uint32_t idx_struct_id, length_struct_id;
5809 uint32_t color_in_struct_id, color_out_struct_id;
5811 CHECK(n_tokens == 6, EINVAL);
5813 m = metarray_find(p, metarray);
5816 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5818 flength = struct_field_parse(p, action, length, &length_struct_id);
5819 CHECK(flength, EINVAL);
5821 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5823 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5824 CHECK(fcout, EINVAL);
5826 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5828 instr->type = INSTR_METER_MMM;
5829 if (idx[0] == 'h' && length[0] == 'h')
5830 instr->type = INSTR_METER_HHM;
5831 if (idx[0] == 'h' && length[0] != 'h')
5832 instr->type = INSTR_METER_HMM;
5833 if (idx[0] != 'h' && length[0] == 'h')
5834 instr->type = INSTR_METER_MHM;
5836 instr->meter.metarray_id = m->id;
5838 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5839 instr->meter.idx.n_bits = fidx->n_bits;
5840 instr->meter.idx.offset = fidx->offset / 8;
5842 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5843 instr->meter.length.n_bits = flength->n_bits;
5844 instr->meter.length.offset = flength->offset / 8;
5846 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5847 instr->meter.color_in.n_bits = fcin->n_bits;
5848 instr->meter.color_in.offset = fcin->offset / 8;
5850 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5851 instr->meter.color_out.n_bits = fcout->n_bits;
5852 instr->meter.color_out.offset = fcout->offset / 8;
5857 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5858 if (fidx && !fcin) {
5859 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5860 CHECK(!color_in[0], EINVAL);
5862 instr->type = INSTR_METER_MMI;
5863 if (idx[0] == 'h' && length[0] == 'h')
5864 instr->type = INSTR_METER_HHI;
5865 if (idx[0] == 'h' && length[0] != 'h')
5866 instr->type = INSTR_METER_HMI;
5867 if (idx[0] != 'h' && length[0] == 'h')
5868 instr->type = INSTR_METER_MHI;
5870 instr->meter.metarray_id = m->id;
5872 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5873 instr->meter.idx.n_bits = fidx->n_bits;
5874 instr->meter.idx.offset = fidx->offset / 8;
5876 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5877 instr->meter.length.n_bits = flength->n_bits;
5878 instr->meter.length.offset = flength->offset / 8;
5880 instr->meter.color_in_val = color_in_val;
5882 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5883 instr->meter.color_out.n_bits = fcout->n_bits;
5884 instr->meter.color_out.offset = fcout->offset / 8;
5889 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5890 if (!fidx && fcin) {
5893 idx_val = strtoul(idx, &idx, 0);
5894 CHECK(!idx[0], EINVAL);
5896 instr->type = INSTR_METER_IMM;
5897 if (length[0] == 'h')
5898 instr->type = INSTR_METER_IHM;
5900 instr->meter.metarray_id = m->id;
5902 instr->meter.idx_val = idx_val;
5904 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5905 instr->meter.length.n_bits = flength->n_bits;
5906 instr->meter.length.offset = flength->offset / 8;
5908 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5909 instr->meter.color_in.n_bits = fcin->n_bits;
5910 instr->meter.color_in.offset = fcin->offset / 8;
5912 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5913 instr->meter.color_out.n_bits = fcout->n_bits;
5914 instr->meter.color_out.offset = fcout->offset / 8;
5919 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
5920 if (!fidx && !fcin) {
5921 uint32_t idx_val, color_in_val;
5923 idx_val = strtoul(idx, &idx, 0);
5924 CHECK(!idx[0], EINVAL);
5926 color_in_val = strtoul(color_in, &color_in, 0);
5927 CHECK(!color_in[0], EINVAL);
5929 instr->type = INSTR_METER_IMI;
5930 if (length[0] == 'h')
5931 instr->type = INSTR_METER_IHI;
5933 instr->meter.metarray_id = m->id;
5935 instr->meter.idx_val = idx_val;
5937 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5938 instr->meter.length.n_bits = flength->n_bits;
5939 instr->meter.length.offset = flength->offset / 8;
5941 instr->meter.color_in_val = color_in_val;
5943 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5944 instr->meter.color_out.n_bits = fcout->n_bits;
5945 instr->meter.color_out.offset = fcout->offset / 8;
5953 static inline struct meter *
5954 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5956 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5958 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5959 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5960 uint64_t idx64 = *idx64_ptr;
5961 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
5962 uint64_t idx = idx64 & idx64_mask & r->size_mask;
5964 return &r->metarray[idx];
5967 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5969 static inline struct meter *
5970 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5972 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5974 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
5975 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
5976 uint64_t idx64 = *idx64_ptr;
5977 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
5979 return &r->metarray[idx];
5984 #define instr_meter_idx_nbo instr_meter_idx_hbo
5988 static inline struct meter *
5989 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5991 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
5993 uint64_t idx = ip->meter.idx_val & r->size_mask;
5995 return &r->metarray[idx];
5998 static inline uint32_t
5999 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6001 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6002 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6003 uint64_t src64 = *src64_ptr;
6004 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6005 uint64_t src = src64 & src64_mask;
6007 return (uint32_t)src;
6010 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6012 static inline uint32_t
6013 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6015 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6016 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6017 uint64_t src64 = *src64_ptr;
6018 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6020 return (uint32_t)src;
6025 #define instr_meter_length_nbo instr_meter_length_hbo
6029 static inline enum rte_color
6030 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6032 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6033 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6034 uint64_t src64 = *src64_ptr;
6035 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6036 uint64_t src = src64 & src64_mask;
6038 return (enum rte_color)src;
6042 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6044 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6045 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6046 uint64_t dst64 = *dst64_ptr;
6047 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6049 uint64_t src = (uint64_t)color_out;
6051 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6055 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6057 struct thread *t = &p->threads[p->thread_id];
6058 struct instruction *ip = t->ip;
6061 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6064 m = instr_meter_idx_nbo(p, t, ip);
6072 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6074 struct thread *t = &p->threads[p->thread_id];
6075 struct instruction *ip = t->ip;
6078 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6081 m = instr_meter_idx_hbo(p, t, ip);
6089 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6091 struct thread *t = &p->threads[p->thread_id];
6092 struct instruction *ip = t->ip;
6095 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6098 m = instr_meter_idx_imm(p, ip);
6106 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6108 struct thread *t = &p->threads[p->thread_id];
6109 struct instruction *ip = t->ip;
6111 uint64_t time, n_pkts, n_bytes;
6113 enum rte_color color_in, color_out;
6115 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6118 m = instr_meter_idx_nbo(p, t, ip);
6119 rte_prefetch0(m->n_pkts);
6120 time = rte_get_tsc_cycles();
6121 length = instr_meter_length_nbo(t, ip);
6122 color_in = instr_meter_color_in_hbo(t, ip);
6124 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6125 &m->profile->profile,
6130 color_out &= m->color_mask;
6132 n_pkts = m->n_pkts[color_out];
6133 n_bytes = m->n_bytes[color_out];
6135 instr_meter_color_out_hbo_set(t, ip, color_out);
6137 m->n_pkts[color_out] = n_pkts + 1;
6138 m->n_bytes[color_out] = n_bytes + length;
6145 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6147 struct thread *t = &p->threads[p->thread_id];
6148 struct instruction *ip = t->ip;
6150 uint64_t time, n_pkts, n_bytes;
6152 enum rte_color color_in, color_out;
6154 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6157 m = instr_meter_idx_nbo(p, t, ip);
6158 rte_prefetch0(m->n_pkts);
6159 time = rte_get_tsc_cycles();
6160 length = instr_meter_length_nbo(t, ip);
6161 color_in = (enum rte_color)ip->meter.color_in_val;
6163 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6164 &m->profile->profile,
6169 color_out &= m->color_mask;
6171 n_pkts = m->n_pkts[color_out];
6172 n_bytes = m->n_bytes[color_out];
6174 instr_meter_color_out_hbo_set(t, ip, color_out);
6176 m->n_pkts[color_out] = n_pkts + 1;
6177 m->n_bytes[color_out] = n_bytes + length;
6184 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6186 struct thread *t = &p->threads[p->thread_id];
6187 struct instruction *ip = t->ip;
6189 uint64_t time, n_pkts, n_bytes;
6191 enum rte_color color_in, color_out;
6193 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6196 m = instr_meter_idx_nbo(p, t, ip);
6197 rte_prefetch0(m->n_pkts);
6198 time = rte_get_tsc_cycles();
6199 length = instr_meter_length_hbo(t, ip);
6200 color_in = instr_meter_color_in_hbo(t, ip);
6202 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6203 &m->profile->profile,
6208 color_out &= m->color_mask;
6210 n_pkts = m->n_pkts[color_out];
6211 n_bytes = m->n_bytes[color_out];
6213 instr_meter_color_out_hbo_set(t, ip, color_out);
6215 m->n_pkts[color_out] = n_pkts + 1;
6216 m->n_bytes[color_out] = n_bytes + length;
6222 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6224 struct thread *t = &p->threads[p->thread_id];
6225 struct instruction *ip = t->ip;
6227 uint64_t time, n_pkts, n_bytes;
6229 enum rte_color color_in, color_out;
6231 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6234 m = instr_meter_idx_nbo(p, t, ip);
6235 rte_prefetch0(m->n_pkts);
6236 time = rte_get_tsc_cycles();
6237 length = instr_meter_length_hbo(t, ip);
6238 color_in = (enum rte_color)ip->meter.color_in_val;
6240 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6241 &m->profile->profile,
6246 color_out &= m->color_mask;
6248 n_pkts = m->n_pkts[color_out];
6249 n_bytes = m->n_bytes[color_out];
6251 instr_meter_color_out_hbo_set(t, ip, color_out);
6253 m->n_pkts[color_out] = n_pkts + 1;
6254 m->n_bytes[color_out] = n_bytes + length;
6261 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6263 struct thread *t = &p->threads[p->thread_id];
6264 struct instruction *ip = t->ip;
6266 uint64_t time, n_pkts, n_bytes;
6268 enum rte_color color_in, color_out;
6270 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6273 m = instr_meter_idx_hbo(p, t, ip);
6274 rte_prefetch0(m->n_pkts);
6275 time = rte_get_tsc_cycles();
6276 length = instr_meter_length_nbo(t, ip);
6277 color_in = instr_meter_color_in_hbo(t, ip);
6279 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6280 &m->profile->profile,
6285 color_out &= m->color_mask;
6287 n_pkts = m->n_pkts[color_out];
6288 n_bytes = m->n_bytes[color_out];
6290 instr_meter_color_out_hbo_set(t, ip, color_out);
6292 m->n_pkts[color_out] = n_pkts + 1;
6293 m->n_bytes[color_out] = n_bytes + length;
6300 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6302 struct thread *t = &p->threads[p->thread_id];
6303 struct instruction *ip = t->ip;
6305 uint64_t time, n_pkts, n_bytes;
6307 enum rte_color color_in, color_out;
6309 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6312 m = instr_meter_idx_hbo(p, t, ip);
6313 rte_prefetch0(m->n_pkts);
6314 time = rte_get_tsc_cycles();
6315 length = instr_meter_length_nbo(t, ip);
6316 color_in = (enum rte_color)ip->meter.color_in_val;
6318 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6319 &m->profile->profile,
6324 color_out &= m->color_mask;
6326 n_pkts = m->n_pkts[color_out];
6327 n_bytes = m->n_bytes[color_out];
6329 instr_meter_color_out_hbo_set(t, ip, color_out);
6331 m->n_pkts[color_out] = n_pkts + 1;
6332 m->n_bytes[color_out] = n_bytes + length;
6339 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6341 struct thread *t = &p->threads[p->thread_id];
6342 struct instruction *ip = t->ip;
6344 uint64_t time, n_pkts, n_bytes;
6346 enum rte_color color_in, color_out;
6348 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6351 m = instr_meter_idx_hbo(p, t, ip);
6352 rte_prefetch0(m->n_pkts);
6353 time = rte_get_tsc_cycles();
6354 length = instr_meter_length_hbo(t, ip);
6355 color_in = instr_meter_color_in_hbo(t, ip);
6357 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6358 &m->profile->profile,
6363 color_out &= m->color_mask;
6365 n_pkts = m->n_pkts[color_out];
6366 n_bytes = m->n_bytes[color_out];
6368 instr_meter_color_out_hbo_set(t, ip, color_out);
6370 m->n_pkts[color_out] = n_pkts + 1;
6371 m->n_bytes[color_out] = n_bytes + length;
6378 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6380 struct thread *t = &p->threads[p->thread_id];
6381 struct instruction *ip = t->ip;
6383 uint64_t time, n_pkts, n_bytes;
6385 enum rte_color color_in, color_out;
6387 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6390 m = instr_meter_idx_hbo(p, t, ip);
6391 rte_prefetch0(m->n_pkts);
6392 time = rte_get_tsc_cycles();
6393 length = instr_meter_length_hbo(t, ip);
6394 color_in = (enum rte_color)ip->meter.color_in_val;
6396 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6397 &m->profile->profile,
6402 color_out &= m->color_mask;
6404 n_pkts = m->n_pkts[color_out];
6405 n_bytes = m->n_bytes[color_out];
6407 instr_meter_color_out_hbo_set(t, ip, color_out);
6409 m->n_pkts[color_out] = n_pkts + 1;
6410 m->n_bytes[color_out] = n_bytes + length;
6417 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6419 struct thread *t = &p->threads[p->thread_id];
6420 struct instruction *ip = t->ip;
6422 uint64_t time, n_pkts, n_bytes;
6424 enum rte_color color_in, color_out;
6426 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6429 m = instr_meter_idx_imm(p, ip);
6430 rte_prefetch0(m->n_pkts);
6431 time = rte_get_tsc_cycles();
6432 length = instr_meter_length_nbo(t, ip);
6433 color_in = instr_meter_color_in_hbo(t, ip);
6435 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6436 &m->profile->profile,
6441 color_out &= m->color_mask;
6443 n_pkts = m->n_pkts[color_out];
6444 n_bytes = m->n_bytes[color_out];
6446 instr_meter_color_out_hbo_set(t, ip, color_out);
6448 m->n_pkts[color_out] = n_pkts + 1;
6449 m->n_bytes[color_out] = n_bytes + length;
6456 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6458 struct thread *t = &p->threads[p->thread_id];
6459 struct instruction *ip = t->ip;
6461 uint64_t time, n_pkts, n_bytes;
6463 enum rte_color color_in, color_out;
6465 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6468 m = instr_meter_idx_imm(p, ip);
6469 rte_prefetch0(m->n_pkts);
6470 time = rte_get_tsc_cycles();
6471 length = instr_meter_length_nbo(t, ip);
6472 color_in = (enum rte_color)ip->meter.color_in_val;
6474 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6475 &m->profile->profile,
6480 color_out &= m->color_mask;
6482 n_pkts = m->n_pkts[color_out];
6483 n_bytes = m->n_bytes[color_out];
6485 instr_meter_color_out_hbo_set(t, ip, color_out);
6487 m->n_pkts[color_out] = n_pkts + 1;
6488 m->n_bytes[color_out] = n_bytes + length;
6495 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6497 struct thread *t = &p->threads[p->thread_id];
6498 struct instruction *ip = t->ip;
6500 uint64_t time, n_pkts, n_bytes;
6502 enum rte_color color_in, color_out;
6504 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6507 m = instr_meter_idx_imm(p, ip);
6508 rte_prefetch0(m->n_pkts);
6509 time = rte_get_tsc_cycles();
6510 length = instr_meter_length_hbo(t, ip);
6511 color_in = instr_meter_color_in_hbo(t, ip);
6513 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6514 &m->profile->profile,
6519 color_out &= m->color_mask;
6521 n_pkts = m->n_pkts[color_out];
6522 n_bytes = m->n_bytes[color_out];
6524 instr_meter_color_out_hbo_set(t, ip, color_out);
6526 m->n_pkts[color_out] = n_pkts + 1;
6527 m->n_bytes[color_out] = n_bytes + length;
6533 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6535 struct thread *t = &p->threads[p->thread_id];
6536 struct instruction *ip = t->ip;
6538 uint64_t time, n_pkts, n_bytes;
6540 enum rte_color color_in, color_out;
6542 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6545 m = instr_meter_idx_imm(p, ip);
6546 rte_prefetch0(m->n_pkts);
6547 time = rte_get_tsc_cycles();
6548 length = instr_meter_length_hbo(t, ip);
6549 color_in = (enum rte_color)ip->meter.color_in_val;
6551 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6552 &m->profile->profile,
6557 color_out &= m->color_mask;
6559 n_pkts = m->n_pkts[color_out];
6560 n_bytes = m->n_bytes[color_out];
6562 instr_meter_color_out_hbo_set(t, ip, color_out);
6564 m->n_pkts[color_out] = n_pkts + 1;
6565 m->n_bytes[color_out] = n_bytes + length;
6574 static struct action *
6575 action_find(struct rte_swx_pipeline *p, const char *name);
6578 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6579 struct action *action __rte_unused,
6582 struct instruction *instr,
6583 struct instruction_data *data)
6585 CHECK(n_tokens == 2, EINVAL);
6587 strcpy(data->jmp_label, tokens[1]);
6589 instr->type = INSTR_JMP;
6590 instr->jmp.ip = NULL; /* Resolved later. */
6595 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6596 struct action *action __rte_unused,
6599 struct instruction *instr,
6600 struct instruction_data *data)
6604 CHECK(n_tokens == 3, EINVAL);
6606 strcpy(data->jmp_label, tokens[1]);
6608 h = header_parse(p, tokens[2]);
6611 instr->type = INSTR_JMP_VALID;
6612 instr->jmp.ip = NULL; /* Resolved later. */
6613 instr->jmp.header_id = h->id;
6618 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6619 struct action *action __rte_unused,
6622 struct instruction *instr,
6623 struct instruction_data *data)
6627 CHECK(n_tokens == 3, EINVAL);
6629 strcpy(data->jmp_label, tokens[1]);
6631 h = header_parse(p, tokens[2]);
6634 instr->type = INSTR_JMP_INVALID;
6635 instr->jmp.ip = NULL; /* Resolved later. */
6636 instr->jmp.header_id = h->id;
6641 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6642 struct action *action,
6645 struct instruction *instr,
6646 struct instruction_data *data)
6648 CHECK(!action, EINVAL);
6649 CHECK(n_tokens == 2, EINVAL);
6651 strcpy(data->jmp_label, tokens[1]);
6653 instr->type = INSTR_JMP_HIT;
6654 instr->jmp.ip = NULL; /* Resolved later. */
6659 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6660 struct action *action,
6663 struct instruction *instr,
6664 struct instruction_data *data)
6666 CHECK(!action, EINVAL);
6667 CHECK(n_tokens == 2, EINVAL);
6669 strcpy(data->jmp_label, tokens[1]);
6671 instr->type = INSTR_JMP_MISS;
6672 instr->jmp.ip = NULL; /* Resolved later. */
6677 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6678 struct action *action,
6681 struct instruction *instr,
6682 struct instruction_data *data)
6686 CHECK(!action, EINVAL);
6687 CHECK(n_tokens == 3, EINVAL);
6689 strcpy(data->jmp_label, tokens[1]);
6691 a = action_find(p, tokens[2]);
6694 instr->type = INSTR_JMP_ACTION_HIT;
6695 instr->jmp.ip = NULL; /* Resolved later. */
6696 instr->jmp.action_id = a->id;
6701 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6702 struct action *action,
6705 struct instruction *instr,
6706 struct instruction_data *data)
6710 CHECK(!action, EINVAL);
6711 CHECK(n_tokens == 3, EINVAL);
6713 strcpy(data->jmp_label, tokens[1]);
6715 a = action_find(p, tokens[2]);
6718 instr->type = INSTR_JMP_ACTION_MISS;
6719 instr->jmp.ip = NULL; /* Resolved later. */
6720 instr->jmp.action_id = a->id;
6725 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6726 struct action *action,
6729 struct instruction *instr,
6730 struct instruction_data *data)
6732 char *a = tokens[2], *b = tokens[3];
6733 struct field *fa, *fb;
6735 uint32_t a_struct_id, b_struct_id;
6737 CHECK(n_tokens == 4, EINVAL);
6739 strcpy(data->jmp_label, tokens[1]);
6741 fa = struct_field_parse(p, action, a, &a_struct_id);
6744 /* JMP_EQ or JMP_EQ_S. */
6745 fb = struct_field_parse(p, action, b, &b_struct_id);
6747 instr->type = INSTR_JMP_EQ;
6748 if ((a[0] == 'h' && b[0] != 'h') ||
6749 (a[0] != 'h' && b[0] == 'h'))
6750 instr->type = INSTR_JMP_EQ_S;
6751 instr->jmp.ip = NULL; /* Resolved later. */
6753 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6754 instr->jmp.a.n_bits = fa->n_bits;
6755 instr->jmp.a.offset = fa->offset / 8;
6756 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6757 instr->jmp.b.n_bits = fb->n_bits;
6758 instr->jmp.b.offset = fb->offset / 8;
6763 b_val = strtoull(b, &b, 0);
6764 CHECK(!b[0], EINVAL);
6767 b_val = hton64(b_val) >> (64 - fa->n_bits);
6769 instr->type = INSTR_JMP_EQ_I;
6770 instr->jmp.ip = NULL; /* Resolved later. */
6771 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6772 instr->jmp.a.n_bits = fa->n_bits;
6773 instr->jmp.a.offset = fa->offset / 8;
6774 instr->jmp.b_val = b_val;
6779 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6780 struct action *action,
6783 struct instruction *instr,
6784 struct instruction_data *data)
6786 char *a = tokens[2], *b = tokens[3];
6787 struct field *fa, *fb;
6789 uint32_t a_struct_id, b_struct_id;
6791 CHECK(n_tokens == 4, EINVAL);
6793 strcpy(data->jmp_label, tokens[1]);
6795 fa = struct_field_parse(p, action, a, &a_struct_id);
6798 /* JMP_NEQ or JMP_NEQ_S. */
6799 fb = struct_field_parse(p, action, b, &b_struct_id);
6801 instr->type = INSTR_JMP_NEQ;
6802 if ((a[0] == 'h' && b[0] != 'h') ||
6803 (a[0] != 'h' && b[0] == 'h'))
6804 instr->type = INSTR_JMP_NEQ_S;
6805 instr->jmp.ip = NULL; /* Resolved later. */
6807 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6808 instr->jmp.a.n_bits = fa->n_bits;
6809 instr->jmp.a.offset = fa->offset / 8;
6810 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6811 instr->jmp.b.n_bits = fb->n_bits;
6812 instr->jmp.b.offset = fb->offset / 8;
6817 b_val = strtoull(b, &b, 0);
6818 CHECK(!b[0], EINVAL);
6821 b_val = hton64(b_val) >> (64 - fa->n_bits);
6823 instr->type = INSTR_JMP_NEQ_I;
6824 instr->jmp.ip = NULL; /* Resolved later. */
6825 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6826 instr->jmp.a.n_bits = fa->n_bits;
6827 instr->jmp.a.offset = fa->offset / 8;
6828 instr->jmp.b_val = b_val;
6833 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6834 struct action *action,
6837 struct instruction *instr,
6838 struct instruction_data *data)
6840 char *a = tokens[2], *b = tokens[3];
6841 struct field *fa, *fb;
6843 uint32_t a_struct_id, b_struct_id;
6845 CHECK(n_tokens == 4, EINVAL);
6847 strcpy(data->jmp_label, tokens[1]);
6849 fa = struct_field_parse(p, action, a, &a_struct_id);
6852 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6853 fb = struct_field_parse(p, action, b, &b_struct_id);
6855 instr->type = INSTR_JMP_LT;
6856 if (a[0] == 'h' && b[0] != 'h')
6857 instr->type = INSTR_JMP_LT_HM;
6858 if (a[0] != 'h' && b[0] == 'h')
6859 instr->type = INSTR_JMP_LT_MH;
6860 if (a[0] == 'h' && b[0] == 'h')
6861 instr->type = INSTR_JMP_LT_HH;
6862 instr->jmp.ip = NULL; /* Resolved later. */
6864 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6865 instr->jmp.a.n_bits = fa->n_bits;
6866 instr->jmp.a.offset = fa->offset / 8;
6867 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6868 instr->jmp.b.n_bits = fb->n_bits;
6869 instr->jmp.b.offset = fb->offset / 8;
6873 /* JMP_LT_MI, JMP_LT_HI. */
6874 b_val = strtoull(b, &b, 0);
6875 CHECK(!b[0], EINVAL);
6877 instr->type = INSTR_JMP_LT_MI;
6879 instr->type = INSTR_JMP_LT_HI;
6880 instr->jmp.ip = NULL; /* Resolved later. */
6882 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6883 instr->jmp.a.n_bits = fa->n_bits;
6884 instr->jmp.a.offset = fa->offset / 8;
6885 instr->jmp.b_val = b_val;
6890 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6891 struct action *action,
6894 struct instruction *instr,
6895 struct instruction_data *data)
6897 char *a = tokens[2], *b = tokens[3];
6898 struct field *fa, *fb;
6900 uint32_t a_struct_id, b_struct_id;
6902 CHECK(n_tokens == 4, EINVAL);
6904 strcpy(data->jmp_label, tokens[1]);
6906 fa = struct_field_parse(p, action, a, &a_struct_id);
6909 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6910 fb = struct_field_parse(p, action, b, &b_struct_id);
6912 instr->type = INSTR_JMP_GT;
6913 if (a[0] == 'h' && b[0] != 'h')
6914 instr->type = INSTR_JMP_GT_HM;
6915 if (a[0] != 'h' && b[0] == 'h')
6916 instr->type = INSTR_JMP_GT_MH;
6917 if (a[0] == 'h' && b[0] == 'h')
6918 instr->type = INSTR_JMP_GT_HH;
6919 instr->jmp.ip = NULL; /* Resolved later. */
6921 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6922 instr->jmp.a.n_bits = fa->n_bits;
6923 instr->jmp.a.offset = fa->offset / 8;
6924 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6925 instr->jmp.b.n_bits = fb->n_bits;
6926 instr->jmp.b.offset = fb->offset / 8;
6930 /* JMP_GT_MI, JMP_GT_HI. */
6931 b_val = strtoull(b, &b, 0);
6932 CHECK(!b[0], EINVAL);
6934 instr->type = INSTR_JMP_GT_MI;
6936 instr->type = INSTR_JMP_GT_HI;
6937 instr->jmp.ip = NULL; /* Resolved later. */
6939 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6940 instr->jmp.a.n_bits = fa->n_bits;
6941 instr->jmp.a.offset = fa->offset / 8;
6942 instr->jmp.b_val = b_val;
6947 instr_jmp_exec(struct rte_swx_pipeline *p)
6949 struct thread *t = &p->threads[p->thread_id];
6950 struct instruction *ip = t->ip;
6952 TRACE("[Thread %2u] jmp\n", p->thread_id);
6954 thread_ip_set(t, ip->jmp.ip);
6958 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
6960 struct thread *t = &p->threads[p->thread_id];
6961 struct instruction *ip = t->ip;
6962 uint32_t header_id = ip->jmp.header_id;
6964 TRACE("[Thread %2u] jmpv\n", p->thread_id);
6966 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
6970 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
6972 struct thread *t = &p->threads[p->thread_id];
6973 struct instruction *ip = t->ip;
6974 uint32_t header_id = ip->jmp.header_id;
6976 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
6978 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
6982 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
6984 struct thread *t = &p->threads[p->thread_id];
6985 struct instruction *ip = t->ip;
6986 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
6988 TRACE("[Thread %2u] jmph\n", p->thread_id);
6990 t->ip = ip_next[t->hit];
6994 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
6996 struct thread *t = &p->threads[p->thread_id];
6997 struct instruction *ip = t->ip;
6998 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7000 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7002 t->ip = ip_next[t->hit];
7006 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7008 struct thread *t = &p->threads[p->thread_id];
7009 struct instruction *ip = t->ip;
7011 TRACE("[Thread %2u] jmpa\n", p->thread_id);
7013 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7017 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7019 struct thread *t = &p->threads[p->thread_id];
7020 struct instruction *ip = t->ip;
7022 TRACE("[Thread %2u] jmpna\n", p->thread_id);
7024 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7028 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7030 struct thread *t = &p->threads[p->thread_id];
7031 struct instruction *ip = t->ip;
7033 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7039 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7041 struct thread *t = &p->threads[p->thread_id];
7042 struct instruction *ip = t->ip;
7044 TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7046 JMP_CMP_S(t, ip, ==);
7050 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7052 struct thread *t = &p->threads[p->thread_id];
7053 struct instruction *ip = t->ip;
7055 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7057 JMP_CMP_I(t, ip, ==);
7061 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7063 struct thread *t = &p->threads[p->thread_id];
7064 struct instruction *ip = t->ip;
7066 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7072 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7074 struct thread *t = &p->threads[p->thread_id];
7075 struct instruction *ip = t->ip;
7077 TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7079 JMP_CMP_S(t, ip, !=);
7083 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7085 struct thread *t = &p->threads[p->thread_id];
7086 struct instruction *ip = t->ip;
7088 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7090 JMP_CMP_I(t, ip, !=);
7094 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7096 struct thread *t = &p->threads[p->thread_id];
7097 struct instruction *ip = t->ip;
7099 TRACE("[Thread %2u] jmplt\n", p->thread_id);
7105 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7107 struct thread *t = &p->threads[p->thread_id];
7108 struct instruction *ip = t->ip;
7110 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7112 JMP_CMP_MH(t, ip, <);
7116 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7118 struct thread *t = &p->threads[p->thread_id];
7119 struct instruction *ip = t->ip;
7121 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7123 JMP_CMP_HM(t, ip, <);
7127 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7129 struct thread *t = &p->threads[p->thread_id];
7130 struct instruction *ip = t->ip;
7132 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7134 JMP_CMP_HH(t, ip, <);
7138 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7140 struct thread *t = &p->threads[p->thread_id];
7141 struct instruction *ip = t->ip;
7143 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7145 JMP_CMP_MI(t, ip, <);
7149 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7151 struct thread *t = &p->threads[p->thread_id];
7152 struct instruction *ip = t->ip;
7154 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7156 JMP_CMP_HI(t, ip, <);
7160 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7162 struct thread *t = &p->threads[p->thread_id];
7163 struct instruction *ip = t->ip;
7165 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7171 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7173 struct thread *t = &p->threads[p->thread_id];
7174 struct instruction *ip = t->ip;
7176 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7178 JMP_CMP_MH(t, ip, >);
7182 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7184 struct thread *t = &p->threads[p->thread_id];
7185 struct instruction *ip = t->ip;
7187 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7189 JMP_CMP_HM(t, ip, >);
7193 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7195 struct thread *t = &p->threads[p->thread_id];
7196 struct instruction *ip = t->ip;
7198 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7200 JMP_CMP_HH(t, ip, >);
7204 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7206 struct thread *t = &p->threads[p->thread_id];
7207 struct instruction *ip = t->ip;
7209 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7211 JMP_CMP_MI(t, ip, >);
7215 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7217 struct thread *t = &p->threads[p->thread_id];
7218 struct instruction *ip = t->ip;
7220 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7222 JMP_CMP_HI(t, ip, >);
7229 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7230 struct action *action,
7231 char **tokens __rte_unused,
7233 struct instruction *instr,
7234 struct instruction_data *data __rte_unused)
7236 CHECK(action, EINVAL);
7237 CHECK(n_tokens == 1, EINVAL);
7239 instr->type = INSTR_RETURN;
7244 instr_return_exec(struct rte_swx_pipeline *p)
7246 struct thread *t = &p->threads[p->thread_id];
7248 TRACE("[Thread %2u] return\n", p->thread_id);
7254 instr_translate(struct rte_swx_pipeline *p,
7255 struct action *action,
7257 struct instruction *instr,
7258 struct instruction_data *data)
7260 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7261 int n_tokens = 0, tpos = 0;
7263 /* Parse the instruction string into tokens. */
7267 token = strtok_r(string, " \t\v", &string);
7271 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7272 CHECK_NAME(token, EINVAL);
7274 tokens[n_tokens] = token;
7278 CHECK(n_tokens, EINVAL);
7280 /* Handle the optional instruction label. */
7281 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7282 strcpy(data->label, tokens[0]);
7285 CHECK(n_tokens - tpos, EINVAL);
7288 /* Identify the instruction type. */
7289 if (!strcmp(tokens[tpos], "rx"))
7290 return instr_rx_translate(p,
7297 if (!strcmp(tokens[tpos], "tx"))
7298 return instr_tx_translate(p,
7305 if (!strcmp(tokens[tpos], "extract"))
7306 return instr_hdr_extract_translate(p,
7313 if (!strcmp(tokens[tpos], "emit"))
7314 return instr_hdr_emit_translate(p,
7321 if (!strcmp(tokens[tpos], "validate"))
7322 return instr_hdr_validate_translate(p,
7329 if (!strcmp(tokens[tpos], "invalidate"))
7330 return instr_hdr_invalidate_translate(p,
7337 if (!strcmp(tokens[tpos], "mov"))
7338 return instr_mov_translate(p,
7345 if (!strcmp(tokens[tpos], "dma"))
7346 return instr_dma_translate(p,
7353 if (!strcmp(tokens[tpos], "add"))
7354 return instr_alu_add_translate(p,
7361 if (!strcmp(tokens[tpos], "sub"))
7362 return instr_alu_sub_translate(p,
7369 if (!strcmp(tokens[tpos], "ckadd"))
7370 return instr_alu_ckadd_translate(p,
7377 if (!strcmp(tokens[tpos], "cksub"))
7378 return instr_alu_cksub_translate(p,
7385 if (!strcmp(tokens[tpos], "and"))
7386 return instr_alu_and_translate(p,
7393 if (!strcmp(tokens[tpos], "or"))
7394 return instr_alu_or_translate(p,
7401 if (!strcmp(tokens[tpos], "xor"))
7402 return instr_alu_xor_translate(p,
7409 if (!strcmp(tokens[tpos], "shl"))
7410 return instr_alu_shl_translate(p,
7417 if (!strcmp(tokens[tpos], "shr"))
7418 return instr_alu_shr_translate(p,
7425 if (!strcmp(tokens[tpos], "regprefetch"))
7426 return instr_regprefetch_translate(p,
7433 if (!strcmp(tokens[tpos], "regrd"))
7434 return instr_regrd_translate(p,
7441 if (!strcmp(tokens[tpos], "regwr"))
7442 return instr_regwr_translate(p,
7449 if (!strcmp(tokens[tpos], "regadd"))
7450 return instr_regadd_translate(p,
7457 if (!strcmp(tokens[tpos], "metprefetch"))
7458 return instr_metprefetch_translate(p,
7465 if (!strcmp(tokens[tpos], "meter"))
7466 return instr_meter_translate(p,
7473 if (!strcmp(tokens[tpos], "table"))
7474 return instr_table_translate(p,
7481 if (!strcmp(tokens[tpos], "extern"))
7482 return instr_extern_translate(p,
7489 if (!strcmp(tokens[tpos], "jmp"))
7490 return instr_jmp_translate(p,
7497 if (!strcmp(tokens[tpos], "jmpv"))
7498 return instr_jmp_valid_translate(p,
7505 if (!strcmp(tokens[tpos], "jmpnv"))
7506 return instr_jmp_invalid_translate(p,
7513 if (!strcmp(tokens[tpos], "jmph"))
7514 return instr_jmp_hit_translate(p,
7521 if (!strcmp(tokens[tpos], "jmpnh"))
7522 return instr_jmp_miss_translate(p,
7529 if (!strcmp(tokens[tpos], "jmpa"))
7530 return instr_jmp_action_hit_translate(p,
7537 if (!strcmp(tokens[tpos], "jmpna"))
7538 return instr_jmp_action_miss_translate(p,
7545 if (!strcmp(tokens[tpos], "jmpeq"))
7546 return instr_jmp_eq_translate(p,
7553 if (!strcmp(tokens[tpos], "jmpneq"))
7554 return instr_jmp_neq_translate(p,
7561 if (!strcmp(tokens[tpos], "jmplt"))
7562 return instr_jmp_lt_translate(p,
7569 if (!strcmp(tokens[tpos], "jmpgt"))
7570 return instr_jmp_gt_translate(p,
7577 if (!strcmp(tokens[tpos], "return"))
7578 return instr_return_translate(p,
7588 static struct instruction_data *
7589 label_find(struct instruction_data *data, uint32_t n, const char *label)
7593 for (i = 0; i < n; i++)
7594 if (!strcmp(label, data[i].label))
7601 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7603 uint32_t count = 0, i;
7608 for (i = 0; i < n; i++)
7609 if (!strcmp(label, data[i].jmp_label))
7616 instr_label_check(struct instruction_data *instruction_data,
7617 uint32_t n_instructions)
7621 /* Check that all instruction labels are unique. */
7622 for (i = 0; i < n_instructions; i++) {
7623 struct instruction_data *data = &instruction_data[i];
7624 char *label = data->label;
7630 for (j = i + 1; j < n_instructions; j++)
7631 CHECK(strcmp(label, data[j].label), EINVAL);
7634 /* Get users for each instruction label. */
7635 for (i = 0; i < n_instructions; i++) {
7636 struct instruction_data *data = &instruction_data[i];
7637 char *label = data->label;
7639 data->n_users = label_is_used(instruction_data,
7648 instr_jmp_resolve(struct instruction *instructions,
7649 struct instruction_data *instruction_data,
7650 uint32_t n_instructions)
7654 for (i = 0; i < n_instructions; i++) {
7655 struct instruction *instr = &instructions[i];
7656 struct instruction_data *data = &instruction_data[i];
7657 struct instruction_data *found;
7659 if (!instruction_is_jmp(instr))
7662 found = label_find(instruction_data,
7665 CHECK(found, EINVAL);
7667 instr->jmp.ip = &instructions[found - instruction_data];
7674 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7676 struct instruction *instr,
7677 struct instruction_data *data __rte_unused,
7678 uint32_t n_instructions)
7681 enum instruction_type type;
7684 /* Check that the first instruction is rx. */
7685 CHECK(instr[0].type == INSTR_RX, EINVAL);
7687 /* Check that there is at least one tx instruction. */
7688 for (i = 0; i < n_instructions; i++) {
7689 type = instr[i].type;
7691 if (type == INSTR_TX)
7694 CHECK(i < n_instructions, EINVAL);
7696 /* Check that the last instruction is either tx or unconditional
7699 type = instr[n_instructions - 1].type;
7700 CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
7704 enum instruction_type type;
7707 /* Check that there is at least one return or tx instruction. */
7708 for (i = 0; i < n_instructions; i++) {
7709 type = instr[i].type;
7711 if ((type == INSTR_RETURN) || (type == INSTR_TX))
7714 CHECK(i < n_instructions, EINVAL);
7721 instr_pattern_extract_many_detect(struct instruction *instr,
7722 struct instruction_data *data,
7724 uint32_t *n_pattern_instr)
7728 for (i = 0; i < n_instr; i++) {
7729 if (data[i].invalid)
7732 if (instr[i].type != INSTR_HDR_EXTRACT)
7735 if (i == RTE_DIM(instr->io.hdr.header_id))
7738 if (i && data[i].n_users)
7745 *n_pattern_instr = i;
7750 instr_pattern_extract_many_optimize(struct instruction *instr,
7751 struct instruction_data *data,
7756 for (i = 1; i < n_instr; i++) {
7758 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7759 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7760 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7762 data[i].invalid = 1;
7767 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7768 struct instruction_data *data,
7770 uint32_t *n_pattern_instr)
7774 for (i = 0; i < n_instr; i++) {
7775 if (data[i].invalid)
7778 if (instr[i].type != INSTR_HDR_EMIT)
7781 if (i == RTE_DIM(instr->io.hdr.header_id))
7784 if (i && data[i].n_users)
7791 if (instr[i].type != INSTR_TX)
7794 if (data[i].n_users)
7799 *n_pattern_instr = i;
7804 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7805 struct instruction_data *data,
7810 /* Any emit instruction in addition to the first one. */
7811 for (i = 1; i < n_instr - 1; i++) {
7813 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7814 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7815 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7817 data[i].invalid = 1;
7820 /* The TX instruction is the last one in the pattern. */
7822 instr[0].io.io.offset = instr[i].io.io.offset;
7823 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7824 data[i].invalid = 1;
7828 instr_pattern_dma_many_detect(struct instruction *instr,
7829 struct instruction_data *data,
7831 uint32_t *n_pattern_instr)
7835 for (i = 0; i < n_instr; i++) {
7836 if (data[i].invalid)
7839 if (instr[i].type != INSTR_DMA_HT)
7842 if (i == RTE_DIM(instr->dma.dst.header_id))
7845 if (i && data[i].n_users)
7852 *n_pattern_instr = i;
7857 instr_pattern_dma_many_optimize(struct instruction *instr,
7858 struct instruction_data *data,
7863 for (i = 1; i < n_instr; i++) {
7865 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7866 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7867 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7868 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7870 data[i].invalid = 1;
7875 instr_optimize(struct instruction *instructions,
7876 struct instruction_data *instruction_data,
7877 uint32_t n_instructions)
7879 uint32_t i, pos = 0;
7881 for (i = 0; i < n_instructions; ) {
7882 struct instruction *instr = &instructions[i];
7883 struct instruction_data *data = &instruction_data[i];
7884 uint32_t n_instr = 0;
7888 detected = instr_pattern_extract_many_detect(instr,
7893 instr_pattern_extract_many_optimize(instr,
7900 /* Emit many + TX. */
7901 detected = instr_pattern_emit_many_tx_detect(instr,
7906 instr_pattern_emit_many_tx_optimize(instr,
7914 detected = instr_pattern_dma_many_detect(instr,
7919 instr_pattern_dma_many_optimize(instr, data, n_instr);
7924 /* No pattern starting at the current instruction. */
7928 /* Eliminate the invalid instructions that have been optimized out. */
7929 for (i = 0; i < n_instructions; i++) {
7930 struct instruction *instr = &instructions[i];
7931 struct instruction_data *data = &instruction_data[i];
7937 memcpy(&instructions[pos], instr, sizeof(*instr));
7938 memcpy(&instruction_data[pos], data, sizeof(*data));
7948 instruction_config(struct rte_swx_pipeline *p,
7950 const char **instructions,
7951 uint32_t n_instructions)
7953 struct instruction *instr = NULL;
7954 struct instruction_data *data = NULL;
7958 CHECK(n_instructions, EINVAL);
7959 CHECK(instructions, EINVAL);
7960 for (i = 0; i < n_instructions; i++)
7961 CHECK_INSTRUCTION(instructions[i], EINVAL);
7963 /* Memory allocation. */
7964 instr = calloc(n_instructions, sizeof(struct instruction));
7970 data = calloc(n_instructions, sizeof(struct instruction_data));
7976 for (i = 0; i < n_instructions; i++) {
7977 char *string = strdup(instructions[i]);
7983 err = instr_translate(p, a, string, &instr[i], &data[i]);
7992 err = instr_label_check(data, n_instructions);
7996 err = instr_verify(p, a, instr, data, n_instructions);
8000 n_instructions = instr_optimize(instr, data, n_instructions);
8002 err = instr_jmp_resolve(instr, data, n_instructions);
8007 a->instructions = instr;
8008 a->n_instructions = n_instructions;
8010 p->instructions = instr;
8011 p->n_instructions = n_instructions;
8023 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8025 static instr_exec_t instruction_table[] = {
8026 [INSTR_RX] = instr_rx_exec,
8027 [INSTR_TX] = instr_tx_exec,
8029 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8030 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8031 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8032 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8033 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8034 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8035 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8036 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8038 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8039 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8040 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8041 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8042 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8043 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8044 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8045 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8046 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8048 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8049 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8051 [INSTR_MOV] = instr_mov_exec,
8052 [INSTR_MOV_S] = instr_mov_s_exec,
8053 [INSTR_MOV_I] = instr_mov_i_exec,
8055 [INSTR_DMA_HT] = instr_dma_ht_exec,
8056 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8057 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8058 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8059 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8060 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8061 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8062 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8064 [INSTR_ALU_ADD] = instr_alu_add_exec,
8065 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8066 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8067 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8068 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8069 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8071 [INSTR_ALU_SUB] = instr_alu_sub_exec,
8072 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8073 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8074 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8075 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8076 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8078 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8079 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8080 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8081 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8083 [INSTR_ALU_AND] = instr_alu_and_exec,
8084 [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8085 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8087 [INSTR_ALU_OR] = instr_alu_or_exec,
8088 [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8089 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8091 [INSTR_ALU_XOR] = instr_alu_xor_exec,
8092 [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8093 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8095 [INSTR_ALU_SHL] = instr_alu_shl_exec,
8096 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8097 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8098 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8099 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8100 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8102 [INSTR_ALU_SHR] = instr_alu_shr_exec,
8103 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8104 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8105 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8106 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8107 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8109 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8110 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8111 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8113 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8114 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8115 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8116 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8117 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8118 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8120 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8121 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8122 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8123 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8124 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8125 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8126 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8127 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8128 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8130 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8131 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8132 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8133 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8134 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8135 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8136 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8137 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8138 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8140 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8141 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8142 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8144 [INSTR_METER_HHM] = instr_meter_hhm_exec,
8145 [INSTR_METER_HHI] = instr_meter_hhi_exec,
8146 [INSTR_METER_HMM] = instr_meter_hmm_exec,
8147 [INSTR_METER_HMI] = instr_meter_hmi_exec,
8148 [INSTR_METER_MHM] = instr_meter_mhm_exec,
8149 [INSTR_METER_MHI] = instr_meter_mhi_exec,
8150 [INSTR_METER_MMM] = instr_meter_mmm_exec,
8151 [INSTR_METER_MMI] = instr_meter_mmi_exec,
8152 [INSTR_METER_IHM] = instr_meter_ihm_exec,
8153 [INSTR_METER_IHI] = instr_meter_ihi_exec,
8154 [INSTR_METER_IMM] = instr_meter_imm_exec,
8155 [INSTR_METER_IMI] = instr_meter_imi_exec,
8157 [INSTR_TABLE] = instr_table_exec,
8158 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8159 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8161 [INSTR_JMP] = instr_jmp_exec,
8162 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8163 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8164 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8165 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8166 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8167 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8169 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8170 [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8171 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8173 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8174 [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8175 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8177 [INSTR_JMP_LT] = instr_jmp_lt_exec,
8178 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8179 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8180 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8181 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8182 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8184 [INSTR_JMP_GT] = instr_jmp_gt_exec,
8185 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8186 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8187 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8188 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8189 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8191 [INSTR_RETURN] = instr_return_exec,
8195 instr_exec(struct rte_swx_pipeline *p)
8197 struct thread *t = &p->threads[p->thread_id];
8198 struct instruction *ip = t->ip;
8199 instr_exec_t instr = instruction_table[ip->type];
8207 static struct action *
8208 action_find(struct rte_swx_pipeline *p, const char *name)
8210 struct action *elem;
8215 TAILQ_FOREACH(elem, &p->actions, node)
8216 if (strcmp(elem->name, name) == 0)
8222 static struct action *
8223 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8225 struct action *action = NULL;
8227 TAILQ_FOREACH(action, &p->actions, node)
8228 if (action->id == id)
8234 static struct field *
8235 action_field_find(struct action *a, const char *name)
8237 return a->st ? struct_type_field_find(a->st, name) : NULL;
8240 static struct field *
8241 action_field_parse(struct action *action, const char *name)
8243 if (name[0] != 't' || name[1] != '.')
8246 return action_field_find(action, &name[2]);
8250 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8252 const char *args_struct_type_name,
8253 const char **instructions,
8254 uint32_t n_instructions)
8256 struct struct_type *args_struct_type;
8262 CHECK_NAME(name, EINVAL);
8263 CHECK(!action_find(p, name), EEXIST);
8265 if (args_struct_type_name) {
8266 CHECK_NAME(args_struct_type_name, EINVAL);
8267 args_struct_type = struct_type_find(p, args_struct_type_name);
8268 CHECK(args_struct_type, EINVAL);
8270 args_struct_type = NULL;
8273 /* Node allocation. */
8274 a = calloc(1, sizeof(struct action));
8277 /* Node initialization. */
8278 strcpy(a->name, name);
8279 a->st = args_struct_type;
8280 a->id = p->n_actions;
8282 /* Instruction translation. */
8283 err = instruction_config(p, a, instructions, n_instructions);
8289 /* Node add to tailq. */
8290 TAILQ_INSERT_TAIL(&p->actions, a, node);
8297 action_build(struct rte_swx_pipeline *p)
8299 struct action *action;
8301 p->action_instructions = calloc(p->n_actions,
8302 sizeof(struct instruction *));
8303 CHECK(p->action_instructions, ENOMEM);
8305 TAILQ_FOREACH(action, &p->actions, node)
8306 p->action_instructions[action->id] = action->instructions;
8312 action_build_free(struct rte_swx_pipeline *p)
8314 free(p->action_instructions);
8315 p->action_instructions = NULL;
8319 action_free(struct rte_swx_pipeline *p)
8321 action_build_free(p);
8324 struct action *action;
8326 action = TAILQ_FIRST(&p->actions);
8330 TAILQ_REMOVE(&p->actions, action, node);
8331 free(action->instructions);
8339 static struct table_type *
8340 table_type_find(struct rte_swx_pipeline *p, const char *name)
8342 struct table_type *elem;
8344 TAILQ_FOREACH(elem, &p->table_types, node)
8345 if (strcmp(elem->name, name) == 0)
8351 static struct table_type *
8352 table_type_resolve(struct rte_swx_pipeline *p,
8353 const char *recommended_type_name,
8354 enum rte_swx_table_match_type match_type)
8356 struct table_type *elem;
8358 /* Only consider the recommended type if the match type is correct. */
8359 if (recommended_type_name)
8360 TAILQ_FOREACH(elem, &p->table_types, node)
8361 if (!strcmp(elem->name, recommended_type_name) &&
8362 (elem->match_type == match_type))
8365 /* Ignore the recommended type and get the first element with this match
8368 TAILQ_FOREACH(elem, &p->table_types, node)
8369 if (elem->match_type == match_type)
8375 static struct table *
8376 table_find(struct rte_swx_pipeline *p, const char *name)
8380 TAILQ_FOREACH(elem, &p->tables, node)
8381 if (strcmp(elem->name, name) == 0)
8387 static struct table *
8388 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8390 struct table *table = NULL;
8392 TAILQ_FOREACH(table, &p->tables, node)
8393 if (table->id == id)
8400 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8402 enum rte_swx_table_match_type match_type,
8403 struct rte_swx_table_ops *ops)
8405 struct table_type *elem;
8409 CHECK_NAME(name, EINVAL);
8410 CHECK(!table_type_find(p, name), EEXIST);
8413 CHECK(ops->create, EINVAL);
8414 CHECK(ops->lkp, EINVAL);
8415 CHECK(ops->free, EINVAL);
8417 /* Node allocation. */
8418 elem = calloc(1, sizeof(struct table_type));
8419 CHECK(elem, ENOMEM);
8421 /* Node initialization. */
8422 strcpy(elem->name, name);
8423 elem->match_type = match_type;
8424 memcpy(&elem->ops, ops, sizeof(*ops));
8426 /* Node add to tailq. */
8427 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8432 static enum rte_swx_table_match_type
8433 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8438 for (i = 0; i < n_fields; i++)
8439 if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
8443 return RTE_SWX_TABLE_MATCH_EXACT;
8445 if ((i == n_fields - 1) &&
8446 (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
8447 return RTE_SWX_TABLE_MATCH_LPM;
8449 return RTE_SWX_TABLE_MATCH_WILDCARD;
8453 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8455 struct rte_swx_pipeline_table_params *params,
8456 const char *recommended_table_type_name,
8460 struct table_type *type;
8462 struct action *default_action;
8463 struct header *header = NULL;
8465 uint32_t offset_prev = 0, action_data_size_max = 0, i;
8469 CHECK_NAME(name, EINVAL);
8470 CHECK(!table_find(p, name), EEXIST);
8472 CHECK(params, EINVAL);
8475 CHECK(!params->n_fields || params->fields, EINVAL);
8476 for (i = 0; i < params->n_fields; i++) {
8477 struct rte_swx_match_field_params *field = ¶ms->fields[i];
8479 struct field *hf, *mf;
8482 CHECK_NAME(field->name, EINVAL);
8484 hf = header_field_parse(p, field->name, &h);
8485 mf = metadata_field_parse(p, field->name);
8486 CHECK(hf || mf, EINVAL);
8488 offset = hf ? hf->offset : mf->offset;
8491 is_header = hf ? 1 : 0;
8492 header = hf ? h : NULL;
8493 offset_prev = offset;
8498 CHECK((is_header && hf && (h->id == header->id)) ||
8499 (!is_header && mf), EINVAL);
8501 CHECK(offset > offset_prev, EINVAL);
8502 offset_prev = offset;
8505 /* Action checks. */
8506 CHECK(params->n_actions, EINVAL);
8507 CHECK(params->action_names, EINVAL);
8508 for (i = 0; i < params->n_actions; i++) {
8509 const char *action_name = params->action_names[i];
8511 uint32_t action_data_size;
8513 CHECK_NAME(action_name, EINVAL);
8515 a = action_find(p, action_name);
8518 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8519 if (action_data_size > action_data_size_max)
8520 action_data_size_max = action_data_size;
8523 CHECK_NAME(params->default_action_name, EINVAL);
8524 for (i = 0; i < p->n_actions; i++)
8525 if (!strcmp(params->action_names[i],
8526 params->default_action_name))
8528 CHECK(i < params->n_actions, EINVAL);
8529 default_action = action_find(p, params->default_action_name);
8530 CHECK((default_action->st && params->default_action_data) ||
8531 !params->default_action_data, EINVAL);
8533 /* Table type checks. */
8534 if (recommended_table_type_name)
8535 CHECK_NAME(recommended_table_type_name, EINVAL);
8537 if (params->n_fields) {
8538 enum rte_swx_table_match_type match_type;
8540 match_type = table_match_type_resolve(params->fields,
8542 type = table_type_resolve(p,
8543 recommended_table_type_name,
8545 CHECK(type, EINVAL);
8550 /* Memory allocation. */
8551 t = calloc(1, sizeof(struct table));
8554 t->fields = calloc(params->n_fields, sizeof(struct match_field));
8560 t->actions = calloc(params->n_actions, sizeof(struct action *));
8567 if (action_data_size_max) {
8568 t->default_action_data = calloc(1, action_data_size_max);
8569 if (!t->default_action_data) {
8577 /* Node initialization. */
8578 strcpy(t->name, name);
8579 if (args && args[0])
8580 strcpy(t->args, args);
8583 for (i = 0; i < params->n_fields; i++) {
8584 struct rte_swx_match_field_params *field = ¶ms->fields[i];
8585 struct match_field *f = &t->fields[i];
8587 f->match_type = field->match_type;
8588 f->field = is_header ?
8589 header_field_parse(p, field->name, NULL) :
8590 metadata_field_parse(p, field->name);
8592 t->n_fields = params->n_fields;
8593 t->is_header = is_header;
8596 for (i = 0; i < params->n_actions; i++)
8597 t->actions[i] = action_find(p, params->action_names[i]);
8598 t->default_action = default_action;
8599 if (default_action->st)
8600 memcpy(t->default_action_data,
8601 params->default_action_data,
8602 default_action->st->n_bits / 8);
8603 t->n_actions = params->n_actions;
8604 t->default_action_is_const = params->default_action_is_const;
8605 t->action_data_size_max = action_data_size_max;
8608 t->id = p->n_tables;
8610 /* Node add to tailq. */
8611 TAILQ_INSERT_TAIL(&p->tables, t, node);
8617 static struct rte_swx_table_params *
8618 table_params_get(struct table *table)
8620 struct rte_swx_table_params *params;
8621 struct field *first, *last;
8623 uint32_t key_size, key_offset, action_data_size, i;
8625 /* Memory allocation. */
8626 params = calloc(1, sizeof(struct rte_swx_table_params));
8630 /* Key offset and size. */
8631 first = table->fields[0].field;
8632 last = table->fields[table->n_fields - 1].field;
8633 key_offset = first->offset / 8;
8634 key_size = (last->offset + last->n_bits - first->offset) / 8;
8636 /* Memory allocation. */
8637 key_mask = calloc(1, key_size);
8644 for (i = 0; i < table->n_fields; i++) {
8645 struct field *f = table->fields[i].field;
8646 uint32_t start = (f->offset - first->offset) / 8;
8647 size_t size = f->n_bits / 8;
8649 memset(&key_mask[start], 0xFF, size);
8652 /* Action data size. */
8653 action_data_size = 0;
8654 for (i = 0; i < table->n_actions; i++) {
8655 struct action *action = table->actions[i];
8656 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8658 if (ads > action_data_size)
8659 action_data_size = ads;
8663 params->match_type = table->type->match_type;
8664 params->key_size = key_size;
8665 params->key_offset = key_offset;
8666 params->key_mask0 = key_mask;
8667 params->action_data_size = action_data_size;
8668 params->n_keys_max = table->size;
8674 table_params_free(struct rte_swx_table_params *params)
8679 free(params->key_mask0);
8684 table_state_build(struct rte_swx_pipeline *p)
8686 struct table *table;
8688 p->table_state = calloc(p->n_tables,
8689 sizeof(struct rte_swx_table_state));
8690 CHECK(p->table_state, ENOMEM);
8692 TAILQ_FOREACH(table, &p->tables, node) {
8693 struct rte_swx_table_state *ts = &p->table_state[table->id];
8696 struct rte_swx_table_params *params;
8699 params = table_params_get(table);
8700 CHECK(params, ENOMEM);
8702 ts->obj = table->type->ops.create(params,
8707 table_params_free(params);
8708 CHECK(ts->obj, ENODEV);
8711 /* ts->default_action_data. */
8712 if (table->action_data_size_max) {
8713 ts->default_action_data =
8714 malloc(table->action_data_size_max);
8715 CHECK(ts->default_action_data, ENOMEM);
8717 memcpy(ts->default_action_data,
8718 table->default_action_data,
8719 table->action_data_size_max);
8722 /* ts->default_action_id. */
8723 ts->default_action_id = table->default_action->id;
8730 table_state_build_free(struct rte_swx_pipeline *p)
8734 if (!p->table_state)
8737 for (i = 0; i < p->n_tables; i++) {
8738 struct rte_swx_table_state *ts = &p->table_state[i];
8739 struct table *table = table_find_by_id(p, i);
8742 if (table->type && ts->obj)
8743 table->type->ops.free(ts->obj);
8745 /* ts->default_action_data. */
8746 free(ts->default_action_data);
8749 free(p->table_state);
8750 p->table_state = NULL;
8754 table_state_free(struct rte_swx_pipeline *p)
8756 table_state_build_free(p);
8760 table_stub_lkp(void *table __rte_unused,
8761 void *mailbox __rte_unused,
8762 uint8_t **key __rte_unused,
8763 uint64_t *action_id __rte_unused,
8764 uint8_t **action_data __rte_unused,
8768 return 1; /* DONE. */
8772 table_build(struct rte_swx_pipeline *p)
8776 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8777 struct thread *t = &p->threads[i];
8778 struct table *table;
8780 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8781 CHECK(t->tables, ENOMEM);
8783 TAILQ_FOREACH(table, &p->tables, node) {
8784 struct table_runtime *r = &t->tables[table->id];
8789 size = table->type->ops.mailbox_size_get();
8792 r->func = table->type->ops.lkp;
8796 r->mailbox = calloc(1, size);
8797 CHECK(r->mailbox, ENOMEM);
8801 r->key = table->is_header ?
8802 &t->structs[table->header->struct_id] :
8803 &t->structs[p->metadata_struct_id];
8805 r->func = table_stub_lkp;
8814 table_build_free(struct rte_swx_pipeline *p)
8818 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8819 struct thread *t = &p->threads[i];
8825 for (j = 0; j < p->n_tables; j++) {
8826 struct table_runtime *r = &t->tables[j];
8837 table_free(struct rte_swx_pipeline *p)
8839 table_build_free(p);
8845 elem = TAILQ_FIRST(&p->tables);
8849 TAILQ_REMOVE(&p->tables, elem, node);
8851 free(elem->actions);
8852 free(elem->default_action_data);
8858 struct table_type *elem;
8860 elem = TAILQ_FIRST(&p->table_types);
8864 TAILQ_REMOVE(&p->table_types, elem, node);
8872 static struct regarray *
8873 regarray_find(struct rte_swx_pipeline *p, const char *name)
8875 struct regarray *elem;
8877 TAILQ_FOREACH(elem, &p->regarrays, node)
8878 if (!strcmp(elem->name, name))
8884 static struct regarray *
8885 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8887 struct regarray *elem = NULL;
8889 TAILQ_FOREACH(elem, &p->regarrays, node)
8897 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
8906 CHECK_NAME(name, EINVAL);
8907 CHECK(!regarray_find(p, name), EEXIST);
8909 CHECK(size, EINVAL);
8910 size = rte_align32pow2(size);
8912 /* Memory allocation. */
8913 r = calloc(1, sizeof(struct regarray));
8916 /* Node initialization. */
8917 strcpy(r->name, name);
8918 r->init_val = init_val;
8920 r->id = p->n_regarrays;
8922 /* Node add to tailq. */
8923 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
8930 regarray_build(struct rte_swx_pipeline *p)
8932 struct regarray *regarray;
8934 if (!p->n_regarrays)
8937 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
8938 CHECK(p->regarray_runtime, ENOMEM);
8940 TAILQ_FOREACH(regarray, &p->regarrays, node) {
8941 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
8944 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
8945 RTE_CACHE_LINE_SIZE,
8947 CHECK(r->regarray, ENOMEM);
8949 if (regarray->init_val)
8950 for (i = 0; i < regarray->size; i++)
8951 r->regarray[i] = regarray->init_val;
8953 r->size_mask = regarray->size - 1;
8960 regarray_build_free(struct rte_swx_pipeline *p)
8964 if (!p->regarray_runtime)
8967 for (i = 0; i < p->n_regarrays; i++) {
8968 struct regarray *regarray = regarray_find_by_id(p, i);
8969 struct regarray_runtime *r = &p->regarray_runtime[i];
8971 env_free(r->regarray, regarray->size * sizeof(uint64_t));
8974 free(p->regarray_runtime);
8975 p->regarray_runtime = NULL;
8979 regarray_free(struct rte_swx_pipeline *p)
8981 regarray_build_free(p);
8984 struct regarray *elem;
8986 elem = TAILQ_FIRST(&p->regarrays);
8990 TAILQ_REMOVE(&p->regarrays, elem, node);
8998 static struct meter_profile *
8999 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9001 struct meter_profile *elem;
9003 TAILQ_FOREACH(elem, &p->meter_profiles, node)
9004 if (!strcmp(elem->name, name))
9010 static struct metarray *
9011 metarray_find(struct rte_swx_pipeline *p, const char *name)
9013 struct metarray *elem;
9015 TAILQ_FOREACH(elem, &p->metarrays, node)
9016 if (!strcmp(elem->name, name))
9022 static struct metarray *
9023 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9025 struct metarray *elem = NULL;
9027 TAILQ_FOREACH(elem, &p->metarrays, node)
9035 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9043 CHECK_NAME(name, EINVAL);
9044 CHECK(!metarray_find(p, name), EEXIST);
9046 CHECK(size, EINVAL);
9047 size = rte_align32pow2(size);
9049 /* Memory allocation. */
9050 m = calloc(1, sizeof(struct metarray));
9053 /* Node initialization. */
9054 strcpy(m->name, name);
9056 m->id = p->n_metarrays;
9058 /* Node add to tailq. */
9059 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9065 struct meter_profile meter_profile_default = {
9074 .cir_bytes_per_period = 1,
9076 .pir_bytes_per_period = 1,
9083 meter_init(struct meter *m)
9085 memset(m, 0, sizeof(struct meter));
9086 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9087 m->profile = &meter_profile_default;
9088 m->color_mask = RTE_COLOR_GREEN;
9090 meter_profile_default.n_users++;
9094 metarray_build(struct rte_swx_pipeline *p)
9098 if (!p->n_metarrays)
9101 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9102 CHECK(p->metarray_runtime, ENOMEM);
9104 TAILQ_FOREACH(m, &p->metarrays, node) {
9105 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9108 r->metarray = env_malloc(m->size * sizeof(struct meter),
9109 RTE_CACHE_LINE_SIZE,
9111 CHECK(r->metarray, ENOMEM);
9113 for (i = 0; i < m->size; i++)
9114 meter_init(&r->metarray[i]);
9116 r->size_mask = m->size - 1;
9123 metarray_build_free(struct rte_swx_pipeline *p)
9127 if (!p->metarray_runtime)
9130 for (i = 0; i < p->n_metarrays; i++) {
9131 struct metarray *m = metarray_find_by_id(p, i);
9132 struct metarray_runtime *r = &p->metarray_runtime[i];
9134 env_free(r->metarray, m->size * sizeof(struct meter));
9137 free(p->metarray_runtime);
9138 p->metarray_runtime = NULL;
9142 metarray_free(struct rte_swx_pipeline *p)
9144 metarray_build_free(p);
9148 struct metarray *elem;
9150 elem = TAILQ_FIRST(&p->metarrays);
9154 TAILQ_REMOVE(&p->metarrays, elem, node);
9158 /* Meter profiles. */
9160 struct meter_profile *elem;
9162 elem = TAILQ_FIRST(&p->meter_profiles);
9166 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9175 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9177 struct rte_swx_pipeline *pipeline;
9179 /* Check input parameters. */
9182 /* Memory allocation. */
9183 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9184 CHECK(pipeline, ENOMEM);
9186 /* Initialization. */
9187 TAILQ_INIT(&pipeline->struct_types);
9188 TAILQ_INIT(&pipeline->port_in_types);
9189 TAILQ_INIT(&pipeline->ports_in);
9190 TAILQ_INIT(&pipeline->port_out_types);
9191 TAILQ_INIT(&pipeline->ports_out);
9192 TAILQ_INIT(&pipeline->extern_types);
9193 TAILQ_INIT(&pipeline->extern_objs);
9194 TAILQ_INIT(&pipeline->extern_funcs);
9195 TAILQ_INIT(&pipeline->headers);
9196 TAILQ_INIT(&pipeline->actions);
9197 TAILQ_INIT(&pipeline->table_types);
9198 TAILQ_INIT(&pipeline->tables);
9199 TAILQ_INIT(&pipeline->regarrays);
9200 TAILQ_INIT(&pipeline->meter_profiles);
9201 TAILQ_INIT(&pipeline->metarrays);
9203 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9204 pipeline->numa_node = numa_node;
9211 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9216 free(p->instructions);
9220 table_state_free(p);
9225 extern_func_free(p);
9235 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9236 const char **instructions,
9237 uint32_t n_instructions)
9242 err = instruction_config(p, NULL, instructions, n_instructions);
9246 /* Thread instruction pointer reset. */
9247 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9248 struct thread *t = &p->threads[i];
9250 thread_ip_reset(p, t);
9257 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9262 CHECK(p->build_done == 0, EEXIST);
9264 status = port_in_build(p);
9268 status = port_out_build(p);
9272 status = struct_build(p);
9276 status = extern_obj_build(p);
9280 status = extern_func_build(p);
9284 status = header_build(p);
9288 status = metadata_build(p);
9292 status = action_build(p);
9296 status = table_build(p);
9300 status = table_state_build(p);
9304 status = regarray_build(p);
9308 status = metarray_build(p);
9316 metarray_build_free(p);
9317 regarray_build_free(p);
9318 table_state_build_free(p);
9319 table_build_free(p);
9320 action_build_free(p);
9321 metadata_build_free(p);
9322 header_build_free(p);
9323 extern_func_build_free(p);
9324 extern_obj_build_free(p);
9325 port_out_build_free(p);
9326 port_in_build_free(p);
9327 struct_build_free(p);
9333 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9337 for (i = 0; i < n_instructions; i++)
9342 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9346 for (i = 0; i < p->n_ports_out; i++) {
9347 struct port_out_runtime *port = &p->out[i];
9350 port->flush(port->obj);
9358 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9359 struct rte_swx_ctl_pipeline_info *pipeline)
9361 struct action *action;
9362 struct table *table;
9363 uint32_t n_actions = 0, n_tables = 0;
9365 if (!p || !pipeline)
9368 TAILQ_FOREACH(action, &p->actions, node)
9371 TAILQ_FOREACH(table, &p->tables, node)
9374 pipeline->n_ports_in = p->n_ports_in;
9375 pipeline->n_ports_out = p->n_ports_out;
9376 pipeline->n_actions = n_actions;
9377 pipeline->n_tables = n_tables;
9378 pipeline->n_regarrays = p->n_regarrays;
9379 pipeline->n_metarrays = p->n_metarrays;
9385 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9387 if (!p || !numa_node)
9390 *numa_node = p->numa_node;
9395 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9397 struct rte_swx_ctl_action_info *action)
9399 struct action *a = NULL;
9401 if (!p || (action_id >= p->n_actions) || !action)
9404 a = action_find_by_id(p, action_id);
9408 strcpy(action->name, a->name);
9409 action->n_args = a->st ? a->st->n_fields : 0;
9414 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9416 uint32_t action_arg_id,
9417 struct rte_swx_ctl_action_arg_info *action_arg)
9419 struct action *a = NULL;
9420 struct field *arg = NULL;
9422 if (!p || (action_id >= p->n_actions) || !action_arg)
9425 a = action_find_by_id(p, action_id);
9426 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9429 arg = &a->st->fields[action_arg_id];
9430 strcpy(action_arg->name, arg->name);
9431 action_arg->n_bits = arg->n_bits;
9437 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9439 struct rte_swx_ctl_table_info *table)
9441 struct table *t = NULL;
9446 t = table_find_by_id(p, table_id);
9450 strcpy(table->name, t->name);
9451 strcpy(table->args, t->args);
9452 table->n_match_fields = t->n_fields;
9453 table->n_actions = t->n_actions;
9454 table->default_action_is_const = t->default_action_is_const;
9455 table->size = t->size;
9460 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9462 uint32_t match_field_id,
9463 struct rte_swx_ctl_table_match_field_info *match_field)
9466 struct match_field *f;
9468 if (!p || (table_id >= p->n_tables) || !match_field)
9471 t = table_find_by_id(p, table_id);
9472 if (!t || (match_field_id >= t->n_fields))
9475 f = &t->fields[match_field_id];
9476 match_field->match_type = f->match_type;
9477 match_field->is_header = t->is_header;
9478 match_field->n_bits = f->field->n_bits;
9479 match_field->offset = f->field->offset;
9485 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9487 uint32_t table_action_id,
9488 struct rte_swx_ctl_table_action_info *table_action)
9492 if (!p || (table_id >= p->n_tables) || !table_action)
9495 t = table_find_by_id(p, table_id);
9496 if (!t || (table_action_id >= t->n_actions))
9499 table_action->action_id = t->actions[table_action_id]->id;
9505 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9507 struct rte_swx_table_ops *table_ops,
9512 if (!p || (table_id >= p->n_tables))
9515 t = table_find_by_id(p, table_id);
9521 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9531 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9532 struct rte_swx_table_state **table_state)
9534 if (!p || !table_state || !p->build_done)
9537 *table_state = p->table_state;
9542 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9543 struct rte_swx_table_state *table_state)
9545 if (!p || !table_state || !p->build_done)
9548 p->table_state = table_state;
9553 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9555 struct rte_swx_port_in_stats *stats)
9557 struct port_in *port;
9562 port = port_in_find(p, port_id);
9566 port->type->ops.stats_read(port->obj, stats);
9571 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9573 struct rte_swx_port_out_stats *stats)
9575 struct port_out *port;
9580 port = port_out_find(p, port_id);
9584 port->type->ops.stats_read(port->obj, stats);
9589 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9590 uint32_t regarray_id,
9591 struct rte_swx_ctl_regarray_info *regarray)
9595 if (!p || !regarray)
9598 r = regarray_find_by_id(p, regarray_id);
9602 strcpy(regarray->name, r->name);
9603 regarray->size = r->size;
9608 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9609 const char *regarray_name,
9610 uint32_t regarray_index,
9613 struct regarray *regarray;
9614 struct regarray_runtime *r;
9616 if (!p || !regarray_name || !value)
9619 regarray = regarray_find(p, regarray_name);
9620 if (!regarray || (regarray_index >= regarray->size))
9623 r = &p->regarray_runtime[regarray->id];
9624 *value = r->regarray[regarray_index];
9629 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9630 const char *regarray_name,
9631 uint32_t regarray_index,
9634 struct regarray *regarray;
9635 struct regarray_runtime *r;
9637 if (!p || !regarray_name)
9640 regarray = regarray_find(p, regarray_name);
9641 if (!regarray || (regarray_index >= regarray->size))
9644 r = &p->regarray_runtime[regarray->id];
9645 r->regarray[regarray_index] = value;
9650 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9651 uint32_t metarray_id,
9652 struct rte_swx_ctl_metarray_info *metarray)
9656 if (!p || !metarray)
9659 m = metarray_find_by_id(p, metarray_id);
9663 strcpy(metarray->name, m->name);
9664 metarray->size = m->size;
9669 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9671 struct rte_meter_trtcm_params *params)
9673 struct meter_profile *mp;
9677 CHECK_NAME(name, EINVAL);
9678 CHECK(params, EINVAL);
9679 CHECK(!meter_profile_find(p, name), EEXIST);
9681 /* Node allocation. */
9682 mp = calloc(1, sizeof(struct meter_profile));
9685 /* Node initialization. */
9686 strcpy(mp->name, name);
9687 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9688 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9694 /* Node add to tailq. */
9695 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9701 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9704 struct meter_profile *mp;
9707 CHECK_NAME(name, EINVAL);
9709 mp = meter_profile_find(p, name);
9711 CHECK(!mp->n_users, EBUSY);
9713 /* Remove node from tailq. */
9714 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9721 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9722 const char *metarray_name,
9723 uint32_t metarray_index)
9725 struct meter_profile *mp_old;
9726 struct metarray *metarray;
9727 struct metarray_runtime *metarray_runtime;
9731 CHECK_NAME(metarray_name, EINVAL);
9733 metarray = metarray_find(p, metarray_name);
9734 CHECK(metarray, EINVAL);
9735 CHECK(metarray_index < metarray->size, EINVAL);
9737 metarray_runtime = &p->metarray_runtime[metarray->id];
9738 m = &metarray_runtime->metarray[metarray_index];
9739 mp_old = m->profile;
9749 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9750 const char *metarray_name,
9751 uint32_t metarray_index,
9752 const char *profile_name)
9754 struct meter_profile *mp, *mp_old;
9755 struct metarray *metarray;
9756 struct metarray_runtime *metarray_runtime;
9760 CHECK_NAME(metarray_name, EINVAL);
9762 metarray = metarray_find(p, metarray_name);
9763 CHECK(metarray, EINVAL);
9764 CHECK(metarray_index < metarray->size, EINVAL);
9766 mp = meter_profile_find(p, profile_name);
9769 metarray_runtime = &p->metarray_runtime[metarray->id];
9770 m = &metarray_runtime->metarray[metarray_index];
9771 mp_old = m->profile;
9773 memset(m, 0, sizeof(struct meter));
9774 rte_meter_trtcm_config(&m->m, &mp->profile);
9776 m->color_mask = RTE_COLORS;
9785 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
9786 const char *metarray_name,
9787 uint32_t metarray_index,
9788 struct rte_swx_ctl_meter_stats *stats)
9790 struct metarray *metarray;
9791 struct metarray_runtime *metarray_runtime;
9795 CHECK_NAME(metarray_name, EINVAL);
9797 metarray = metarray_find(p, metarray_name);
9798 CHECK(metarray, EINVAL);
9799 CHECK(metarray_index < metarray->size, EINVAL);
9801 CHECK(stats, EINVAL);
9803 metarray_runtime = &p->metarray_runtime[metarray->id];
9804 m = &metarray_runtime->metarray[metarray_index];
9806 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
9807 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));