1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <arpa/inet.h>
12 #include <rte_common.h>
13 #include <rte_prefetch.h>
14 #include <rte_byteorder.h>
15 #include <rte_cycles.h>
16 #include <rte_meter.h>
18 #include "rte_swx_pipeline.h"
19 #include "rte_swx_ctl.h"
21 #define CHECK(condition, err_code) \
27 #define CHECK_NAME(name, err_code) \
30 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
33 #define CHECK_INSTRUCTION(instr, err_code) \
36 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
37 RTE_SWX_INSTRUCTION_SIZE), \
45 #define TRACE(...) printf(__VA_ARGS__)
53 #define ntoh64(x) rte_be_to_cpu_64(x)
54 #define hton64(x) rte_cpu_to_be_64(x)
56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
58 #include <rte_malloc.h>
61 env_malloc(size_t size, size_t alignment, int numa_node)
63 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
67 env_free(void *start, size_t size __rte_unused)
77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
81 if (numa_available() == -1)
84 start = numa_alloc_onnode(size, numa_node);
88 memset(start, 0, size);
93 env_free(void *start, size_t size)
95 if (numa_available() == -1)
98 numa_free(start, size);
107 char name[RTE_SWX_NAME_SIZE];
113 TAILQ_ENTRY(struct_type) node;
114 char name[RTE_SWX_NAME_SIZE];
115 struct field *fields;
120 TAILQ_HEAD(struct_type_tailq, struct_type);
125 struct port_in_type {
126 TAILQ_ENTRY(port_in_type) node;
127 char name[RTE_SWX_NAME_SIZE];
128 struct rte_swx_port_in_ops ops;
131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
134 TAILQ_ENTRY(port_in) node;
135 struct port_in_type *type;
140 TAILQ_HEAD(port_in_tailq, port_in);
142 struct port_in_runtime {
143 rte_swx_port_in_pkt_rx_t pkt_rx;
150 struct port_out_type {
151 TAILQ_ENTRY(port_out_type) node;
152 char name[RTE_SWX_NAME_SIZE];
153 struct rte_swx_port_out_ops ops;
156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
159 TAILQ_ENTRY(port_out) node;
160 struct port_out_type *type;
165 TAILQ_HEAD(port_out_tailq, port_out);
167 struct port_out_runtime {
168 rte_swx_port_out_pkt_tx_t pkt_tx;
169 rte_swx_port_out_flush_t flush;
176 struct extern_type_member_func {
177 TAILQ_ENTRY(extern_type_member_func) node;
178 char name[RTE_SWX_NAME_SIZE];
179 rte_swx_extern_type_member_func_t func;
183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
186 TAILQ_ENTRY(extern_type) node;
187 char name[RTE_SWX_NAME_SIZE];
188 struct struct_type *mailbox_struct_type;
189 rte_swx_extern_type_constructor_t constructor;
190 rte_swx_extern_type_destructor_t destructor;
191 struct extern_type_member_func_tailq funcs;
195 TAILQ_HEAD(extern_type_tailq, extern_type);
198 TAILQ_ENTRY(extern_obj) node;
199 char name[RTE_SWX_NAME_SIZE];
200 struct extern_type *type;
206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
212 struct extern_obj_runtime {
215 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
222 TAILQ_ENTRY(extern_func) node;
223 char name[RTE_SWX_NAME_SIZE];
224 struct struct_type *mailbox_struct_type;
225 rte_swx_extern_func_t func;
230 TAILQ_HEAD(extern_func_tailq, extern_func);
232 struct extern_func_runtime {
234 rte_swx_extern_func_t func;
241 TAILQ_ENTRY(header) node;
242 char name[RTE_SWX_NAME_SIZE];
243 struct struct_type *st;
248 TAILQ_HEAD(header_tailq, header);
250 struct header_runtime {
254 struct header_out_runtime {
264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
265 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
266 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
267 * when transferred to packet meta-data and in NBO when transferred to packet
271 /* Notation conventions:
272 * -Header field: H = h.header.field (dst/src)
273 * -Meta-data field: M = m.field (dst/src)
274 * -Extern object mailbox field: E = e.field (dst/src)
275 * -Extern function mailbox field: F = f.field (dst/src)
276 * -Table action data field: T = t.field (src only)
277 * -Immediate value: I = 32-bit unsigned value (src only)
280 enum instruction_type {
287 INSTR_TX, /* port_out = M */
288 INSTR_TX_I, /* port_out = I */
290 /* extract h.header */
311 /* validate h.header */
314 /* invalidate h.header */
315 INSTR_HDR_INVALIDATE,
319 * dst = HMEF, src = HMEFTI
321 INSTR_MOV, /* dst = MEF, src = MEFT */
322 INSTR_MOV_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
323 INSTR_MOV_I, /* dst = HMEF, src = I */
325 /* dma h.header t.field
326 * memcpy(h.header, t.field, sizeof(h.header))
339 * dst = HMEF, src = HMEFTI
341 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
342 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
343 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
344 INSTR_ALU_ADD_HH, /* dst = H, src = H */
345 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
346 INSTR_ALU_ADD_HI, /* dst = H, src = I */
350 * dst = HMEF, src = HMEFTI
352 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
353 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
354 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
355 INSTR_ALU_SUB_HH, /* dst = H, src = H */
356 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
357 INSTR_ALU_SUB_HI, /* dst = H, src = I */
360 * dst = dst '+ src[0:1] '+ src[2:3] + ...
361 * dst = H, src = {H, h.header}
363 INSTR_ALU_CKADD_FIELD, /* src = H */
364 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
365 INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
371 INSTR_ALU_CKSUB_FIELD,
375 * dst = HMEF, src = HMEFTI
377 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
378 INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
379 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
383 * dst = HMEF, src = HMEFTI
385 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
386 INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
387 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
391 * dst = HMEF, src = HMEFTI
393 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
394 INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
395 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
399 * dst = HMEF, src = HMEFTI
401 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
402 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
403 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
404 INSTR_ALU_SHL_HH, /* dst = H, src = H */
405 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
406 INSTR_ALU_SHL_HI, /* dst = H, src = I */
410 * dst = HMEF, src = HMEFTI
412 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
413 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
414 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
415 INSTR_ALU_SHR_HH, /* dst = H, src = H */
416 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
417 INSTR_ALU_SHR_HI, /* dst = H, src = I */
419 /* regprefetch REGARRAY index
420 * prefetch REGARRAY[index]
423 INSTR_REGPREFETCH_RH, /* index = H */
424 INSTR_REGPREFETCH_RM, /* index = MEFT */
425 INSTR_REGPREFETCH_RI, /* index = I */
427 /* regrd dst REGARRAY index
428 * dst = REGARRAY[index]
429 * dst = HMEF, index = HMEFTI
431 INSTR_REGRD_HRH, /* dst = H, index = H */
432 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
433 INSTR_REGRD_HRI, /* dst = H, index = I */
434 INSTR_REGRD_MRH, /* dst = MEF, index = H */
435 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
436 INSTR_REGRD_MRI, /* dst = MEF, index = I */
438 /* regwr REGARRAY index src
439 * REGARRAY[index] = src
440 * index = HMEFTI, src = HMEFTI
442 INSTR_REGWR_RHH, /* index = H, src = H */
443 INSTR_REGWR_RHM, /* index = H, src = MEFT */
444 INSTR_REGWR_RHI, /* index = H, src = I */
445 INSTR_REGWR_RMH, /* index = MEFT, src = H */
446 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
447 INSTR_REGWR_RMI, /* index = MEFT, src = I */
448 INSTR_REGWR_RIH, /* index = I, src = H */
449 INSTR_REGWR_RIM, /* index = I, src = MEFT */
450 INSTR_REGWR_RII, /* index = I, src = I */
452 /* regadd REGARRAY index src
453 * REGARRAY[index] += src
454 * index = HMEFTI, src = HMEFTI
456 INSTR_REGADD_RHH, /* index = H, src = H */
457 INSTR_REGADD_RHM, /* index = H, src = MEFT */
458 INSTR_REGADD_RHI, /* index = H, src = I */
459 INSTR_REGADD_RMH, /* index = MEFT, src = H */
460 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
461 INSTR_REGADD_RMI, /* index = MEFT, src = I */
462 INSTR_REGADD_RIH, /* index = I, src = H */
463 INSTR_REGADD_RIM, /* index = I, src = MEFT */
464 INSTR_REGADD_RII, /* index = I, src = I */
466 /* metprefetch METARRAY index
467 * prefetch METARRAY[index]
470 INSTR_METPREFETCH_H, /* index = H */
471 INSTR_METPREFETCH_M, /* index = MEFT */
472 INSTR_METPREFETCH_I, /* index = I */
474 /* meter METARRAY index length color_in color_out
475 * color_out = meter(METARRAY[index], length, color_in)
476 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
478 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
479 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
480 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
481 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
482 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
483 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
484 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
485 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
486 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
487 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
488 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
489 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
494 /* extern e.obj.func */
505 /* jmpv LABEL h.header
506 * Jump if header is valid
510 /* jmpnv LABEL h.header
511 * Jump if header is invalid
516 * Jump if table lookup hit
521 * Jump if table lookup miss
528 INSTR_JMP_ACTION_HIT,
530 /* jmpna LABEL ACTION
531 * Jump if action not run
533 INSTR_JMP_ACTION_MISS,
536 * Jump is a is equal to b
537 * a = HMEFT, b = HMEFTI
539 INSTR_JMP_EQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
540 INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
541 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
544 * Jump is a is not equal to b
545 * a = HMEFT, b = HMEFTI
547 INSTR_JMP_NEQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
548 INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
549 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
552 * Jump if a is less than b
553 * a = HMEFT, b = HMEFTI
555 INSTR_JMP_LT, /* a = MEF, b = MEF */
556 INSTR_JMP_LT_MH, /* a = MEF, b = H */
557 INSTR_JMP_LT_HM, /* a = H, b = MEF */
558 INSTR_JMP_LT_HH, /* a = H, b = H */
559 INSTR_JMP_LT_MI, /* a = MEF, b = I */
560 INSTR_JMP_LT_HI, /* a = H, b = I */
563 * Jump if a is greater than b
564 * a = HMEFT, b = HMEFTI
566 INSTR_JMP_GT, /* a = MEF, b = MEF */
567 INSTR_JMP_GT_MH, /* a = MEF, b = H */
568 INSTR_JMP_GT_HM, /* a = H, b = MEF */
569 INSTR_JMP_GT_HH, /* a = H, b = H */
570 INSTR_JMP_GT_MI, /* a = MEF, b = I */
571 INSTR_JMP_GT_HI, /* a = H, b = I */
579 struct instr_operand {
600 uint8_t header_id[8];
601 uint8_t struct_id[8];
606 struct instr_hdr_validity {
614 struct instr_extern_obj {
619 struct instr_extern_func {
623 struct instr_dst_src {
624 struct instr_operand dst;
626 struct instr_operand src;
631 struct instr_regarray {
636 struct instr_operand idx;
641 struct instr_operand dstsrc;
651 struct instr_operand idx;
655 struct instr_operand length;
658 struct instr_operand color_in;
659 uint32_t color_in_val;
662 struct instr_operand color_out;
667 uint8_t header_id[8];
668 uint8_t struct_id[8];
679 struct instruction *ip;
682 struct instr_operand a;
688 struct instr_operand b;
694 enum instruction_type type;
697 struct instr_hdr_validity valid;
698 struct instr_dst_src mov;
699 struct instr_regarray regarray;
700 struct instr_meter meter;
701 struct instr_dma dma;
702 struct instr_dst_src alu;
703 struct instr_table table;
704 struct instr_extern_obj ext_obj;
705 struct instr_extern_func ext_func;
706 struct instr_jmp jmp;
710 struct instruction_data {
711 char label[RTE_SWX_NAME_SIZE];
712 char jmp_label[RTE_SWX_NAME_SIZE];
713 uint32_t n_users; /* user = jmp instruction to this instruction. */
721 TAILQ_ENTRY(action) node;
722 char name[RTE_SWX_NAME_SIZE];
723 struct struct_type *st;
724 struct instruction *instructions;
725 uint32_t n_instructions;
729 TAILQ_HEAD(action_tailq, action);
735 TAILQ_ENTRY(table_type) node;
736 char name[RTE_SWX_NAME_SIZE];
737 enum rte_swx_table_match_type match_type;
738 struct rte_swx_table_ops ops;
741 TAILQ_HEAD(table_type_tailq, table_type);
744 enum rte_swx_table_match_type match_type;
749 TAILQ_ENTRY(table) node;
750 char name[RTE_SWX_NAME_SIZE];
751 char args[RTE_SWX_NAME_SIZE];
752 struct table_type *type; /* NULL when n_fields == 0. */
755 struct match_field *fields;
757 struct header *header; /* Only valid when n_fields > 0. */
760 struct action **actions;
761 struct action *default_action;
762 uint8_t *default_action_data;
764 int default_action_is_const;
765 uint32_t action_data_size_max;
771 TAILQ_HEAD(table_tailq, table);
773 struct table_runtime {
774 rte_swx_table_lookup_t func;
779 struct table_statistics {
780 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
781 uint64_t *n_pkts_action;
788 TAILQ_ENTRY(regarray) node;
789 char name[RTE_SWX_NAME_SIZE];
795 TAILQ_HEAD(regarray_tailq, regarray);
797 struct regarray_runtime {
805 struct meter_profile {
806 TAILQ_ENTRY(meter_profile) node;
807 char name[RTE_SWX_NAME_SIZE];
808 struct rte_meter_trtcm_params params;
809 struct rte_meter_trtcm_profile profile;
813 TAILQ_HEAD(meter_profile_tailq, meter_profile);
816 TAILQ_ENTRY(metarray) node;
817 char name[RTE_SWX_NAME_SIZE];
822 TAILQ_HEAD(metarray_tailq, metarray);
825 struct rte_meter_trtcm m;
826 struct meter_profile *profile;
827 enum rte_color color_mask;
830 uint64_t n_pkts[RTE_COLORS];
831 uint64_t n_bytes[RTE_COLORS];
834 struct metarray_runtime {
835 struct meter *metarray;
844 struct rte_swx_pkt pkt;
850 /* Packet headers. */
851 struct header_runtime *headers; /* Extracted or generated headers. */
852 struct header_out_runtime *headers_out; /* Emitted headers. */
853 uint8_t *header_storage;
854 uint8_t *header_out_storage;
855 uint64_t valid_headers;
856 uint32_t n_headers_out;
858 /* Packet meta-data. */
862 struct table_runtime *tables;
863 struct rte_swx_table_state *table_state;
865 int hit; /* 0 = Miss, 1 = Hit. */
867 /* Extern objects and functions. */
868 struct extern_obj_runtime *extern_objs;
869 struct extern_func_runtime *extern_funcs;
872 struct instruction *ip;
873 struct instruction *ret;
876 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
877 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
878 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
880 #define HEADER_VALID(thread, header_id) \
881 MASK64_BIT_GET((thread)->valid_headers, header_id)
883 #define ALU(thread, ip, operator) \
885 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
886 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
887 uint64_t dst64 = *dst64_ptr; \
888 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
889 uint64_t dst = dst64 & dst64_mask; \
891 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
892 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
893 uint64_t src64 = *src64_ptr; \
894 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
895 uint64_t src = src64 & src64_mask; \
897 uint64_t result = dst operator src; \
899 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
902 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
904 #define ALU_S(thread, ip, operator) \
906 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
907 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
908 uint64_t dst64 = *dst64_ptr; \
909 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
910 uint64_t dst = dst64 & dst64_mask; \
912 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
913 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
914 uint64_t src64 = *src64_ptr; \
915 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
917 uint64_t result = dst operator src; \
919 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
924 #define ALU_HM(thread, ip, operator) \
926 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
927 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
928 uint64_t dst64 = *dst64_ptr; \
929 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
930 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
932 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
933 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
934 uint64_t src64 = *src64_ptr; \
935 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
936 uint64_t src = src64 & src64_mask; \
938 uint64_t result = dst operator src; \
939 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
941 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
944 #define ALU_HH(thread, ip, operator) \
946 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
947 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
948 uint64_t dst64 = *dst64_ptr; \
949 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
950 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
952 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
953 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
954 uint64_t src64 = *src64_ptr; \
955 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
957 uint64_t result = dst operator src; \
958 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
960 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
972 #define ALU_I(thread, ip, operator) \
974 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
975 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
976 uint64_t dst64 = *dst64_ptr; \
977 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
978 uint64_t dst = dst64 & dst64_mask; \
980 uint64_t src = (ip)->alu.src_val; \
982 uint64_t result = dst operator src; \
984 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
989 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
991 #define ALU_HI(thread, ip, operator) \
993 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
994 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
995 uint64_t dst64 = *dst64_ptr; \
996 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
997 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
999 uint64_t src = (ip)->alu.src_val; \
1001 uint64_t result = dst operator src; \
1002 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1004 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1009 #define ALU_HI ALU_I
1013 #define MOV(thread, ip) \
1015 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1016 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1017 uint64_t dst64 = *dst64_ptr; \
1018 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1020 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1021 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1022 uint64_t src64 = *src64_ptr; \
1023 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1024 uint64_t src = src64 & src64_mask; \
1026 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1029 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1031 #define MOV_S(thread, ip) \
1033 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1034 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1035 uint64_t dst64 = *dst64_ptr; \
1036 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1038 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1039 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1040 uint64_t src64 = *src64_ptr; \
1041 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1043 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1052 #define MOV_I(thread, ip) \
1054 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1055 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1056 uint64_t dst64 = *dst64_ptr; \
1057 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1059 uint64_t src = (ip)->mov.src_val; \
1061 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1064 #define JMP_CMP(thread, ip, operator) \
1066 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1067 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1068 uint64_t a64 = *a64_ptr; \
1069 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1070 uint64_t a = a64 & a64_mask; \
1072 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1073 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1074 uint64_t b64 = *b64_ptr; \
1075 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1076 uint64_t b = b64 & b64_mask; \
1078 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1081 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1083 #define JMP_CMP_S(thread, ip, operator) \
1085 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1086 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1087 uint64_t a64 = *a64_ptr; \
1088 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1089 uint64_t a = a64 & a64_mask; \
1091 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1092 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1093 uint64_t b64 = *b64_ptr; \
1094 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1096 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1099 #define JMP_CMP_MH JMP_CMP_S
1101 #define JMP_CMP_HM(thread, ip, operator) \
1103 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1104 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1105 uint64_t a64 = *a64_ptr; \
1106 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1108 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1109 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1110 uint64_t b64 = *b64_ptr; \
1111 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1112 uint64_t b = b64 & b64_mask; \
1114 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1117 #define JMP_CMP_HH(thread, ip, operator) \
1119 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1120 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1121 uint64_t a64 = *a64_ptr; \
1122 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1124 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1125 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1126 uint64_t b64 = *b64_ptr; \
1127 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1129 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1134 #define JMP_CMP_S JMP_CMP
1135 #define JMP_CMP_MH JMP_CMP
1136 #define JMP_CMP_HM JMP_CMP
1137 #define JMP_CMP_HH JMP_CMP
1141 #define JMP_CMP_I(thread, ip, operator) \
1143 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1144 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1145 uint64_t a64 = *a64_ptr; \
1146 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1147 uint64_t a = a64 & a64_mask; \
1149 uint64_t b = (ip)->jmp.b_val; \
1151 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1154 #define JMP_CMP_MI JMP_CMP_I
1156 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1158 #define JMP_CMP_HI(thread, ip, operator) \
1160 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1161 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1162 uint64_t a64 = *a64_ptr; \
1163 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1165 uint64_t b = (ip)->jmp.b_val; \
1167 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1172 #define JMP_CMP_HI JMP_CMP_I
1176 #define METADATA_READ(thread, offset, n_bits) \
1178 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1179 uint64_t m64 = *m64_ptr; \
1180 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1184 #define METADATA_WRITE(thread, offset, n_bits, value) \
1186 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1187 uint64_t m64 = *m64_ptr; \
1188 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1190 uint64_t m_new = value; \
1192 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1195 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1196 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1199 struct rte_swx_pipeline {
1200 struct struct_type_tailq struct_types;
1201 struct port_in_type_tailq port_in_types;
1202 struct port_in_tailq ports_in;
1203 struct port_out_type_tailq port_out_types;
1204 struct port_out_tailq ports_out;
1205 struct extern_type_tailq extern_types;
1206 struct extern_obj_tailq extern_objs;
1207 struct extern_func_tailq extern_funcs;
1208 struct header_tailq headers;
1209 struct struct_type *metadata_st;
1210 uint32_t metadata_struct_id;
1211 struct action_tailq actions;
1212 struct table_type_tailq table_types;
1213 struct table_tailq tables;
1214 struct regarray_tailq regarrays;
1215 struct meter_profile_tailq meter_profiles;
1216 struct metarray_tailq metarrays;
1218 struct port_in_runtime *in;
1219 struct port_out_runtime *out;
1220 struct instruction **action_instructions;
1221 struct rte_swx_table_state *table_state;
1222 struct table_statistics *table_stats;
1223 struct regarray_runtime *regarray_runtime;
1224 struct metarray_runtime *metarray_runtime;
1225 struct instruction *instructions;
1226 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1229 uint32_t n_ports_in;
1230 uint32_t n_ports_out;
1231 uint32_t n_extern_objs;
1232 uint32_t n_extern_funcs;
1235 uint32_t n_regarrays;
1236 uint32_t n_metarrays;
1240 uint32_t n_instructions;
1248 static struct struct_type *
1249 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1251 struct struct_type *elem;
1253 TAILQ_FOREACH(elem, &p->struct_types, node)
1254 if (strcmp(elem->name, name) == 0)
1260 static struct field *
1261 struct_type_field_find(struct struct_type *st, const char *name)
1265 for (i = 0; i < st->n_fields; i++) {
1266 struct field *f = &st->fields[i];
1268 if (strcmp(f->name, name) == 0)
1276 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1278 struct rte_swx_field_params *fields,
1281 struct struct_type *st;
1285 CHECK_NAME(name, EINVAL);
1286 CHECK(fields, EINVAL);
1287 CHECK(n_fields, EINVAL);
1289 for (i = 0; i < n_fields; i++) {
1290 struct rte_swx_field_params *f = &fields[i];
1293 CHECK_NAME(f->name, EINVAL);
1294 CHECK(f->n_bits, EINVAL);
1295 CHECK(f->n_bits <= 64, EINVAL);
1296 CHECK((f->n_bits & 7) == 0, EINVAL);
1298 for (j = 0; j < i; j++) {
1299 struct rte_swx_field_params *f_prev = &fields[j];
1301 CHECK(strcmp(f->name, f_prev->name), EINVAL);
1305 CHECK(!struct_type_find(p, name), EEXIST);
1307 /* Node allocation. */
1308 st = calloc(1, sizeof(struct struct_type));
1311 st->fields = calloc(n_fields, sizeof(struct field));
1317 /* Node initialization. */
1318 strcpy(st->name, name);
1319 for (i = 0; i < n_fields; i++) {
1320 struct field *dst = &st->fields[i];
1321 struct rte_swx_field_params *src = &fields[i];
1323 strcpy(dst->name, src->name);
1324 dst->n_bits = src->n_bits;
1325 dst->offset = st->n_bits;
1327 st->n_bits += src->n_bits;
1329 st->n_fields = n_fields;
1331 /* Node add to tailq. */
1332 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1338 struct_build(struct rte_swx_pipeline *p)
1342 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1343 struct thread *t = &p->threads[i];
1345 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1346 CHECK(t->structs, ENOMEM);
1353 struct_build_free(struct rte_swx_pipeline *p)
1357 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1358 struct thread *t = &p->threads[i];
1366 struct_free(struct rte_swx_pipeline *p)
1368 struct_build_free(p);
1372 struct struct_type *elem;
1374 elem = TAILQ_FIRST(&p->struct_types);
1378 TAILQ_REMOVE(&p->struct_types, elem, node);
1387 static struct port_in_type *
1388 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1390 struct port_in_type *elem;
1395 TAILQ_FOREACH(elem, &p->port_in_types, node)
1396 if (strcmp(elem->name, name) == 0)
1403 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1405 struct rte_swx_port_in_ops *ops)
1407 struct port_in_type *elem;
1410 CHECK_NAME(name, EINVAL);
1412 CHECK(ops->create, EINVAL);
1413 CHECK(ops->free, EINVAL);
1414 CHECK(ops->pkt_rx, EINVAL);
1415 CHECK(ops->stats_read, EINVAL);
1417 CHECK(!port_in_type_find(p, name), EEXIST);
1419 /* Node allocation. */
1420 elem = calloc(1, sizeof(struct port_in_type));
1421 CHECK(elem, ENOMEM);
1423 /* Node initialization. */
1424 strcpy(elem->name, name);
1425 memcpy(&elem->ops, ops, sizeof(*ops));
1427 /* Node add to tailq. */
1428 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1433 static struct port_in *
1434 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1436 struct port_in *port;
1438 TAILQ_FOREACH(port, &p->ports_in, node)
1439 if (port->id == port_id)
1446 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1448 const char *port_type_name,
1451 struct port_in_type *type = NULL;
1452 struct port_in *port = NULL;
1457 CHECK(!port_in_find(p, port_id), EINVAL);
1459 CHECK_NAME(port_type_name, EINVAL);
1460 type = port_in_type_find(p, port_type_name);
1461 CHECK(type, EINVAL);
1463 obj = type->ops.create(args);
1466 /* Node allocation. */
1467 port = calloc(1, sizeof(struct port_in));
1468 CHECK(port, ENOMEM);
1470 /* Node initialization. */
1475 /* Node add to tailq. */
1476 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1477 if (p->n_ports_in < port_id + 1)
1478 p->n_ports_in = port_id + 1;
1484 port_in_build(struct rte_swx_pipeline *p)
1486 struct port_in *port;
1489 CHECK(p->n_ports_in, EINVAL);
1490 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1492 for (i = 0; i < p->n_ports_in; i++)
1493 CHECK(port_in_find(p, i), EINVAL);
1495 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1496 CHECK(p->in, ENOMEM);
1498 TAILQ_FOREACH(port, &p->ports_in, node) {
1499 struct port_in_runtime *in = &p->in[port->id];
1501 in->pkt_rx = port->type->ops.pkt_rx;
1502 in->obj = port->obj;
1509 port_in_build_free(struct rte_swx_pipeline *p)
1516 port_in_free(struct rte_swx_pipeline *p)
1518 port_in_build_free(p);
1522 struct port_in *port;
1524 port = TAILQ_FIRST(&p->ports_in);
1528 TAILQ_REMOVE(&p->ports_in, port, node);
1529 port->type->ops.free(port->obj);
1533 /* Input port types. */
1535 struct port_in_type *elem;
1537 elem = TAILQ_FIRST(&p->port_in_types);
1541 TAILQ_REMOVE(&p->port_in_types, elem, node);
1549 static struct port_out_type *
1550 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1552 struct port_out_type *elem;
1557 TAILQ_FOREACH(elem, &p->port_out_types, node)
1558 if (!strcmp(elem->name, name))
1565 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1567 struct rte_swx_port_out_ops *ops)
1569 struct port_out_type *elem;
1572 CHECK_NAME(name, EINVAL);
1574 CHECK(ops->create, EINVAL);
1575 CHECK(ops->free, EINVAL);
1576 CHECK(ops->pkt_tx, EINVAL);
1577 CHECK(ops->stats_read, EINVAL);
1579 CHECK(!port_out_type_find(p, name), EEXIST);
1581 /* Node allocation. */
1582 elem = calloc(1, sizeof(struct port_out_type));
1583 CHECK(elem, ENOMEM);
1585 /* Node initialization. */
1586 strcpy(elem->name, name);
1587 memcpy(&elem->ops, ops, sizeof(*ops));
1589 /* Node add to tailq. */
1590 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1595 static struct port_out *
1596 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1598 struct port_out *port;
1600 TAILQ_FOREACH(port, &p->ports_out, node)
1601 if (port->id == port_id)
1608 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1610 const char *port_type_name,
1613 struct port_out_type *type = NULL;
1614 struct port_out *port = NULL;
1619 CHECK(!port_out_find(p, port_id), EINVAL);
1621 CHECK_NAME(port_type_name, EINVAL);
1622 type = port_out_type_find(p, port_type_name);
1623 CHECK(type, EINVAL);
1625 obj = type->ops.create(args);
1628 /* Node allocation. */
1629 port = calloc(1, sizeof(struct port_out));
1630 CHECK(port, ENOMEM);
1632 /* Node initialization. */
1637 /* Node add to tailq. */
1638 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1639 if (p->n_ports_out < port_id + 1)
1640 p->n_ports_out = port_id + 1;
1646 port_out_build(struct rte_swx_pipeline *p)
1648 struct port_out *port;
1651 CHECK(p->n_ports_out, EINVAL);
1653 for (i = 0; i < p->n_ports_out; i++)
1654 CHECK(port_out_find(p, i), EINVAL);
1656 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1657 CHECK(p->out, ENOMEM);
1659 TAILQ_FOREACH(port, &p->ports_out, node) {
1660 struct port_out_runtime *out = &p->out[port->id];
1662 out->pkt_tx = port->type->ops.pkt_tx;
1663 out->flush = port->type->ops.flush;
1664 out->obj = port->obj;
1671 port_out_build_free(struct rte_swx_pipeline *p)
1678 port_out_free(struct rte_swx_pipeline *p)
1680 port_out_build_free(p);
1684 struct port_out *port;
1686 port = TAILQ_FIRST(&p->ports_out);
1690 TAILQ_REMOVE(&p->ports_out, port, node);
1691 port->type->ops.free(port->obj);
1695 /* Output port types. */
1697 struct port_out_type *elem;
1699 elem = TAILQ_FIRST(&p->port_out_types);
1703 TAILQ_REMOVE(&p->port_out_types, elem, node);
1711 static struct extern_type *
1712 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1714 struct extern_type *elem;
1716 TAILQ_FOREACH(elem, &p->extern_types, node)
1717 if (strcmp(elem->name, name) == 0)
1723 static struct extern_type_member_func *
1724 extern_type_member_func_find(struct extern_type *type, const char *name)
1726 struct extern_type_member_func *elem;
1728 TAILQ_FOREACH(elem, &type->funcs, node)
1729 if (strcmp(elem->name, name) == 0)
1735 static struct extern_obj *
1736 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1738 struct extern_obj *elem;
1740 TAILQ_FOREACH(elem, &p->extern_objs, node)
1741 if (strcmp(elem->name, name) == 0)
1747 static struct extern_type_member_func *
1748 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1750 struct extern_obj **obj)
1752 struct extern_obj *object;
1753 struct extern_type_member_func *func;
1754 char *object_name, *func_name;
1756 if (name[0] != 'e' || name[1] != '.')
1759 object_name = strdup(&name[2]);
1763 func_name = strchr(object_name, '.');
1772 object = extern_obj_find(p, object_name);
1778 func = extern_type_member_func_find(object->type, func_name);
1791 static struct field *
1792 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1794 struct extern_obj **object)
1796 struct extern_obj *obj;
1798 char *obj_name, *field_name;
1800 if ((name[0] != 'e') || (name[1] != '.'))
1803 obj_name = strdup(&name[2]);
1807 field_name = strchr(obj_name, '.');
1816 obj = extern_obj_find(p, obj_name);
1822 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1836 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1838 const char *mailbox_struct_type_name,
1839 rte_swx_extern_type_constructor_t constructor,
1840 rte_swx_extern_type_destructor_t destructor)
1842 struct extern_type *elem;
1843 struct struct_type *mailbox_struct_type;
1847 CHECK_NAME(name, EINVAL);
1848 CHECK(!extern_type_find(p, name), EEXIST);
1850 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1851 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1852 CHECK(mailbox_struct_type, EINVAL);
1854 CHECK(constructor, EINVAL);
1855 CHECK(destructor, EINVAL);
1857 /* Node allocation. */
1858 elem = calloc(1, sizeof(struct extern_type));
1859 CHECK(elem, ENOMEM);
1861 /* Node initialization. */
1862 strcpy(elem->name, name);
1863 elem->mailbox_struct_type = mailbox_struct_type;
1864 elem->constructor = constructor;
1865 elem->destructor = destructor;
1866 TAILQ_INIT(&elem->funcs);
1868 /* Node add to tailq. */
1869 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1875 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1876 const char *extern_type_name,
1878 rte_swx_extern_type_member_func_t member_func)
1880 struct extern_type *type;
1881 struct extern_type_member_func *type_member;
1885 CHECK_NAME(extern_type_name, EINVAL);
1886 type = extern_type_find(p, extern_type_name);
1887 CHECK(type, EINVAL);
1888 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1890 CHECK_NAME(name, EINVAL);
1891 CHECK(!extern_type_member_func_find(type, name), EEXIST);
1893 CHECK(member_func, EINVAL);
1895 /* Node allocation. */
1896 type_member = calloc(1, sizeof(struct extern_type_member_func));
1897 CHECK(type_member, ENOMEM);
1899 /* Node initialization. */
1900 strcpy(type_member->name, name);
1901 type_member->func = member_func;
1902 type_member->id = type->n_funcs;
1904 /* Node add to tailq. */
1905 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
1912 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
1913 const char *extern_type_name,
1917 struct extern_type *type;
1918 struct extern_obj *obj;
1923 CHECK_NAME(extern_type_name, EINVAL);
1924 type = extern_type_find(p, extern_type_name);
1925 CHECK(type, EINVAL);
1927 CHECK_NAME(name, EINVAL);
1928 CHECK(!extern_obj_find(p, name), EEXIST);
1930 /* Node allocation. */
1931 obj = calloc(1, sizeof(struct extern_obj));
1934 /* Object construction. */
1935 obj_handle = type->constructor(args);
1941 /* Node initialization. */
1942 strcpy(obj->name, name);
1944 obj->obj = obj_handle;
1945 obj->struct_id = p->n_structs;
1946 obj->id = p->n_extern_objs;
1948 /* Node add to tailq. */
1949 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
1957 extern_obj_build(struct rte_swx_pipeline *p)
1961 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1962 struct thread *t = &p->threads[i];
1963 struct extern_obj *obj;
1965 t->extern_objs = calloc(p->n_extern_objs,
1966 sizeof(struct extern_obj_runtime));
1967 CHECK(t->extern_objs, ENOMEM);
1969 TAILQ_FOREACH(obj, &p->extern_objs, node) {
1970 struct extern_obj_runtime *r =
1971 &t->extern_objs[obj->id];
1972 struct extern_type_member_func *func;
1973 uint32_t mailbox_size =
1974 obj->type->mailbox_struct_type->n_bits / 8;
1978 r->mailbox = calloc(1, mailbox_size);
1979 CHECK(r->mailbox, ENOMEM);
1981 TAILQ_FOREACH(func, &obj->type->funcs, node)
1982 r->funcs[func->id] = func->func;
1984 t->structs[obj->struct_id] = r->mailbox;
1992 extern_obj_build_free(struct rte_swx_pipeline *p)
1996 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1997 struct thread *t = &p->threads[i];
2000 if (!t->extern_objs)
2003 for (j = 0; j < p->n_extern_objs; j++) {
2004 struct extern_obj_runtime *r = &t->extern_objs[j];
2009 free(t->extern_objs);
2010 t->extern_objs = NULL;
2015 extern_obj_free(struct rte_swx_pipeline *p)
2017 extern_obj_build_free(p);
2019 /* Extern objects. */
2021 struct extern_obj *elem;
2023 elem = TAILQ_FIRST(&p->extern_objs);
2027 TAILQ_REMOVE(&p->extern_objs, elem, node);
2029 elem->type->destructor(elem->obj);
2035 struct extern_type *elem;
2037 elem = TAILQ_FIRST(&p->extern_types);
2041 TAILQ_REMOVE(&p->extern_types, elem, node);
2044 struct extern_type_member_func *func;
2046 func = TAILQ_FIRST(&elem->funcs);
2050 TAILQ_REMOVE(&elem->funcs, func, node);
2061 static struct extern_func *
2062 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2064 struct extern_func *elem;
2066 TAILQ_FOREACH(elem, &p->extern_funcs, node)
2067 if (strcmp(elem->name, name) == 0)
2073 static struct extern_func *
2074 extern_func_parse(struct rte_swx_pipeline *p,
2077 if (name[0] != 'f' || name[1] != '.')
2080 return extern_func_find(p, &name[2]);
2083 static struct field *
2084 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2086 struct extern_func **function)
2088 struct extern_func *func;
2090 char *func_name, *field_name;
2092 if ((name[0] != 'f') || (name[1] != '.'))
2095 func_name = strdup(&name[2]);
2099 field_name = strchr(func_name, '.');
2108 func = extern_func_find(p, func_name);
2114 f = struct_type_field_find(func->mailbox_struct_type, field_name);
2128 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2130 const char *mailbox_struct_type_name,
2131 rte_swx_extern_func_t func)
2133 struct extern_func *f;
2134 struct struct_type *mailbox_struct_type;
2138 CHECK_NAME(name, EINVAL);
2139 CHECK(!extern_func_find(p, name), EEXIST);
2141 CHECK_NAME(mailbox_struct_type_name, EINVAL);
2142 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2143 CHECK(mailbox_struct_type, EINVAL);
2145 CHECK(func, EINVAL);
2147 /* Node allocation. */
2148 f = calloc(1, sizeof(struct extern_func));
2149 CHECK(func, ENOMEM);
2151 /* Node initialization. */
2152 strcpy(f->name, name);
2153 f->mailbox_struct_type = mailbox_struct_type;
2155 f->struct_id = p->n_structs;
2156 f->id = p->n_extern_funcs;
2158 /* Node add to tailq. */
2159 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2160 p->n_extern_funcs++;
2167 extern_func_build(struct rte_swx_pipeline *p)
2171 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2172 struct thread *t = &p->threads[i];
2173 struct extern_func *func;
2175 /* Memory allocation. */
2176 t->extern_funcs = calloc(p->n_extern_funcs,
2177 sizeof(struct extern_func_runtime));
2178 CHECK(t->extern_funcs, ENOMEM);
2180 /* Extern function. */
2181 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2182 struct extern_func_runtime *r =
2183 &t->extern_funcs[func->id];
2184 uint32_t mailbox_size =
2185 func->mailbox_struct_type->n_bits / 8;
2187 r->func = func->func;
2189 r->mailbox = calloc(1, mailbox_size);
2190 CHECK(r->mailbox, ENOMEM);
2192 t->structs[func->struct_id] = r->mailbox;
2200 extern_func_build_free(struct rte_swx_pipeline *p)
2204 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2205 struct thread *t = &p->threads[i];
2208 if (!t->extern_funcs)
2211 for (j = 0; j < p->n_extern_funcs; j++) {
2212 struct extern_func_runtime *r = &t->extern_funcs[j];
2217 free(t->extern_funcs);
2218 t->extern_funcs = NULL;
2223 extern_func_free(struct rte_swx_pipeline *p)
2225 extern_func_build_free(p);
2228 struct extern_func *elem;
2230 elem = TAILQ_FIRST(&p->extern_funcs);
2234 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2242 static struct header *
2243 header_find(struct rte_swx_pipeline *p, const char *name)
2245 struct header *elem;
2247 TAILQ_FOREACH(elem, &p->headers, node)
2248 if (strcmp(elem->name, name) == 0)
2254 static struct header *
2255 header_parse(struct rte_swx_pipeline *p,
2258 if (name[0] != 'h' || name[1] != '.')
2261 return header_find(p, &name[2]);
2264 static struct field *
2265 header_field_parse(struct rte_swx_pipeline *p,
2267 struct header **header)
2271 char *header_name, *field_name;
2273 if ((name[0] != 'h') || (name[1] != '.'))
2276 header_name = strdup(&name[2]);
2280 field_name = strchr(header_name, '.');
2289 h = header_find(p, header_name);
2295 f = struct_type_field_find(h->st, field_name);
2309 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2311 const char *struct_type_name)
2313 struct struct_type *st;
2315 size_t n_headers_max;
2318 CHECK_NAME(name, EINVAL);
2319 CHECK_NAME(struct_type_name, EINVAL);
2321 CHECK(!header_find(p, name), EEXIST);
2323 st = struct_type_find(p, struct_type_name);
2326 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2327 CHECK(p->n_headers < n_headers_max, ENOSPC);
2329 /* Node allocation. */
2330 h = calloc(1, sizeof(struct header));
2333 /* Node initialization. */
2334 strcpy(h->name, name);
2336 h->struct_id = p->n_structs;
2337 h->id = p->n_headers;
2339 /* Node add to tailq. */
2340 TAILQ_INSERT_TAIL(&p->headers, h, node);
2348 header_build(struct rte_swx_pipeline *p)
2351 uint32_t n_bytes = 0, i;
2353 TAILQ_FOREACH(h, &p->headers, node) {
2354 n_bytes += h->st->n_bits / 8;
2357 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2358 struct thread *t = &p->threads[i];
2359 uint32_t offset = 0;
2361 t->headers = calloc(p->n_headers,
2362 sizeof(struct header_runtime));
2363 CHECK(t->headers, ENOMEM);
2365 t->headers_out = calloc(p->n_headers,
2366 sizeof(struct header_out_runtime));
2367 CHECK(t->headers_out, ENOMEM);
2369 t->header_storage = calloc(1, n_bytes);
2370 CHECK(t->header_storage, ENOMEM);
2372 t->header_out_storage = calloc(1, n_bytes);
2373 CHECK(t->header_out_storage, ENOMEM);
2375 TAILQ_FOREACH(h, &p->headers, node) {
2376 uint8_t *header_storage;
2378 header_storage = &t->header_storage[offset];
2379 offset += h->st->n_bits / 8;
2381 t->headers[h->id].ptr0 = header_storage;
2382 t->structs[h->struct_id] = header_storage;
2390 header_build_free(struct rte_swx_pipeline *p)
2394 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2395 struct thread *t = &p->threads[i];
2397 free(t->headers_out);
2398 t->headers_out = NULL;
2403 free(t->header_out_storage);
2404 t->header_out_storage = NULL;
2406 free(t->header_storage);
2407 t->header_storage = NULL;
2412 header_free(struct rte_swx_pipeline *p)
2414 header_build_free(p);
2417 struct header *elem;
2419 elem = TAILQ_FIRST(&p->headers);
2423 TAILQ_REMOVE(&p->headers, elem, node);
2431 static struct field *
2432 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2434 if (!p->metadata_st)
2437 if (name[0] != 'm' || name[1] != '.')
2440 return struct_type_field_find(p->metadata_st, &name[2]);
2444 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2445 const char *struct_type_name)
2447 struct struct_type *st = NULL;
2451 CHECK_NAME(struct_type_name, EINVAL);
2452 st = struct_type_find(p, struct_type_name);
2454 CHECK(!p->metadata_st, EINVAL);
2456 p->metadata_st = st;
2457 p->metadata_struct_id = p->n_structs;
2465 metadata_build(struct rte_swx_pipeline *p)
2467 uint32_t n_bytes = p->metadata_st->n_bits / 8;
2470 /* Thread-level initialization. */
2471 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2472 struct thread *t = &p->threads[i];
2475 metadata = calloc(1, n_bytes);
2476 CHECK(metadata, ENOMEM);
2478 t->metadata = metadata;
2479 t->structs[p->metadata_struct_id] = metadata;
2486 metadata_build_free(struct rte_swx_pipeline *p)
2490 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2491 struct thread *t = &p->threads[i];
2499 metadata_free(struct rte_swx_pipeline *p)
2501 metadata_build_free(p);
2508 instruction_is_tx(enum instruction_type type)
2521 instruction_is_jmp(struct instruction *instr)
2523 switch (instr->type) {
2525 case INSTR_JMP_VALID:
2526 case INSTR_JMP_INVALID:
2528 case INSTR_JMP_MISS:
2529 case INSTR_JMP_ACTION_HIT:
2530 case INSTR_JMP_ACTION_MISS:
2532 case INSTR_JMP_EQ_S:
2533 case INSTR_JMP_EQ_I:
2535 case INSTR_JMP_NEQ_S:
2536 case INSTR_JMP_NEQ_I:
2538 case INSTR_JMP_LT_MH:
2539 case INSTR_JMP_LT_HM:
2540 case INSTR_JMP_LT_HH:
2541 case INSTR_JMP_LT_MI:
2542 case INSTR_JMP_LT_HI:
2544 case INSTR_JMP_GT_MH:
2545 case INSTR_JMP_GT_HM:
2546 case INSTR_JMP_GT_HH:
2547 case INSTR_JMP_GT_MI:
2548 case INSTR_JMP_GT_HI:
2556 static struct field *
2557 action_field_parse(struct action *action, const char *name);
2559 static struct field *
2560 struct_field_parse(struct rte_swx_pipeline *p,
2561 struct action *action,
2563 uint32_t *struct_id)
2570 struct header *header;
2572 f = header_field_parse(p, name, &header);
2576 *struct_id = header->struct_id;
2582 f = metadata_field_parse(p, name);
2586 *struct_id = p->metadata_struct_id;
2595 f = action_field_parse(action, name);
2605 struct extern_obj *obj;
2607 f = extern_obj_mailbox_field_parse(p, name, &obj);
2611 *struct_id = obj->struct_id;
2617 struct extern_func *func;
2619 f = extern_func_mailbox_field_parse(p, name, &func);
2623 *struct_id = func->struct_id;
2633 pipeline_port_inc(struct rte_swx_pipeline *p)
2635 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2639 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2641 t->ip = p->instructions;
2645 thread_ip_set(struct thread *t, struct instruction *ip)
2651 thread_ip_action_call(struct rte_swx_pipeline *p,
2656 t->ip = p->action_instructions[action_id];
2660 thread_ip_inc(struct rte_swx_pipeline *p);
2663 thread_ip_inc(struct rte_swx_pipeline *p)
2665 struct thread *t = &p->threads[p->thread_id];
2671 thread_ip_inc_cond(struct thread *t, int cond)
2677 thread_yield(struct rte_swx_pipeline *p)
2679 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2683 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2685 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2692 instr_rx_translate(struct rte_swx_pipeline *p,
2693 struct action *action,
2696 struct instruction *instr,
2697 struct instruction_data *data __rte_unused)
2701 CHECK(!action, EINVAL);
2702 CHECK(n_tokens == 2, EINVAL);
2704 f = metadata_field_parse(p, tokens[1]);
2707 instr->type = INSTR_RX;
2708 instr->io.io.offset = f->offset / 8;
2709 instr->io.io.n_bits = f->n_bits;
2714 instr_rx_exec(struct rte_swx_pipeline *p);
2717 instr_rx_exec(struct rte_swx_pipeline *p)
2719 struct thread *t = &p->threads[p->thread_id];
2720 struct instruction *ip = t->ip;
2721 struct port_in_runtime *port = &p->in[p->port_id];
2722 struct rte_swx_pkt *pkt = &t->pkt;
2726 pkt_received = port->pkt_rx(port->obj, pkt);
2727 t->ptr = &pkt->pkt[pkt->offset];
2728 rte_prefetch0(t->ptr);
2730 TRACE("[Thread %2u] rx %s from port %u\n",
2732 pkt_received ? "1 pkt" : "0 pkts",
2736 t->valid_headers = 0;
2737 t->n_headers_out = 0;
2740 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2743 t->table_state = p->table_state;
2746 pipeline_port_inc(p);
2747 thread_ip_inc_cond(t, pkt_received);
2755 instr_tx_translate(struct rte_swx_pipeline *p,
2756 struct action *action __rte_unused,
2759 struct instruction *instr,
2760 struct instruction_data *data __rte_unused)
2762 char *port = tokens[1];
2766 CHECK(n_tokens == 2, EINVAL);
2768 f = metadata_field_parse(p, port);
2770 instr->type = INSTR_TX;
2771 instr->io.io.offset = f->offset / 8;
2772 instr->io.io.n_bits = f->n_bits;
2777 port_val = strtoul(port, &port, 0);
2778 CHECK(!port[0], EINVAL);
2780 instr->type = INSTR_TX_I;
2781 instr->io.io.val = port_val;
2786 instr_drop_translate(struct rte_swx_pipeline *p,
2787 struct action *action __rte_unused,
2788 char **tokens __rte_unused,
2790 struct instruction *instr,
2791 struct instruction_data *data __rte_unused)
2793 CHECK(n_tokens == 1, EINVAL);
2796 instr->type = INSTR_TX_I;
2797 instr->io.io.val = p->n_ports_out - 1;
2802 emit_handler(struct thread *t)
2804 struct header_out_runtime *h0 = &t->headers_out[0];
2805 struct header_out_runtime *h1 = &t->headers_out[1];
2806 uint32_t offset = 0, i;
2808 /* No header change or header decapsulation. */
2809 if ((t->n_headers_out == 1) &&
2810 (h0->ptr + h0->n_bytes == t->ptr)) {
2811 TRACE("Emit handler: no header change or header decap.\n");
2813 t->pkt.offset -= h0->n_bytes;
2814 t->pkt.length += h0->n_bytes;
2819 /* Header encapsulation (optionally, with prior header decasulation). */
2820 if ((t->n_headers_out == 2) &&
2821 (h1->ptr + h1->n_bytes == t->ptr) &&
2822 (h0->ptr == h0->ptr0)) {
2825 TRACE("Emit handler: header encapsulation.\n");
2827 offset = h0->n_bytes + h1->n_bytes;
2828 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2829 t->pkt.offset -= offset;
2830 t->pkt.length += offset;
2835 /* Header insertion. */
2838 /* Header extraction. */
2841 /* For any other case. */
2842 TRACE("Emit handler: complex case.\n");
2844 for (i = 0; i < t->n_headers_out; i++) {
2845 struct header_out_runtime *h = &t->headers_out[i];
2847 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2848 offset += h->n_bytes;
2852 memcpy(t->ptr - offset, t->header_out_storage, offset);
2853 t->pkt.offset -= offset;
2854 t->pkt.length += offset;
2859 instr_tx_exec(struct rte_swx_pipeline *p);
2862 instr_tx_exec(struct rte_swx_pipeline *p)
2864 struct thread *t = &p->threads[p->thread_id];
2865 struct instruction *ip = t->ip;
2866 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2867 struct port_out_runtime *port = &p->out[port_id];
2868 struct rte_swx_pkt *pkt = &t->pkt;
2870 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2878 port->pkt_tx(port->obj, pkt);
2881 thread_ip_reset(p, t);
2886 instr_tx_i_exec(struct rte_swx_pipeline *p)
2888 struct thread *t = &p->threads[p->thread_id];
2889 struct instruction *ip = t->ip;
2890 uint64_t port_id = ip->io.io.val;
2891 struct port_out_runtime *port = &p->out[port_id];
2892 struct rte_swx_pkt *pkt = &t->pkt;
2894 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
2902 port->pkt_tx(port->obj, pkt);
2905 thread_ip_reset(p, t);
2913 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
2914 struct action *action,
2917 struct instruction *instr,
2918 struct instruction_data *data __rte_unused)
2922 CHECK(!action, EINVAL);
2923 CHECK(n_tokens == 2, EINVAL);
2925 h = header_parse(p, tokens[1]);
2928 instr->type = INSTR_HDR_EXTRACT;
2929 instr->io.hdr.header_id[0] = h->id;
2930 instr->io.hdr.struct_id[0] = h->struct_id;
2931 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
2936 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
2939 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
2941 struct thread *t = &p->threads[p->thread_id];
2942 struct instruction *ip = t->ip;
2943 uint64_t valid_headers = t->valid_headers;
2944 uint8_t *ptr = t->ptr;
2945 uint32_t offset = t->pkt.offset;
2946 uint32_t length = t->pkt.length;
2949 for (i = 0; i < n_extract; i++) {
2950 uint32_t header_id = ip->io.hdr.header_id[i];
2951 uint32_t struct_id = ip->io.hdr.struct_id[i];
2952 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
2954 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
2960 t->structs[struct_id] = ptr;
2961 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2970 t->valid_headers = valid_headers;
2973 t->pkt.offset = offset;
2974 t->pkt.length = length;
2979 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
2981 __instr_hdr_extract_exec(p, 1);
2988 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
2990 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
2993 __instr_hdr_extract_exec(p, 2);
3000 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3002 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3005 __instr_hdr_extract_exec(p, 3);
3012 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3014 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3017 __instr_hdr_extract_exec(p, 4);
3024 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3026 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3029 __instr_hdr_extract_exec(p, 5);
3036 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3038 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3041 __instr_hdr_extract_exec(p, 6);
3048 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3050 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3053 __instr_hdr_extract_exec(p, 7);
3060 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3062 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3065 __instr_hdr_extract_exec(p, 8);
3075 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3076 struct action *action __rte_unused,
3079 struct instruction *instr,
3080 struct instruction_data *data __rte_unused)
3084 CHECK(n_tokens == 2, EINVAL);
3086 h = header_parse(p, tokens[1]);
3089 instr->type = INSTR_HDR_EMIT;
3090 instr->io.hdr.header_id[0] = h->id;
3091 instr->io.hdr.struct_id[0] = h->struct_id;
3092 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3097 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3100 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3102 struct thread *t = &p->threads[p->thread_id];
3103 struct instruction *ip = t->ip;
3104 uint64_t valid_headers = t->valid_headers;
3105 uint32_t n_headers_out = t->n_headers_out;
3106 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3107 uint8_t *ho_ptr = NULL;
3108 uint32_t ho_nbytes = 0, first = 1, i;
3110 for (i = 0; i < n_emit; i++) {
3111 uint32_t header_id = ip->io.hdr.header_id[i];
3112 uint32_t struct_id = ip->io.hdr.struct_id[i];
3113 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3115 struct header_runtime *hi = &t->headers[header_id];
3116 uint8_t *hi_ptr = t->structs[struct_id];
3118 if (!MASK64_BIT_GET(valid_headers, header_id))
3121 TRACE("[Thread %2u]: emit header %u\n",
3129 if (!t->n_headers_out) {
3130 ho = &t->headers_out[0];
3132 ho->ptr0 = hi->ptr0;
3136 ho_nbytes = n_bytes;
3143 ho_nbytes = ho->n_bytes;
3147 if (ho_ptr + ho_nbytes == hi_ptr) {
3148 ho_nbytes += n_bytes;
3150 ho->n_bytes = ho_nbytes;
3153 ho->ptr0 = hi->ptr0;
3157 ho_nbytes = n_bytes;
3163 ho->n_bytes = ho_nbytes;
3164 t->n_headers_out = n_headers_out;
3168 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3170 __instr_hdr_emit_exec(p, 1);
3177 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3179 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3182 __instr_hdr_emit_exec(p, 1);
3187 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3189 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3192 __instr_hdr_emit_exec(p, 2);
3197 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3199 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3202 __instr_hdr_emit_exec(p, 3);
3207 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3209 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3212 __instr_hdr_emit_exec(p, 4);
3217 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3219 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3222 __instr_hdr_emit_exec(p, 5);
3227 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3229 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3232 __instr_hdr_emit_exec(p, 6);
3237 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3239 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3242 __instr_hdr_emit_exec(p, 7);
3247 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3249 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3252 __instr_hdr_emit_exec(p, 8);
3260 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3261 struct action *action __rte_unused,
3264 struct instruction *instr,
3265 struct instruction_data *data __rte_unused)
3269 CHECK(n_tokens == 2, EINVAL);
3271 h = header_parse(p, tokens[1]);
3274 instr->type = INSTR_HDR_VALIDATE;
3275 instr->valid.header_id = h->id;
3280 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3282 struct thread *t = &p->threads[p->thread_id];
3283 struct instruction *ip = t->ip;
3284 uint32_t header_id = ip->valid.header_id;
3286 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3289 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3299 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3300 struct action *action __rte_unused,
3303 struct instruction *instr,
3304 struct instruction_data *data __rte_unused)
3308 CHECK(n_tokens == 2, EINVAL);
3310 h = header_parse(p, tokens[1]);
3313 instr->type = INSTR_HDR_INVALIDATE;
3314 instr->valid.header_id = h->id;
3319 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3321 struct thread *t = &p->threads[p->thread_id];
3322 struct instruction *ip = t->ip;
3323 uint32_t header_id = ip->valid.header_id;
3325 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3328 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3337 static struct table *
3338 table_find(struct rte_swx_pipeline *p, const char *name);
3341 instr_table_translate(struct rte_swx_pipeline *p,
3342 struct action *action,
3345 struct instruction *instr,
3346 struct instruction_data *data __rte_unused)
3350 CHECK(!action, EINVAL);
3351 CHECK(n_tokens == 2, EINVAL);
3353 t = table_find(p, tokens[1]);
3356 instr->type = INSTR_TABLE;
3357 instr->table.table_id = t->id;
3362 instr_table_exec(struct rte_swx_pipeline *p)
3364 struct thread *t = &p->threads[p->thread_id];
3365 struct instruction *ip = t->ip;
3366 uint32_t table_id = ip->table.table_id;
3367 struct rte_swx_table_state *ts = &t->table_state[table_id];
3368 struct table_runtime *table = &t->tables[table_id];
3369 struct table_statistics *stats = &p->table_stats[table_id];
3370 uint64_t action_id, n_pkts_hit, n_pkts_action;
3371 uint8_t *action_data;
3375 done = table->func(ts->obj,
3383 TRACE("[Thread %2u] table %u (not finalized)\n",
3391 action_id = hit ? action_id : ts->default_action_id;
3392 action_data = hit ? action_data : ts->default_action_data;
3393 n_pkts_hit = stats->n_pkts_hit[hit];
3394 n_pkts_action = stats->n_pkts_action[action_id];
3396 TRACE("[Thread %2u] table %u (%s, action %u)\n",
3399 hit ? "hit" : "miss",
3400 (uint32_t)action_id);
3402 t->action_id = action_id;
3403 t->structs[0] = action_data;
3405 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3406 stats->n_pkts_action[action_id] = n_pkts_action + 1;
3409 thread_ip_action_call(p, t, action_id);
3416 instr_extern_translate(struct rte_swx_pipeline *p,
3417 struct action *action __rte_unused,
3420 struct instruction *instr,
3421 struct instruction_data *data __rte_unused)
3423 char *token = tokens[1];
3425 CHECK(n_tokens == 2, EINVAL);
3427 if (token[0] == 'e') {
3428 struct extern_obj *obj;
3429 struct extern_type_member_func *func;
3431 func = extern_obj_member_func_parse(p, token, &obj);
3432 CHECK(func, EINVAL);
3434 instr->type = INSTR_EXTERN_OBJ;
3435 instr->ext_obj.ext_obj_id = obj->id;
3436 instr->ext_obj.func_id = func->id;
3441 if (token[0] == 'f') {
3442 struct extern_func *func;
3444 func = extern_func_parse(p, token);
3445 CHECK(func, EINVAL);
3447 instr->type = INSTR_EXTERN_FUNC;
3448 instr->ext_func.ext_func_id = func->id;
3457 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3459 struct thread *t = &p->threads[p->thread_id];
3460 struct instruction *ip = t->ip;
3461 uint32_t obj_id = ip->ext_obj.ext_obj_id;
3462 uint32_t func_id = ip->ext_obj.func_id;
3463 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3464 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3466 TRACE("[Thread %2u] extern obj %u member func %u\n",
3471 /* Extern object member function execute. */
3472 uint32_t done = func(obj->obj, obj->mailbox);
3475 thread_ip_inc_cond(t, done);
3476 thread_yield_cond(p, done ^ 1);
3480 instr_extern_func_exec(struct rte_swx_pipeline *p)
3482 struct thread *t = &p->threads[p->thread_id];
3483 struct instruction *ip = t->ip;
3484 uint32_t ext_func_id = ip->ext_func.ext_func_id;
3485 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3486 rte_swx_extern_func_t func = ext_func->func;
3488 TRACE("[Thread %2u] extern func %u\n",
3492 /* Extern function execute. */
3493 uint32_t done = func(ext_func->mailbox);
3496 thread_ip_inc_cond(t, done);
3497 thread_yield_cond(p, done ^ 1);
3504 instr_mov_translate(struct rte_swx_pipeline *p,
3505 struct action *action,
3508 struct instruction *instr,
3509 struct instruction_data *data __rte_unused)
3511 char *dst = tokens[1], *src = tokens[2];
3512 struct field *fdst, *fsrc;
3514 uint32_t dst_struct_id, src_struct_id;
3516 CHECK(n_tokens == 3, EINVAL);
3518 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3519 CHECK(fdst, EINVAL);
3522 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3524 instr->type = INSTR_MOV;
3525 if ((dst[0] == 'h' && src[0] != 'h') ||
3526 (dst[0] != 'h' && src[0] == 'h'))
3527 instr->type = INSTR_MOV_S;
3529 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3530 instr->mov.dst.n_bits = fdst->n_bits;
3531 instr->mov.dst.offset = fdst->offset / 8;
3532 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3533 instr->mov.src.n_bits = fsrc->n_bits;
3534 instr->mov.src.offset = fsrc->offset / 8;
3539 src_val = strtoull(src, &src, 0);
3540 CHECK(!src[0], EINVAL);
3543 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3545 instr->type = INSTR_MOV_I;
3546 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3547 instr->mov.dst.n_bits = fdst->n_bits;
3548 instr->mov.dst.offset = fdst->offset / 8;
3549 instr->mov.src_val = src_val;
3554 instr_mov_exec(struct rte_swx_pipeline *p)
3556 struct thread *t = &p->threads[p->thread_id];
3557 struct instruction *ip = t->ip;
3559 TRACE("[Thread %2u] mov\n",
3569 instr_mov_s_exec(struct rte_swx_pipeline *p)
3571 struct thread *t = &p->threads[p->thread_id];
3572 struct instruction *ip = t->ip;
3574 TRACE("[Thread %2u] mov (s)\n",
3584 instr_mov_i_exec(struct rte_swx_pipeline *p)
3586 struct thread *t = &p->threads[p->thread_id];
3587 struct instruction *ip = t->ip;
3589 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3603 instr_dma_translate(struct rte_swx_pipeline *p,
3604 struct action *action,
3607 struct instruction *instr,
3608 struct instruction_data *data __rte_unused)
3610 char *dst = tokens[1];
3611 char *src = tokens[2];
3615 CHECK(action, EINVAL);
3616 CHECK(n_tokens == 3, EINVAL);
3618 h = header_parse(p, dst);
3621 tf = action_field_parse(action, src);
3624 instr->type = INSTR_DMA_HT;
3625 instr->dma.dst.header_id[0] = h->id;
3626 instr->dma.dst.struct_id[0] = h->struct_id;
3627 instr->dma.n_bytes[0] = h->st->n_bits / 8;
3628 instr->dma.src.offset[0] = tf->offset / 8;
3634 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3637 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3639 struct thread *t = &p->threads[p->thread_id];
3640 struct instruction *ip = t->ip;
3641 uint8_t *action_data = t->structs[0];
3642 uint64_t valid_headers = t->valid_headers;
3645 for (i = 0; i < n_dma; i++) {
3646 uint32_t header_id = ip->dma.dst.header_id[i];
3647 uint32_t struct_id = ip->dma.dst.struct_id[i];
3648 uint32_t offset = ip->dma.src.offset[i];
3649 uint32_t n_bytes = ip->dma.n_bytes[i];
3651 struct header_runtime *h = &t->headers[header_id];
3652 uint8_t *h_ptr0 = h->ptr0;
3653 uint8_t *h_ptr = t->structs[struct_id];
3655 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3657 void *src = &action_data[offset];
3659 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3662 memcpy(dst, src, n_bytes);
3663 t->structs[struct_id] = dst;
3664 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3667 t->valid_headers = valid_headers;
3671 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3673 __instr_dma_ht_exec(p, 1);
3680 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3682 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3685 __instr_dma_ht_exec(p, 2);
3692 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3694 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3697 __instr_dma_ht_exec(p, 3);
3704 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3706 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3709 __instr_dma_ht_exec(p, 4);
3716 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3718 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3721 __instr_dma_ht_exec(p, 5);
3728 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3730 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3733 __instr_dma_ht_exec(p, 6);
3740 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3742 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3745 __instr_dma_ht_exec(p, 7);
3752 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3754 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3757 __instr_dma_ht_exec(p, 8);
3767 instr_alu_add_translate(struct rte_swx_pipeline *p,
3768 struct action *action,
3771 struct instruction *instr,
3772 struct instruction_data *data __rte_unused)
3774 char *dst = tokens[1], *src = tokens[2];
3775 struct field *fdst, *fsrc;
3777 uint32_t dst_struct_id, src_struct_id;
3779 CHECK(n_tokens == 3, EINVAL);
3781 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3782 CHECK(fdst, EINVAL);
3784 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3785 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3787 instr->type = INSTR_ALU_ADD;
3788 if (dst[0] == 'h' && src[0] != 'h')
3789 instr->type = INSTR_ALU_ADD_HM;
3790 if (dst[0] != 'h' && src[0] == 'h')
3791 instr->type = INSTR_ALU_ADD_MH;
3792 if (dst[0] == 'h' && src[0] == 'h')
3793 instr->type = INSTR_ALU_ADD_HH;
3795 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3796 instr->alu.dst.n_bits = fdst->n_bits;
3797 instr->alu.dst.offset = fdst->offset / 8;
3798 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3799 instr->alu.src.n_bits = fsrc->n_bits;
3800 instr->alu.src.offset = fsrc->offset / 8;
3804 /* ADD_MI, ADD_HI. */
3805 src_val = strtoull(src, &src, 0);
3806 CHECK(!src[0], EINVAL);
3808 instr->type = INSTR_ALU_ADD_MI;
3810 instr->type = INSTR_ALU_ADD_HI;
3812 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3813 instr->alu.dst.n_bits = fdst->n_bits;
3814 instr->alu.dst.offset = fdst->offset / 8;
3815 instr->alu.src_val = src_val;
3820 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3821 struct action *action,
3824 struct instruction *instr,
3825 struct instruction_data *data __rte_unused)
3827 char *dst = tokens[1], *src = tokens[2];
3828 struct field *fdst, *fsrc;
3830 uint32_t dst_struct_id, src_struct_id;
3832 CHECK(n_tokens == 3, EINVAL);
3834 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3835 CHECK(fdst, EINVAL);
3837 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3838 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3840 instr->type = INSTR_ALU_SUB;
3841 if (dst[0] == 'h' && src[0] != 'h')
3842 instr->type = INSTR_ALU_SUB_HM;
3843 if (dst[0] != 'h' && src[0] == 'h')
3844 instr->type = INSTR_ALU_SUB_MH;
3845 if (dst[0] == 'h' && src[0] == 'h')
3846 instr->type = INSTR_ALU_SUB_HH;
3848 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3849 instr->alu.dst.n_bits = fdst->n_bits;
3850 instr->alu.dst.offset = fdst->offset / 8;
3851 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3852 instr->alu.src.n_bits = fsrc->n_bits;
3853 instr->alu.src.offset = fsrc->offset / 8;
3857 /* SUB_MI, SUB_HI. */
3858 src_val = strtoull(src, &src, 0);
3859 CHECK(!src[0], EINVAL);
3861 instr->type = INSTR_ALU_SUB_MI;
3863 instr->type = INSTR_ALU_SUB_HI;
3865 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3866 instr->alu.dst.n_bits = fdst->n_bits;
3867 instr->alu.dst.offset = fdst->offset / 8;
3868 instr->alu.src_val = src_val;
3873 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
3874 struct action *action __rte_unused,
3877 struct instruction *instr,
3878 struct instruction_data *data __rte_unused)
3880 char *dst = tokens[1], *src = tokens[2];
3881 struct header *hdst, *hsrc;
3882 struct field *fdst, *fsrc;
3884 CHECK(n_tokens == 3, EINVAL);
3886 fdst = header_field_parse(p, dst, &hdst);
3887 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3890 fsrc = header_field_parse(p, src, &hsrc);
3892 instr->type = INSTR_ALU_CKADD_FIELD;
3893 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3894 instr->alu.dst.n_bits = fdst->n_bits;
3895 instr->alu.dst.offset = fdst->offset / 8;
3896 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3897 instr->alu.src.n_bits = fsrc->n_bits;
3898 instr->alu.src.offset = fsrc->offset / 8;
3902 /* CKADD_STRUCT, CKADD_STRUCT20. */
3903 hsrc = header_parse(p, src);
3904 CHECK(hsrc, EINVAL);
3906 instr->type = INSTR_ALU_CKADD_STRUCT;
3907 if ((hsrc->st->n_bits / 8) == 20)
3908 instr->type = INSTR_ALU_CKADD_STRUCT20;
3910 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3911 instr->alu.dst.n_bits = fdst->n_bits;
3912 instr->alu.dst.offset = fdst->offset / 8;
3913 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3914 instr->alu.src.n_bits = hsrc->st->n_bits;
3915 instr->alu.src.offset = 0; /* Unused. */
3920 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
3921 struct action *action __rte_unused,
3924 struct instruction *instr,
3925 struct instruction_data *data __rte_unused)
3927 char *dst = tokens[1], *src = tokens[2];
3928 struct header *hdst, *hsrc;
3929 struct field *fdst, *fsrc;
3931 CHECK(n_tokens == 3, EINVAL);
3933 fdst = header_field_parse(p, dst, &hdst);
3934 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
3936 fsrc = header_field_parse(p, src, &hsrc);
3937 CHECK(fsrc, EINVAL);
3939 instr->type = INSTR_ALU_CKSUB_FIELD;
3940 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
3941 instr->alu.dst.n_bits = fdst->n_bits;
3942 instr->alu.dst.offset = fdst->offset / 8;
3943 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
3944 instr->alu.src.n_bits = fsrc->n_bits;
3945 instr->alu.src.offset = fsrc->offset / 8;
3950 instr_alu_shl_translate(struct rte_swx_pipeline *p,
3951 struct action *action,
3954 struct instruction *instr,
3955 struct instruction_data *data __rte_unused)
3957 char *dst = tokens[1], *src = tokens[2];
3958 struct field *fdst, *fsrc;
3960 uint32_t dst_struct_id, src_struct_id;
3962 CHECK(n_tokens == 3, EINVAL);
3964 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3965 CHECK(fdst, EINVAL);
3967 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
3968 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3970 instr->type = INSTR_ALU_SHL;
3971 if (dst[0] == 'h' && src[0] != 'h')
3972 instr->type = INSTR_ALU_SHL_HM;
3973 if (dst[0] != 'h' && src[0] == 'h')
3974 instr->type = INSTR_ALU_SHL_MH;
3975 if (dst[0] == 'h' && src[0] == 'h')
3976 instr->type = INSTR_ALU_SHL_HH;
3978 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3979 instr->alu.dst.n_bits = fdst->n_bits;
3980 instr->alu.dst.offset = fdst->offset / 8;
3981 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3982 instr->alu.src.n_bits = fsrc->n_bits;
3983 instr->alu.src.offset = fsrc->offset / 8;
3987 /* SHL_MI, SHL_HI. */
3988 src_val = strtoull(src, &src, 0);
3989 CHECK(!src[0], EINVAL);
3991 instr->type = INSTR_ALU_SHL_MI;
3993 instr->type = INSTR_ALU_SHL_HI;
3995 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3996 instr->alu.dst.n_bits = fdst->n_bits;
3997 instr->alu.dst.offset = fdst->offset / 8;
3998 instr->alu.src_val = src_val;
4003 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4004 struct action *action,
4007 struct instruction *instr,
4008 struct instruction_data *data __rte_unused)
4010 char *dst = tokens[1], *src = tokens[2];
4011 struct field *fdst, *fsrc;
4013 uint32_t dst_struct_id, src_struct_id;
4015 CHECK(n_tokens == 3, EINVAL);
4017 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4018 CHECK(fdst, EINVAL);
4020 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4021 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4023 instr->type = INSTR_ALU_SHR;
4024 if (dst[0] == 'h' && src[0] != 'h')
4025 instr->type = INSTR_ALU_SHR_HM;
4026 if (dst[0] != 'h' && src[0] == 'h')
4027 instr->type = INSTR_ALU_SHR_MH;
4028 if (dst[0] == 'h' && src[0] == 'h')
4029 instr->type = INSTR_ALU_SHR_HH;
4031 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4032 instr->alu.dst.n_bits = fdst->n_bits;
4033 instr->alu.dst.offset = fdst->offset / 8;
4034 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4035 instr->alu.src.n_bits = fsrc->n_bits;
4036 instr->alu.src.offset = fsrc->offset / 8;
4040 /* SHR_MI, SHR_HI. */
4041 src_val = strtoull(src, &src, 0);
4042 CHECK(!src[0], EINVAL);
4044 instr->type = INSTR_ALU_SHR_MI;
4046 instr->type = INSTR_ALU_SHR_HI;
4048 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4049 instr->alu.dst.n_bits = fdst->n_bits;
4050 instr->alu.dst.offset = fdst->offset / 8;
4051 instr->alu.src_val = src_val;
4056 instr_alu_and_translate(struct rte_swx_pipeline *p,
4057 struct action *action,
4060 struct instruction *instr,
4061 struct instruction_data *data __rte_unused)
4063 char *dst = tokens[1], *src = tokens[2];
4064 struct field *fdst, *fsrc;
4066 uint32_t dst_struct_id, src_struct_id;
4068 CHECK(n_tokens == 3, EINVAL);
4070 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4071 CHECK(fdst, EINVAL);
4074 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4076 instr->type = INSTR_ALU_AND;
4077 if ((dst[0] == 'h' && src[0] != 'h') ||
4078 (dst[0] != 'h' && src[0] == 'h'))
4079 instr->type = INSTR_ALU_AND_S;
4081 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4082 instr->alu.dst.n_bits = fdst->n_bits;
4083 instr->alu.dst.offset = fdst->offset / 8;
4084 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4085 instr->alu.src.n_bits = fsrc->n_bits;
4086 instr->alu.src.offset = fsrc->offset / 8;
4091 src_val = strtoull(src, &src, 0);
4092 CHECK(!src[0], EINVAL);
4095 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4097 instr->type = INSTR_ALU_AND_I;
4098 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4099 instr->alu.dst.n_bits = fdst->n_bits;
4100 instr->alu.dst.offset = fdst->offset / 8;
4101 instr->alu.src_val = src_val;
4106 instr_alu_or_translate(struct rte_swx_pipeline *p,
4107 struct action *action,
4110 struct instruction *instr,
4111 struct instruction_data *data __rte_unused)
4113 char *dst = tokens[1], *src = tokens[2];
4114 struct field *fdst, *fsrc;
4116 uint32_t dst_struct_id, src_struct_id;
4118 CHECK(n_tokens == 3, EINVAL);
4120 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4121 CHECK(fdst, EINVAL);
4124 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4126 instr->type = INSTR_ALU_OR;
4127 if ((dst[0] == 'h' && src[0] != 'h') ||
4128 (dst[0] != 'h' && src[0] == 'h'))
4129 instr->type = INSTR_ALU_OR_S;
4131 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4132 instr->alu.dst.n_bits = fdst->n_bits;
4133 instr->alu.dst.offset = fdst->offset / 8;
4134 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4135 instr->alu.src.n_bits = fsrc->n_bits;
4136 instr->alu.src.offset = fsrc->offset / 8;
4141 src_val = strtoull(src, &src, 0);
4142 CHECK(!src[0], EINVAL);
4145 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4147 instr->type = INSTR_ALU_OR_I;
4148 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4149 instr->alu.dst.n_bits = fdst->n_bits;
4150 instr->alu.dst.offset = fdst->offset / 8;
4151 instr->alu.src_val = src_val;
4156 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4157 struct action *action,
4160 struct instruction *instr,
4161 struct instruction_data *data __rte_unused)
4163 char *dst = tokens[1], *src = tokens[2];
4164 struct field *fdst, *fsrc;
4166 uint32_t dst_struct_id, src_struct_id;
4168 CHECK(n_tokens == 3, EINVAL);
4170 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4171 CHECK(fdst, EINVAL);
4174 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4176 instr->type = INSTR_ALU_XOR;
4177 if ((dst[0] == 'h' && src[0] != 'h') ||
4178 (dst[0] != 'h' && src[0] == 'h'))
4179 instr->type = INSTR_ALU_XOR_S;
4181 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4182 instr->alu.dst.n_bits = fdst->n_bits;
4183 instr->alu.dst.offset = fdst->offset / 8;
4184 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4185 instr->alu.src.n_bits = fsrc->n_bits;
4186 instr->alu.src.offset = fsrc->offset / 8;
4191 src_val = strtoull(src, &src, 0);
4192 CHECK(!src[0], EINVAL);
4195 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4197 instr->type = INSTR_ALU_XOR_I;
4198 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4199 instr->alu.dst.n_bits = fdst->n_bits;
4200 instr->alu.dst.offset = fdst->offset / 8;
4201 instr->alu.src_val = src_val;
4206 instr_alu_add_exec(struct rte_swx_pipeline *p)
4208 struct thread *t = &p->threads[p->thread_id];
4209 struct instruction *ip = t->ip;
4211 TRACE("[Thread %2u] add\n", p->thread_id);
4221 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4223 struct thread *t = &p->threads[p->thread_id];
4224 struct instruction *ip = t->ip;
4226 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4236 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4238 struct thread *t = &p->threads[p->thread_id];
4239 struct instruction *ip = t->ip;
4241 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4251 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4253 struct thread *t = &p->threads[p->thread_id];
4254 struct instruction *ip = t->ip;
4256 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4266 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4268 struct thread *t = &p->threads[p->thread_id];
4269 struct instruction *ip = t->ip;
4271 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4281 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4283 struct thread *t = &p->threads[p->thread_id];
4284 struct instruction *ip = t->ip;
4286 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4296 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4298 struct thread *t = &p->threads[p->thread_id];
4299 struct instruction *ip = t->ip;
4301 TRACE("[Thread %2u] sub\n", p->thread_id);
4311 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4313 struct thread *t = &p->threads[p->thread_id];
4314 struct instruction *ip = t->ip;
4316 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4326 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4328 struct thread *t = &p->threads[p->thread_id];
4329 struct instruction *ip = t->ip;
4331 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4341 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4343 struct thread *t = &p->threads[p->thread_id];
4344 struct instruction *ip = t->ip;
4346 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4356 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4358 struct thread *t = &p->threads[p->thread_id];
4359 struct instruction *ip = t->ip;
4361 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4371 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4373 struct thread *t = &p->threads[p->thread_id];
4374 struct instruction *ip = t->ip;
4376 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4386 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4388 struct thread *t = &p->threads[p->thread_id];
4389 struct instruction *ip = t->ip;
4391 TRACE("[Thread %2u] shl\n", p->thread_id);
4401 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4403 struct thread *t = &p->threads[p->thread_id];
4404 struct instruction *ip = t->ip;
4406 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4416 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4418 struct thread *t = &p->threads[p->thread_id];
4419 struct instruction *ip = t->ip;
4421 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4431 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4433 struct thread *t = &p->threads[p->thread_id];
4434 struct instruction *ip = t->ip;
4436 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4446 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4448 struct thread *t = &p->threads[p->thread_id];
4449 struct instruction *ip = t->ip;
4451 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4461 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4463 struct thread *t = &p->threads[p->thread_id];
4464 struct instruction *ip = t->ip;
4466 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4476 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4478 struct thread *t = &p->threads[p->thread_id];
4479 struct instruction *ip = t->ip;
4481 TRACE("[Thread %2u] shr\n", p->thread_id);
4491 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4493 struct thread *t = &p->threads[p->thread_id];
4494 struct instruction *ip = t->ip;
4496 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4506 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4508 struct thread *t = &p->threads[p->thread_id];
4509 struct instruction *ip = t->ip;
4511 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4521 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4523 struct thread *t = &p->threads[p->thread_id];
4524 struct instruction *ip = t->ip;
4526 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4536 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4538 struct thread *t = &p->threads[p->thread_id];
4539 struct instruction *ip = t->ip;
4541 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4551 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4553 struct thread *t = &p->threads[p->thread_id];
4554 struct instruction *ip = t->ip;
4556 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4566 instr_alu_and_exec(struct rte_swx_pipeline *p)
4568 struct thread *t = &p->threads[p->thread_id];
4569 struct instruction *ip = t->ip;
4571 TRACE("[Thread %2u] and\n", p->thread_id);
4581 instr_alu_and_s_exec(struct rte_swx_pipeline *p)
4583 struct thread *t = &p->threads[p->thread_id];
4584 struct instruction *ip = t->ip;
4586 TRACE("[Thread %2u] and (s)\n", p->thread_id);
4596 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4598 struct thread *t = &p->threads[p->thread_id];
4599 struct instruction *ip = t->ip;
4601 TRACE("[Thread %2u] and (i)\n", p->thread_id);
4611 instr_alu_or_exec(struct rte_swx_pipeline *p)
4613 struct thread *t = &p->threads[p->thread_id];
4614 struct instruction *ip = t->ip;
4616 TRACE("[Thread %2u] or\n", p->thread_id);
4626 instr_alu_or_s_exec(struct rte_swx_pipeline *p)
4628 struct thread *t = &p->threads[p->thread_id];
4629 struct instruction *ip = t->ip;
4631 TRACE("[Thread %2u] or (s)\n", p->thread_id);
4641 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4643 struct thread *t = &p->threads[p->thread_id];
4644 struct instruction *ip = t->ip;
4646 TRACE("[Thread %2u] or (i)\n", p->thread_id);
4656 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4658 struct thread *t = &p->threads[p->thread_id];
4659 struct instruction *ip = t->ip;
4661 TRACE("[Thread %2u] xor\n", p->thread_id);
4671 instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
4673 struct thread *t = &p->threads[p->thread_id];
4674 struct instruction *ip = t->ip;
4676 TRACE("[Thread %2u] xor (s)\n", p->thread_id);
4686 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4688 struct thread *t = &p->threads[p->thread_id];
4689 struct instruction *ip = t->ip;
4691 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4701 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4703 struct thread *t = &p->threads[p->thread_id];
4704 struct instruction *ip = t->ip;
4705 uint8_t *dst_struct, *src_struct;
4706 uint16_t *dst16_ptr, dst;
4707 uint64_t *src64_ptr, src64, src64_mask, src;
4710 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4713 dst_struct = t->structs[ip->alu.dst.struct_id];
4714 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4717 src_struct = t->structs[ip->alu.src.struct_id];
4718 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4720 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4721 src = src64 & src64_mask;
4726 /* The first input (r) is a 16-bit number. The second and the third
4727 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4728 * three numbers (output r) is a 34-bit number.
4730 r += (src >> 32) + (src & 0xFFFFFFFF);
4732 /* The first input is a 16-bit number. The second input is an 18-bit
4733 * number. In the worst case scenario, the sum of the two numbers is a
4736 r = (r & 0xFFFF) + (r >> 16);
4738 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4739 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4741 r = (r & 0xFFFF) + (r >> 16);
4743 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4744 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4745 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4746 * therefore the output r is always a 16-bit number.
4748 r = (r & 0xFFFF) + (r >> 16);
4753 *dst16_ptr = (uint16_t)r;
4760 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4762 struct thread *t = &p->threads[p->thread_id];
4763 struct instruction *ip = t->ip;
4764 uint8_t *dst_struct, *src_struct;
4765 uint16_t *dst16_ptr, dst;
4766 uint64_t *src64_ptr, src64, src64_mask, src;
4769 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
4772 dst_struct = t->structs[ip->alu.dst.struct_id];
4773 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4776 src_struct = t->structs[ip->alu.src.struct_id];
4777 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4779 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4780 src = src64 & src64_mask;
4785 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
4786 * the following sequence of operations in 2's complement arithmetic:
4787 * a '- b = (a - b) % 0xFFFF.
4789 * In order to prevent an underflow for the below subtraction, in which
4790 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
4791 * minuend), we first add a multiple of the 0xFFFF modulus to the
4792 * minuend. The number we add to the minuend needs to be a 34-bit number
4793 * or higher, so for readability reasons we picked the 36-bit multiple.
4794 * We are effectively turning the 16-bit minuend into a 36-bit number:
4795 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
4797 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
4799 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
4800 * result (the output r) is a 36-bit number.
4802 r -= (src >> 32) + (src & 0xFFFFFFFF);
4804 /* The first input is a 16-bit number. The second input is a 20-bit
4805 * number. Their sum is a 21-bit number.
4807 r = (r & 0xFFFF) + (r >> 16);
4809 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4810 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
4812 r = (r & 0xFFFF) + (r >> 16);
4814 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4815 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4816 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4817 * generated, therefore the output r is always a 16-bit number.
4819 r = (r & 0xFFFF) + (r >> 16);
4824 *dst16_ptr = (uint16_t)r;
4831 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
4833 struct thread *t = &p->threads[p->thread_id];
4834 struct instruction *ip = t->ip;
4835 uint8_t *dst_struct, *src_struct;
4836 uint16_t *dst16_ptr;
4837 uint32_t *src32_ptr;
4840 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
4843 dst_struct = t->structs[ip->alu.dst.struct_id];
4844 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4846 src_struct = t->structs[ip->alu.src.struct_id];
4847 src32_ptr = (uint32_t *)&src_struct[0];
4849 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
4850 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
4851 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
4852 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
4853 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
4855 /* The first input is a 16-bit number. The second input is a 19-bit
4856 * number. Their sum is a 20-bit number.
4858 r0 = (r0 & 0xFFFF) + (r0 >> 16);
4860 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4861 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
4863 r0 = (r0 & 0xFFFF) + (r0 >> 16);
4865 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4866 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4867 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
4868 * generated, therefore the output r is always a 16-bit number.
4870 r0 = (r0 & 0xFFFF) + (r0 >> 16);
4873 r0 = r0 ? r0 : 0xFFFF;
4875 *dst16_ptr = (uint16_t)r0;
4882 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
4884 struct thread *t = &p->threads[p->thread_id];
4885 struct instruction *ip = t->ip;
4886 uint8_t *dst_struct, *src_struct;
4887 uint16_t *dst16_ptr;
4888 uint32_t *src32_ptr;
4892 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
4895 dst_struct = t->structs[ip->alu.dst.struct_id];
4896 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4898 src_struct = t->structs[ip->alu.src.struct_id];
4899 src32_ptr = (uint32_t *)&src_struct[0];
4901 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
4902 * Therefore, in the worst case scenario, a 35-bit number is added to a
4903 * 16-bit number (the input r), so the output r is 36-bit number.
4905 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
4908 /* The first input is a 16-bit number. The second input is a 20-bit
4909 * number. Their sum is a 21-bit number.
4911 r = (r & 0xFFFF) + (r >> 16);
4913 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4914 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
4916 r = (r & 0xFFFF) + (r >> 16);
4918 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4919 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4920 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
4921 * generated, therefore the output r is always a 16-bit number.
4923 r = (r & 0xFFFF) + (r >> 16);
4928 *dst16_ptr = (uint16_t)r;
4937 static struct regarray *
4938 regarray_find(struct rte_swx_pipeline *p, const char *name);
4941 instr_regprefetch_translate(struct rte_swx_pipeline *p,
4942 struct action *action,
4945 struct instruction *instr,
4946 struct instruction_data *data __rte_unused)
4948 char *regarray = tokens[1], *idx = tokens[2];
4951 uint32_t idx_struct_id, idx_val;
4953 CHECK(n_tokens == 3, EINVAL);
4955 r = regarray_find(p, regarray);
4958 /* REGPREFETCH_RH, REGPREFETCH_RM. */
4959 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
4961 instr->type = INSTR_REGPREFETCH_RM;
4963 instr->type = INSTR_REGPREFETCH_RH;
4965 instr->regarray.regarray_id = r->id;
4966 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
4967 instr->regarray.idx.n_bits = fidx->n_bits;
4968 instr->regarray.idx.offset = fidx->offset / 8;
4969 instr->regarray.dstsrc_val = 0; /* Unused. */
4973 /* REGPREFETCH_RI. */
4974 idx_val = strtoul(idx, &idx, 0);
4975 CHECK(!idx[0], EINVAL);
4977 instr->type = INSTR_REGPREFETCH_RI;
4978 instr->regarray.regarray_id = r->id;
4979 instr->regarray.idx_val = idx_val;
4980 instr->regarray.dstsrc_val = 0; /* Unused. */
4985 instr_regrd_translate(struct rte_swx_pipeline *p,
4986 struct action *action,
4989 struct instruction *instr,
4990 struct instruction_data *data __rte_unused)
4992 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
4994 struct field *fdst, *fidx;
4995 uint32_t dst_struct_id, idx_struct_id, idx_val;
4997 CHECK(n_tokens == 4, EINVAL);
4999 r = regarray_find(p, regarray);
5002 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5003 CHECK(fdst, EINVAL);
5005 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5006 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5008 instr->type = INSTR_REGRD_MRM;
5009 if (dst[0] == 'h' && idx[0] != 'h')
5010 instr->type = INSTR_REGRD_HRM;
5011 if (dst[0] != 'h' && idx[0] == 'h')
5012 instr->type = INSTR_REGRD_MRH;
5013 if (dst[0] == 'h' && idx[0] == 'h')
5014 instr->type = INSTR_REGRD_HRH;
5016 instr->regarray.regarray_id = r->id;
5017 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5018 instr->regarray.idx.n_bits = fidx->n_bits;
5019 instr->regarray.idx.offset = fidx->offset / 8;
5020 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5021 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5022 instr->regarray.dstsrc.offset = fdst->offset / 8;
5026 /* REGRD_MRI, REGRD_HRI. */
5027 idx_val = strtoul(idx, &idx, 0);
5028 CHECK(!idx[0], EINVAL);
5030 instr->type = INSTR_REGRD_MRI;
5032 instr->type = INSTR_REGRD_HRI;
5034 instr->regarray.regarray_id = r->id;
5035 instr->regarray.idx_val = idx_val;
5036 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5037 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5038 instr->regarray.dstsrc.offset = fdst->offset / 8;
5043 instr_regwr_translate(struct rte_swx_pipeline *p,
5044 struct action *action,
5047 struct instruction *instr,
5048 struct instruction_data *data __rte_unused)
5050 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5052 struct field *fidx, *fsrc;
5054 uint32_t idx_struct_id, idx_val, src_struct_id;
5056 CHECK(n_tokens == 4, EINVAL);
5058 r = regarray_find(p, regarray);
5061 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5062 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5063 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5065 instr->type = INSTR_REGWR_RMM;
5066 if (idx[0] == 'h' && src[0] != 'h')
5067 instr->type = INSTR_REGWR_RHM;
5068 if (idx[0] != 'h' && src[0] == 'h')
5069 instr->type = INSTR_REGWR_RMH;
5070 if (idx[0] == 'h' && src[0] == 'h')
5071 instr->type = INSTR_REGWR_RHH;
5073 instr->regarray.regarray_id = r->id;
5074 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5075 instr->regarray.idx.n_bits = fidx->n_bits;
5076 instr->regarray.idx.offset = fidx->offset / 8;
5077 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5078 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5079 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5083 /* REGWR_RHI, REGWR_RMI. */
5084 if (fidx && !fsrc) {
5085 src_val = strtoull(src, &src, 0);
5086 CHECK(!src[0], EINVAL);
5088 instr->type = INSTR_REGWR_RMI;
5090 instr->type = INSTR_REGWR_RHI;
5092 instr->regarray.regarray_id = r->id;
5093 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5094 instr->regarray.idx.n_bits = fidx->n_bits;
5095 instr->regarray.idx.offset = fidx->offset / 8;
5096 instr->regarray.dstsrc_val = src_val;
5100 /* REGWR_RIH, REGWR_RIM. */
5101 if (!fidx && fsrc) {
5102 idx_val = strtoul(idx, &idx, 0);
5103 CHECK(!idx[0], EINVAL);
5105 instr->type = INSTR_REGWR_RIM;
5107 instr->type = INSTR_REGWR_RIH;
5109 instr->regarray.regarray_id = r->id;
5110 instr->regarray.idx_val = idx_val;
5111 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5112 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5113 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5118 src_val = strtoull(src, &src, 0);
5119 CHECK(!src[0], EINVAL);
5121 idx_val = strtoul(idx, &idx, 0);
5122 CHECK(!idx[0], EINVAL);
5124 instr->type = INSTR_REGWR_RII;
5125 instr->regarray.idx_val = idx_val;
5126 instr->regarray.dstsrc_val = src_val;
5132 instr_regadd_translate(struct rte_swx_pipeline *p,
5133 struct action *action,
5136 struct instruction *instr,
5137 struct instruction_data *data __rte_unused)
5139 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5141 struct field *fidx, *fsrc;
5143 uint32_t idx_struct_id, idx_val, src_struct_id;
5145 CHECK(n_tokens == 4, EINVAL);
5147 r = regarray_find(p, regarray);
5150 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5151 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5152 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5154 instr->type = INSTR_REGADD_RMM;
5155 if (idx[0] == 'h' && src[0] != 'h')
5156 instr->type = INSTR_REGADD_RHM;
5157 if (idx[0] != 'h' && src[0] == 'h')
5158 instr->type = INSTR_REGADD_RMH;
5159 if (idx[0] == 'h' && src[0] == 'h')
5160 instr->type = INSTR_REGADD_RHH;
5162 instr->regarray.regarray_id = r->id;
5163 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5164 instr->regarray.idx.n_bits = fidx->n_bits;
5165 instr->regarray.idx.offset = fidx->offset / 8;
5166 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5167 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5168 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5172 /* REGADD_RHI, REGADD_RMI. */
5173 if (fidx && !fsrc) {
5174 src_val = strtoull(src, &src, 0);
5175 CHECK(!src[0], EINVAL);
5177 instr->type = INSTR_REGADD_RMI;
5179 instr->type = INSTR_REGADD_RHI;
5181 instr->regarray.regarray_id = r->id;
5182 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5183 instr->regarray.idx.n_bits = fidx->n_bits;
5184 instr->regarray.idx.offset = fidx->offset / 8;
5185 instr->regarray.dstsrc_val = src_val;
5189 /* REGADD_RIH, REGADD_RIM. */
5190 if (!fidx && fsrc) {
5191 idx_val = strtoul(idx, &idx, 0);
5192 CHECK(!idx[0], EINVAL);
5194 instr->type = INSTR_REGADD_RIM;
5196 instr->type = INSTR_REGADD_RIH;
5198 instr->regarray.regarray_id = r->id;
5199 instr->regarray.idx_val = idx_val;
5200 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5201 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5202 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5207 src_val = strtoull(src, &src, 0);
5208 CHECK(!src[0], EINVAL);
5210 idx_val = strtoul(idx, &idx, 0);
5211 CHECK(!idx[0], EINVAL);
5213 instr->type = INSTR_REGADD_RII;
5214 instr->regarray.idx_val = idx_val;
5215 instr->regarray.dstsrc_val = src_val;
5219 static inline uint64_t *
5220 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5222 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5226 static inline uint64_t
5227 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5229 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5231 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5232 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5233 uint64_t idx64 = *idx64_ptr;
5234 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5235 uint64_t idx = idx64 & idx64_mask & r->size_mask;
5240 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5242 static inline uint64_t
5243 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5245 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5247 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5248 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5249 uint64_t idx64 = *idx64_ptr;
5250 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5257 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5261 static inline uint64_t
5262 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5264 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5266 uint64_t idx = ip->regarray.idx_val & r->size_mask;
5271 static inline uint64_t
5272 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5274 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5275 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5276 uint64_t src64 = *src64_ptr;
5277 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5278 uint64_t src = src64 & src64_mask;
5283 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5285 static inline uint64_t
5286 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5288 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5289 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5290 uint64_t src64 = *src64_ptr;
5291 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5298 #define instr_regarray_src_nbo instr_regarray_src_hbo
5303 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5305 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5306 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5307 uint64_t dst64 = *dst64_ptr;
5308 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5310 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5314 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5317 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5319 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5320 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5321 uint64_t dst64 = *dst64_ptr;
5322 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5324 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5325 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5330 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5335 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5337 struct thread *t = &p->threads[p->thread_id];
5338 struct instruction *ip = t->ip;
5339 uint64_t *regarray, idx;
5341 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5344 regarray = instr_regarray_regarray(p, ip);
5345 idx = instr_regarray_idx_nbo(p, t, ip);
5346 rte_prefetch0(®array[idx]);
5353 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5355 struct thread *t = &p->threads[p->thread_id];
5356 struct instruction *ip = t->ip;
5357 uint64_t *regarray, idx;
5359 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5362 regarray = instr_regarray_regarray(p, ip);
5363 idx = instr_regarray_idx_hbo(p, t, ip);
5364 rte_prefetch0(®array[idx]);
5371 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5373 struct thread *t = &p->threads[p->thread_id];
5374 struct instruction *ip = t->ip;
5375 uint64_t *regarray, idx;
5377 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5380 regarray = instr_regarray_regarray(p, ip);
5381 idx = instr_regarray_idx_imm(p, ip);
5382 rte_prefetch0(®array[idx]);
5389 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5391 struct thread *t = &p->threads[p->thread_id];
5392 struct instruction *ip = t->ip;
5393 uint64_t *regarray, idx;
5395 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5398 regarray = instr_regarray_regarray(p, ip);
5399 idx = instr_regarray_idx_nbo(p, t, ip);
5400 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5407 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5409 struct thread *t = &p->threads[p->thread_id];
5410 struct instruction *ip = t->ip;
5411 uint64_t *regarray, idx;
5413 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5416 regarray = instr_regarray_regarray(p, ip);
5417 idx = instr_regarray_idx_hbo(p, t, ip);
5418 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5425 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5427 struct thread *t = &p->threads[p->thread_id];
5428 struct instruction *ip = t->ip;
5429 uint64_t *regarray, idx;
5431 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5434 regarray = instr_regarray_regarray(p, ip);
5435 idx = instr_regarray_idx_nbo(p, t, ip);
5436 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5443 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5445 struct thread *t = &p->threads[p->thread_id];
5446 struct instruction *ip = t->ip;
5447 uint64_t *regarray, idx;
5450 regarray = instr_regarray_regarray(p, ip);
5451 idx = instr_regarray_idx_hbo(p, t, ip);
5452 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5459 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5461 struct thread *t = &p->threads[p->thread_id];
5462 struct instruction *ip = t->ip;
5463 uint64_t *regarray, idx;
5465 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5468 regarray = instr_regarray_regarray(p, ip);
5469 idx = instr_regarray_idx_imm(p, ip);
5470 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5477 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5479 struct thread *t = &p->threads[p->thread_id];
5480 struct instruction *ip = t->ip;
5481 uint64_t *regarray, idx;
5483 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5486 regarray = instr_regarray_regarray(p, ip);
5487 idx = instr_regarray_idx_imm(p, ip);
5488 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5495 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5497 struct thread *t = &p->threads[p->thread_id];
5498 struct instruction *ip = t->ip;
5499 uint64_t *regarray, idx, src;
5501 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5504 regarray = instr_regarray_regarray(p, ip);
5505 idx = instr_regarray_idx_nbo(p, t, ip);
5506 src = instr_regarray_src_nbo(t, ip);
5507 regarray[idx] = src;
5514 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5516 struct thread *t = &p->threads[p->thread_id];
5517 struct instruction *ip = t->ip;
5518 uint64_t *regarray, idx, src;
5520 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5523 regarray = instr_regarray_regarray(p, ip);
5524 idx = instr_regarray_idx_nbo(p, t, ip);
5525 src = instr_regarray_src_hbo(t, ip);
5526 regarray[idx] = src;
5533 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5535 struct thread *t = &p->threads[p->thread_id];
5536 struct instruction *ip = t->ip;
5537 uint64_t *regarray, idx, src;
5539 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5542 regarray = instr_regarray_regarray(p, ip);
5543 idx = instr_regarray_idx_hbo(p, t, ip);
5544 src = instr_regarray_src_nbo(t, ip);
5545 regarray[idx] = src;
5552 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5554 struct thread *t = &p->threads[p->thread_id];
5555 struct instruction *ip = t->ip;
5556 uint64_t *regarray, idx, src;
5558 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5561 regarray = instr_regarray_regarray(p, ip);
5562 idx = instr_regarray_idx_hbo(p, t, ip);
5563 src = instr_regarray_src_hbo(t, ip);
5564 regarray[idx] = src;
5571 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5573 struct thread *t = &p->threads[p->thread_id];
5574 struct instruction *ip = t->ip;
5575 uint64_t *regarray, idx, src;
5577 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5580 regarray = instr_regarray_regarray(p, ip);
5581 idx = instr_regarray_idx_nbo(p, t, ip);
5582 src = ip->regarray.dstsrc_val;
5583 regarray[idx] = src;
5590 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5592 struct thread *t = &p->threads[p->thread_id];
5593 struct instruction *ip = t->ip;
5594 uint64_t *regarray, idx, src;
5596 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5599 regarray = instr_regarray_regarray(p, ip);
5600 idx = instr_regarray_idx_hbo(p, t, ip);
5601 src = ip->regarray.dstsrc_val;
5602 regarray[idx] = src;
5609 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5611 struct thread *t = &p->threads[p->thread_id];
5612 struct instruction *ip = t->ip;
5613 uint64_t *regarray, idx, src;
5615 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5618 regarray = instr_regarray_regarray(p, ip);
5619 idx = instr_regarray_idx_imm(p, ip);
5620 src = instr_regarray_src_nbo(t, ip);
5621 regarray[idx] = src;
5628 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5630 struct thread *t = &p->threads[p->thread_id];
5631 struct instruction *ip = t->ip;
5632 uint64_t *regarray, idx, src;
5634 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5637 regarray = instr_regarray_regarray(p, ip);
5638 idx = instr_regarray_idx_imm(p, ip);
5639 src = instr_regarray_src_hbo(t, ip);
5640 regarray[idx] = src;
5647 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5649 struct thread *t = &p->threads[p->thread_id];
5650 struct instruction *ip = t->ip;
5651 uint64_t *regarray, idx, src;
5653 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5656 regarray = instr_regarray_regarray(p, ip);
5657 idx = instr_regarray_idx_imm(p, ip);
5658 src = ip->regarray.dstsrc_val;
5659 regarray[idx] = src;
5666 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5668 struct thread *t = &p->threads[p->thread_id];
5669 struct instruction *ip = t->ip;
5670 uint64_t *regarray, idx, src;
5672 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5675 regarray = instr_regarray_regarray(p, ip);
5676 idx = instr_regarray_idx_nbo(p, t, ip);
5677 src = instr_regarray_src_nbo(t, ip);
5678 regarray[idx] += src;
5685 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5687 struct thread *t = &p->threads[p->thread_id];
5688 struct instruction *ip = t->ip;
5689 uint64_t *regarray, idx, src;
5691 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5694 regarray = instr_regarray_regarray(p, ip);
5695 idx = instr_regarray_idx_nbo(p, t, ip);
5696 src = instr_regarray_src_hbo(t, ip);
5697 regarray[idx] += src;
5704 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5706 struct thread *t = &p->threads[p->thread_id];
5707 struct instruction *ip = t->ip;
5708 uint64_t *regarray, idx, src;
5710 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5713 regarray = instr_regarray_regarray(p, ip);
5714 idx = instr_regarray_idx_hbo(p, t, ip);
5715 src = instr_regarray_src_nbo(t, ip);
5716 regarray[idx] += src;
5723 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5725 struct thread *t = &p->threads[p->thread_id];
5726 struct instruction *ip = t->ip;
5727 uint64_t *regarray, idx, src;
5729 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5732 regarray = instr_regarray_regarray(p, ip);
5733 idx = instr_regarray_idx_hbo(p, t, ip);
5734 src = instr_regarray_src_hbo(t, ip);
5735 regarray[idx] += src;
5742 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5744 struct thread *t = &p->threads[p->thread_id];
5745 struct instruction *ip = t->ip;
5746 uint64_t *regarray, idx, src;
5748 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5751 regarray = instr_regarray_regarray(p, ip);
5752 idx = instr_regarray_idx_nbo(p, t, ip);
5753 src = ip->regarray.dstsrc_val;
5754 regarray[idx] += src;
5761 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5763 struct thread *t = &p->threads[p->thread_id];
5764 struct instruction *ip = t->ip;
5765 uint64_t *regarray, idx, src;
5767 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
5770 regarray = instr_regarray_regarray(p, ip);
5771 idx = instr_regarray_idx_hbo(p, t, ip);
5772 src = ip->regarray.dstsrc_val;
5773 regarray[idx] += src;
5780 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
5782 struct thread *t = &p->threads[p->thread_id];
5783 struct instruction *ip = t->ip;
5784 uint64_t *regarray, idx, src;
5786 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
5789 regarray = instr_regarray_regarray(p, ip);
5790 idx = instr_regarray_idx_imm(p, ip);
5791 src = instr_regarray_src_nbo(t, ip);
5792 regarray[idx] += src;
5799 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
5801 struct thread *t = &p->threads[p->thread_id];
5802 struct instruction *ip = t->ip;
5803 uint64_t *regarray, idx, src;
5805 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
5808 regarray = instr_regarray_regarray(p, ip);
5809 idx = instr_regarray_idx_imm(p, ip);
5810 src = instr_regarray_src_hbo(t, ip);
5811 regarray[idx] += src;
5818 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
5820 struct thread *t = &p->threads[p->thread_id];
5821 struct instruction *ip = t->ip;
5822 uint64_t *regarray, idx, src;
5824 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
5827 regarray = instr_regarray_regarray(p, ip);
5828 idx = instr_regarray_idx_imm(p, ip);
5829 src = ip->regarray.dstsrc_val;
5830 regarray[idx] += src;
5839 static struct metarray *
5840 metarray_find(struct rte_swx_pipeline *p, const char *name);
5843 instr_metprefetch_translate(struct rte_swx_pipeline *p,
5844 struct action *action,
5847 struct instruction *instr,
5848 struct instruction_data *data __rte_unused)
5850 char *metarray = tokens[1], *idx = tokens[2];
5853 uint32_t idx_struct_id, idx_val;
5855 CHECK(n_tokens == 3, EINVAL);
5857 m = metarray_find(p, metarray);
5860 /* METPREFETCH_H, METPREFETCH_M. */
5861 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5863 instr->type = INSTR_METPREFETCH_M;
5865 instr->type = INSTR_METPREFETCH_H;
5867 instr->meter.metarray_id = m->id;
5868 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5869 instr->meter.idx.n_bits = fidx->n_bits;
5870 instr->meter.idx.offset = fidx->offset / 8;
5874 /* METPREFETCH_I. */
5875 idx_val = strtoul(idx, &idx, 0);
5876 CHECK(!idx[0], EINVAL);
5878 instr->type = INSTR_METPREFETCH_I;
5879 instr->meter.metarray_id = m->id;
5880 instr->meter.idx_val = idx_val;
5885 instr_meter_translate(struct rte_swx_pipeline *p,
5886 struct action *action,
5889 struct instruction *instr,
5890 struct instruction_data *data __rte_unused)
5892 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
5893 char *color_in = tokens[4], *color_out = tokens[5];
5895 struct field *fidx, *flength, *fcin, *fcout;
5896 uint32_t idx_struct_id, length_struct_id;
5897 uint32_t color_in_struct_id, color_out_struct_id;
5899 CHECK(n_tokens == 6, EINVAL);
5901 m = metarray_find(p, metarray);
5904 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5906 flength = struct_field_parse(p, action, length, &length_struct_id);
5907 CHECK(flength, EINVAL);
5909 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
5911 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
5912 CHECK(fcout, EINVAL);
5914 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
5916 instr->type = INSTR_METER_MMM;
5917 if (idx[0] == 'h' && length[0] == 'h')
5918 instr->type = INSTR_METER_HHM;
5919 if (idx[0] == 'h' && length[0] != 'h')
5920 instr->type = INSTR_METER_HMM;
5921 if (idx[0] != 'h' && length[0] == 'h')
5922 instr->type = INSTR_METER_MHM;
5924 instr->meter.metarray_id = m->id;
5926 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5927 instr->meter.idx.n_bits = fidx->n_bits;
5928 instr->meter.idx.offset = fidx->offset / 8;
5930 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5931 instr->meter.length.n_bits = flength->n_bits;
5932 instr->meter.length.offset = flength->offset / 8;
5934 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5935 instr->meter.color_in.n_bits = fcin->n_bits;
5936 instr->meter.color_in.offset = fcin->offset / 8;
5938 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5939 instr->meter.color_out.n_bits = fcout->n_bits;
5940 instr->meter.color_out.offset = fcout->offset / 8;
5945 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
5946 if (fidx && !fcin) {
5947 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
5948 CHECK(!color_in[0], EINVAL);
5950 instr->type = INSTR_METER_MMI;
5951 if (idx[0] == 'h' && length[0] == 'h')
5952 instr->type = INSTR_METER_HHI;
5953 if (idx[0] == 'h' && length[0] != 'h')
5954 instr->type = INSTR_METER_HMI;
5955 if (idx[0] != 'h' && length[0] == 'h')
5956 instr->type = INSTR_METER_MHI;
5958 instr->meter.metarray_id = m->id;
5960 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
5961 instr->meter.idx.n_bits = fidx->n_bits;
5962 instr->meter.idx.offset = fidx->offset / 8;
5964 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5965 instr->meter.length.n_bits = flength->n_bits;
5966 instr->meter.length.offset = flength->offset / 8;
5968 instr->meter.color_in_val = color_in_val;
5970 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
5971 instr->meter.color_out.n_bits = fcout->n_bits;
5972 instr->meter.color_out.offset = fcout->offset / 8;
5977 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
5978 if (!fidx && fcin) {
5981 idx_val = strtoul(idx, &idx, 0);
5982 CHECK(!idx[0], EINVAL);
5984 instr->type = INSTR_METER_IMM;
5985 if (length[0] == 'h')
5986 instr->type = INSTR_METER_IHM;
5988 instr->meter.metarray_id = m->id;
5990 instr->meter.idx_val = idx_val;
5992 instr->meter.length.struct_id = (uint8_t)length_struct_id;
5993 instr->meter.length.n_bits = flength->n_bits;
5994 instr->meter.length.offset = flength->offset / 8;
5996 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
5997 instr->meter.color_in.n_bits = fcin->n_bits;
5998 instr->meter.color_in.offset = fcin->offset / 8;
6000 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6001 instr->meter.color_out.n_bits = fcout->n_bits;
6002 instr->meter.color_out.offset = fcout->offset / 8;
6007 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6008 if (!fidx && !fcin) {
6009 uint32_t idx_val, color_in_val;
6011 idx_val = strtoul(idx, &idx, 0);
6012 CHECK(!idx[0], EINVAL);
6014 color_in_val = strtoul(color_in, &color_in, 0);
6015 CHECK(!color_in[0], EINVAL);
6017 instr->type = INSTR_METER_IMI;
6018 if (length[0] == 'h')
6019 instr->type = INSTR_METER_IHI;
6021 instr->meter.metarray_id = m->id;
6023 instr->meter.idx_val = idx_val;
6025 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6026 instr->meter.length.n_bits = flength->n_bits;
6027 instr->meter.length.offset = flength->offset / 8;
6029 instr->meter.color_in_val = color_in_val;
6031 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6032 instr->meter.color_out.n_bits = fcout->n_bits;
6033 instr->meter.color_out.offset = fcout->offset / 8;
6041 static inline struct meter *
6042 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6044 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6046 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6047 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6048 uint64_t idx64 = *idx64_ptr;
6049 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6050 uint64_t idx = idx64 & idx64_mask & r->size_mask;
6052 return &r->metarray[idx];
6055 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6057 static inline struct meter *
6058 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6060 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6062 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6063 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6064 uint64_t idx64 = *idx64_ptr;
6065 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6067 return &r->metarray[idx];
6072 #define instr_meter_idx_nbo instr_meter_idx_hbo
6076 static inline struct meter *
6077 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6079 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6081 uint64_t idx = ip->meter.idx_val & r->size_mask;
6083 return &r->metarray[idx];
6086 static inline uint32_t
6087 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6089 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6090 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6091 uint64_t src64 = *src64_ptr;
6092 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6093 uint64_t src = src64 & src64_mask;
6095 return (uint32_t)src;
6098 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6100 static inline uint32_t
6101 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6103 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6104 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6105 uint64_t src64 = *src64_ptr;
6106 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6108 return (uint32_t)src;
6113 #define instr_meter_length_nbo instr_meter_length_hbo
6117 static inline enum rte_color
6118 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6120 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6121 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6122 uint64_t src64 = *src64_ptr;
6123 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6124 uint64_t src = src64 & src64_mask;
6126 return (enum rte_color)src;
6130 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6132 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6133 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6134 uint64_t dst64 = *dst64_ptr;
6135 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6137 uint64_t src = (uint64_t)color_out;
6139 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6143 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6145 struct thread *t = &p->threads[p->thread_id];
6146 struct instruction *ip = t->ip;
6149 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6152 m = instr_meter_idx_nbo(p, t, ip);
6160 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6162 struct thread *t = &p->threads[p->thread_id];
6163 struct instruction *ip = t->ip;
6166 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6169 m = instr_meter_idx_hbo(p, t, ip);
6177 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6179 struct thread *t = &p->threads[p->thread_id];
6180 struct instruction *ip = t->ip;
6183 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6186 m = instr_meter_idx_imm(p, ip);
6194 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6196 struct thread *t = &p->threads[p->thread_id];
6197 struct instruction *ip = t->ip;
6199 uint64_t time, n_pkts, n_bytes;
6201 enum rte_color color_in, color_out;
6203 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6206 m = instr_meter_idx_nbo(p, t, ip);
6207 rte_prefetch0(m->n_pkts);
6208 time = rte_get_tsc_cycles();
6209 length = instr_meter_length_nbo(t, ip);
6210 color_in = instr_meter_color_in_hbo(t, ip);
6212 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6213 &m->profile->profile,
6218 color_out &= m->color_mask;
6220 n_pkts = m->n_pkts[color_out];
6221 n_bytes = m->n_bytes[color_out];
6223 instr_meter_color_out_hbo_set(t, ip, color_out);
6225 m->n_pkts[color_out] = n_pkts + 1;
6226 m->n_bytes[color_out] = n_bytes + length;
6233 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6235 struct thread *t = &p->threads[p->thread_id];
6236 struct instruction *ip = t->ip;
6238 uint64_t time, n_pkts, n_bytes;
6240 enum rte_color color_in, color_out;
6242 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6245 m = instr_meter_idx_nbo(p, t, ip);
6246 rte_prefetch0(m->n_pkts);
6247 time = rte_get_tsc_cycles();
6248 length = instr_meter_length_nbo(t, ip);
6249 color_in = (enum rte_color)ip->meter.color_in_val;
6251 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6252 &m->profile->profile,
6257 color_out &= m->color_mask;
6259 n_pkts = m->n_pkts[color_out];
6260 n_bytes = m->n_bytes[color_out];
6262 instr_meter_color_out_hbo_set(t, ip, color_out);
6264 m->n_pkts[color_out] = n_pkts + 1;
6265 m->n_bytes[color_out] = n_bytes + length;
6272 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6274 struct thread *t = &p->threads[p->thread_id];
6275 struct instruction *ip = t->ip;
6277 uint64_t time, n_pkts, n_bytes;
6279 enum rte_color color_in, color_out;
6281 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6284 m = instr_meter_idx_nbo(p, t, ip);
6285 rte_prefetch0(m->n_pkts);
6286 time = rte_get_tsc_cycles();
6287 length = instr_meter_length_hbo(t, ip);
6288 color_in = instr_meter_color_in_hbo(t, ip);
6290 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6291 &m->profile->profile,
6296 color_out &= m->color_mask;
6298 n_pkts = m->n_pkts[color_out];
6299 n_bytes = m->n_bytes[color_out];
6301 instr_meter_color_out_hbo_set(t, ip, color_out);
6303 m->n_pkts[color_out] = n_pkts + 1;
6304 m->n_bytes[color_out] = n_bytes + length;
6310 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6312 struct thread *t = &p->threads[p->thread_id];
6313 struct instruction *ip = t->ip;
6315 uint64_t time, n_pkts, n_bytes;
6317 enum rte_color color_in, color_out;
6319 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6322 m = instr_meter_idx_nbo(p, t, ip);
6323 rte_prefetch0(m->n_pkts);
6324 time = rte_get_tsc_cycles();
6325 length = instr_meter_length_hbo(t, ip);
6326 color_in = (enum rte_color)ip->meter.color_in_val;
6328 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6329 &m->profile->profile,
6334 color_out &= m->color_mask;
6336 n_pkts = m->n_pkts[color_out];
6337 n_bytes = m->n_bytes[color_out];
6339 instr_meter_color_out_hbo_set(t, ip, color_out);
6341 m->n_pkts[color_out] = n_pkts + 1;
6342 m->n_bytes[color_out] = n_bytes + length;
6349 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6351 struct thread *t = &p->threads[p->thread_id];
6352 struct instruction *ip = t->ip;
6354 uint64_t time, n_pkts, n_bytes;
6356 enum rte_color color_in, color_out;
6358 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6361 m = instr_meter_idx_hbo(p, t, ip);
6362 rte_prefetch0(m->n_pkts);
6363 time = rte_get_tsc_cycles();
6364 length = instr_meter_length_nbo(t, ip);
6365 color_in = instr_meter_color_in_hbo(t, ip);
6367 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6368 &m->profile->profile,
6373 color_out &= m->color_mask;
6375 n_pkts = m->n_pkts[color_out];
6376 n_bytes = m->n_bytes[color_out];
6378 instr_meter_color_out_hbo_set(t, ip, color_out);
6380 m->n_pkts[color_out] = n_pkts + 1;
6381 m->n_bytes[color_out] = n_bytes + length;
6388 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6390 struct thread *t = &p->threads[p->thread_id];
6391 struct instruction *ip = t->ip;
6393 uint64_t time, n_pkts, n_bytes;
6395 enum rte_color color_in, color_out;
6397 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6400 m = instr_meter_idx_hbo(p, t, ip);
6401 rte_prefetch0(m->n_pkts);
6402 time = rte_get_tsc_cycles();
6403 length = instr_meter_length_nbo(t, ip);
6404 color_in = (enum rte_color)ip->meter.color_in_val;
6406 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6407 &m->profile->profile,
6412 color_out &= m->color_mask;
6414 n_pkts = m->n_pkts[color_out];
6415 n_bytes = m->n_bytes[color_out];
6417 instr_meter_color_out_hbo_set(t, ip, color_out);
6419 m->n_pkts[color_out] = n_pkts + 1;
6420 m->n_bytes[color_out] = n_bytes + length;
6427 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6429 struct thread *t = &p->threads[p->thread_id];
6430 struct instruction *ip = t->ip;
6432 uint64_t time, n_pkts, n_bytes;
6434 enum rte_color color_in, color_out;
6436 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6439 m = instr_meter_idx_hbo(p, t, ip);
6440 rte_prefetch0(m->n_pkts);
6441 time = rte_get_tsc_cycles();
6442 length = instr_meter_length_hbo(t, ip);
6443 color_in = instr_meter_color_in_hbo(t, ip);
6445 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6446 &m->profile->profile,
6451 color_out &= m->color_mask;
6453 n_pkts = m->n_pkts[color_out];
6454 n_bytes = m->n_bytes[color_out];
6456 instr_meter_color_out_hbo_set(t, ip, color_out);
6458 m->n_pkts[color_out] = n_pkts + 1;
6459 m->n_bytes[color_out] = n_bytes + length;
6466 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6468 struct thread *t = &p->threads[p->thread_id];
6469 struct instruction *ip = t->ip;
6471 uint64_t time, n_pkts, n_bytes;
6473 enum rte_color color_in, color_out;
6475 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6478 m = instr_meter_idx_hbo(p, t, ip);
6479 rte_prefetch0(m->n_pkts);
6480 time = rte_get_tsc_cycles();
6481 length = instr_meter_length_hbo(t, ip);
6482 color_in = (enum rte_color)ip->meter.color_in_val;
6484 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6485 &m->profile->profile,
6490 color_out &= m->color_mask;
6492 n_pkts = m->n_pkts[color_out];
6493 n_bytes = m->n_bytes[color_out];
6495 instr_meter_color_out_hbo_set(t, ip, color_out);
6497 m->n_pkts[color_out] = n_pkts + 1;
6498 m->n_bytes[color_out] = n_bytes + length;
6505 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6507 struct thread *t = &p->threads[p->thread_id];
6508 struct instruction *ip = t->ip;
6510 uint64_t time, n_pkts, n_bytes;
6512 enum rte_color color_in, color_out;
6514 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6517 m = instr_meter_idx_imm(p, ip);
6518 rte_prefetch0(m->n_pkts);
6519 time = rte_get_tsc_cycles();
6520 length = instr_meter_length_nbo(t, ip);
6521 color_in = instr_meter_color_in_hbo(t, ip);
6523 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6524 &m->profile->profile,
6529 color_out &= m->color_mask;
6531 n_pkts = m->n_pkts[color_out];
6532 n_bytes = m->n_bytes[color_out];
6534 instr_meter_color_out_hbo_set(t, ip, color_out);
6536 m->n_pkts[color_out] = n_pkts + 1;
6537 m->n_bytes[color_out] = n_bytes + length;
6544 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6546 struct thread *t = &p->threads[p->thread_id];
6547 struct instruction *ip = t->ip;
6549 uint64_t time, n_pkts, n_bytes;
6551 enum rte_color color_in, color_out;
6553 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6556 m = instr_meter_idx_imm(p, ip);
6557 rte_prefetch0(m->n_pkts);
6558 time = rte_get_tsc_cycles();
6559 length = instr_meter_length_nbo(t, ip);
6560 color_in = (enum rte_color)ip->meter.color_in_val;
6562 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6563 &m->profile->profile,
6568 color_out &= m->color_mask;
6570 n_pkts = m->n_pkts[color_out];
6571 n_bytes = m->n_bytes[color_out];
6573 instr_meter_color_out_hbo_set(t, ip, color_out);
6575 m->n_pkts[color_out] = n_pkts + 1;
6576 m->n_bytes[color_out] = n_bytes + length;
6583 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6585 struct thread *t = &p->threads[p->thread_id];
6586 struct instruction *ip = t->ip;
6588 uint64_t time, n_pkts, n_bytes;
6590 enum rte_color color_in, color_out;
6592 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6595 m = instr_meter_idx_imm(p, ip);
6596 rte_prefetch0(m->n_pkts);
6597 time = rte_get_tsc_cycles();
6598 length = instr_meter_length_hbo(t, ip);
6599 color_in = instr_meter_color_in_hbo(t, ip);
6601 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6602 &m->profile->profile,
6607 color_out &= m->color_mask;
6609 n_pkts = m->n_pkts[color_out];
6610 n_bytes = m->n_bytes[color_out];
6612 instr_meter_color_out_hbo_set(t, ip, color_out);
6614 m->n_pkts[color_out] = n_pkts + 1;
6615 m->n_bytes[color_out] = n_bytes + length;
6621 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6623 struct thread *t = &p->threads[p->thread_id];
6624 struct instruction *ip = t->ip;
6626 uint64_t time, n_pkts, n_bytes;
6628 enum rte_color color_in, color_out;
6630 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6633 m = instr_meter_idx_imm(p, ip);
6634 rte_prefetch0(m->n_pkts);
6635 time = rte_get_tsc_cycles();
6636 length = instr_meter_length_hbo(t, ip);
6637 color_in = (enum rte_color)ip->meter.color_in_val;
6639 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6640 &m->profile->profile,
6645 color_out &= m->color_mask;
6647 n_pkts = m->n_pkts[color_out];
6648 n_bytes = m->n_bytes[color_out];
6650 instr_meter_color_out_hbo_set(t, ip, color_out);
6652 m->n_pkts[color_out] = n_pkts + 1;
6653 m->n_bytes[color_out] = n_bytes + length;
6662 static struct action *
6663 action_find(struct rte_swx_pipeline *p, const char *name);
6666 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6667 struct action *action __rte_unused,
6670 struct instruction *instr,
6671 struct instruction_data *data)
6673 CHECK(n_tokens == 2, EINVAL);
6675 strcpy(data->jmp_label, tokens[1]);
6677 instr->type = INSTR_JMP;
6678 instr->jmp.ip = NULL; /* Resolved later. */
6683 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6684 struct action *action __rte_unused,
6687 struct instruction *instr,
6688 struct instruction_data *data)
6692 CHECK(n_tokens == 3, EINVAL);
6694 strcpy(data->jmp_label, tokens[1]);
6696 h = header_parse(p, tokens[2]);
6699 instr->type = INSTR_JMP_VALID;
6700 instr->jmp.ip = NULL; /* Resolved later. */
6701 instr->jmp.header_id = h->id;
6706 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6707 struct action *action __rte_unused,
6710 struct instruction *instr,
6711 struct instruction_data *data)
6715 CHECK(n_tokens == 3, EINVAL);
6717 strcpy(data->jmp_label, tokens[1]);
6719 h = header_parse(p, tokens[2]);
6722 instr->type = INSTR_JMP_INVALID;
6723 instr->jmp.ip = NULL; /* Resolved later. */
6724 instr->jmp.header_id = h->id;
6729 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6730 struct action *action,
6733 struct instruction *instr,
6734 struct instruction_data *data)
6736 CHECK(!action, EINVAL);
6737 CHECK(n_tokens == 2, EINVAL);
6739 strcpy(data->jmp_label, tokens[1]);
6741 instr->type = INSTR_JMP_HIT;
6742 instr->jmp.ip = NULL; /* Resolved later. */
6747 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6748 struct action *action,
6751 struct instruction *instr,
6752 struct instruction_data *data)
6754 CHECK(!action, EINVAL);
6755 CHECK(n_tokens == 2, EINVAL);
6757 strcpy(data->jmp_label, tokens[1]);
6759 instr->type = INSTR_JMP_MISS;
6760 instr->jmp.ip = NULL; /* Resolved later. */
6765 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6766 struct action *action,
6769 struct instruction *instr,
6770 struct instruction_data *data)
6774 CHECK(!action, EINVAL);
6775 CHECK(n_tokens == 3, EINVAL);
6777 strcpy(data->jmp_label, tokens[1]);
6779 a = action_find(p, tokens[2]);
6782 instr->type = INSTR_JMP_ACTION_HIT;
6783 instr->jmp.ip = NULL; /* Resolved later. */
6784 instr->jmp.action_id = a->id;
6789 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
6790 struct action *action,
6793 struct instruction *instr,
6794 struct instruction_data *data)
6798 CHECK(!action, EINVAL);
6799 CHECK(n_tokens == 3, EINVAL);
6801 strcpy(data->jmp_label, tokens[1]);
6803 a = action_find(p, tokens[2]);
6806 instr->type = INSTR_JMP_ACTION_MISS;
6807 instr->jmp.ip = NULL; /* Resolved later. */
6808 instr->jmp.action_id = a->id;
6813 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
6814 struct action *action,
6817 struct instruction *instr,
6818 struct instruction_data *data)
6820 char *a = tokens[2], *b = tokens[3];
6821 struct field *fa, *fb;
6823 uint32_t a_struct_id, b_struct_id;
6825 CHECK(n_tokens == 4, EINVAL);
6827 strcpy(data->jmp_label, tokens[1]);
6829 fa = struct_field_parse(p, action, a, &a_struct_id);
6832 /* JMP_EQ or JMP_EQ_S. */
6833 fb = struct_field_parse(p, action, b, &b_struct_id);
6835 instr->type = INSTR_JMP_EQ;
6836 if ((a[0] == 'h' && b[0] != 'h') ||
6837 (a[0] != 'h' && b[0] == 'h'))
6838 instr->type = INSTR_JMP_EQ_S;
6839 instr->jmp.ip = NULL; /* Resolved later. */
6841 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6842 instr->jmp.a.n_bits = fa->n_bits;
6843 instr->jmp.a.offset = fa->offset / 8;
6844 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6845 instr->jmp.b.n_bits = fb->n_bits;
6846 instr->jmp.b.offset = fb->offset / 8;
6851 b_val = strtoull(b, &b, 0);
6852 CHECK(!b[0], EINVAL);
6855 b_val = hton64(b_val) >> (64 - fa->n_bits);
6857 instr->type = INSTR_JMP_EQ_I;
6858 instr->jmp.ip = NULL; /* Resolved later. */
6859 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6860 instr->jmp.a.n_bits = fa->n_bits;
6861 instr->jmp.a.offset = fa->offset / 8;
6862 instr->jmp.b_val = b_val;
6867 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
6868 struct action *action,
6871 struct instruction *instr,
6872 struct instruction_data *data)
6874 char *a = tokens[2], *b = tokens[3];
6875 struct field *fa, *fb;
6877 uint32_t a_struct_id, b_struct_id;
6879 CHECK(n_tokens == 4, EINVAL);
6881 strcpy(data->jmp_label, tokens[1]);
6883 fa = struct_field_parse(p, action, a, &a_struct_id);
6886 /* JMP_NEQ or JMP_NEQ_S. */
6887 fb = struct_field_parse(p, action, b, &b_struct_id);
6889 instr->type = INSTR_JMP_NEQ;
6890 if ((a[0] == 'h' && b[0] != 'h') ||
6891 (a[0] != 'h' && b[0] == 'h'))
6892 instr->type = INSTR_JMP_NEQ_S;
6893 instr->jmp.ip = NULL; /* Resolved later. */
6895 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6896 instr->jmp.a.n_bits = fa->n_bits;
6897 instr->jmp.a.offset = fa->offset / 8;
6898 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6899 instr->jmp.b.n_bits = fb->n_bits;
6900 instr->jmp.b.offset = fb->offset / 8;
6905 b_val = strtoull(b, &b, 0);
6906 CHECK(!b[0], EINVAL);
6909 b_val = hton64(b_val) >> (64 - fa->n_bits);
6911 instr->type = INSTR_JMP_NEQ_I;
6912 instr->jmp.ip = NULL; /* Resolved later. */
6913 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6914 instr->jmp.a.n_bits = fa->n_bits;
6915 instr->jmp.a.offset = fa->offset / 8;
6916 instr->jmp.b_val = b_val;
6921 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
6922 struct action *action,
6925 struct instruction *instr,
6926 struct instruction_data *data)
6928 char *a = tokens[2], *b = tokens[3];
6929 struct field *fa, *fb;
6931 uint32_t a_struct_id, b_struct_id;
6933 CHECK(n_tokens == 4, EINVAL);
6935 strcpy(data->jmp_label, tokens[1]);
6937 fa = struct_field_parse(p, action, a, &a_struct_id);
6940 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
6941 fb = struct_field_parse(p, action, b, &b_struct_id);
6943 instr->type = INSTR_JMP_LT;
6944 if (a[0] == 'h' && b[0] != 'h')
6945 instr->type = INSTR_JMP_LT_HM;
6946 if (a[0] != 'h' && b[0] == 'h')
6947 instr->type = INSTR_JMP_LT_MH;
6948 if (a[0] == 'h' && b[0] == 'h')
6949 instr->type = INSTR_JMP_LT_HH;
6950 instr->jmp.ip = NULL; /* Resolved later. */
6952 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6953 instr->jmp.a.n_bits = fa->n_bits;
6954 instr->jmp.a.offset = fa->offset / 8;
6955 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
6956 instr->jmp.b.n_bits = fb->n_bits;
6957 instr->jmp.b.offset = fb->offset / 8;
6961 /* JMP_LT_MI, JMP_LT_HI. */
6962 b_val = strtoull(b, &b, 0);
6963 CHECK(!b[0], EINVAL);
6965 instr->type = INSTR_JMP_LT_MI;
6967 instr->type = INSTR_JMP_LT_HI;
6968 instr->jmp.ip = NULL; /* Resolved later. */
6970 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
6971 instr->jmp.a.n_bits = fa->n_bits;
6972 instr->jmp.a.offset = fa->offset / 8;
6973 instr->jmp.b_val = b_val;
6978 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
6979 struct action *action,
6982 struct instruction *instr,
6983 struct instruction_data *data)
6985 char *a = tokens[2], *b = tokens[3];
6986 struct field *fa, *fb;
6988 uint32_t a_struct_id, b_struct_id;
6990 CHECK(n_tokens == 4, EINVAL);
6992 strcpy(data->jmp_label, tokens[1]);
6994 fa = struct_field_parse(p, action, a, &a_struct_id);
6997 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
6998 fb = struct_field_parse(p, action, b, &b_struct_id);
7000 instr->type = INSTR_JMP_GT;
7001 if (a[0] == 'h' && b[0] != 'h')
7002 instr->type = INSTR_JMP_GT_HM;
7003 if (a[0] != 'h' && b[0] == 'h')
7004 instr->type = INSTR_JMP_GT_MH;
7005 if (a[0] == 'h' && b[0] == 'h')
7006 instr->type = INSTR_JMP_GT_HH;
7007 instr->jmp.ip = NULL; /* Resolved later. */
7009 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7010 instr->jmp.a.n_bits = fa->n_bits;
7011 instr->jmp.a.offset = fa->offset / 8;
7012 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7013 instr->jmp.b.n_bits = fb->n_bits;
7014 instr->jmp.b.offset = fb->offset / 8;
7018 /* JMP_GT_MI, JMP_GT_HI. */
7019 b_val = strtoull(b, &b, 0);
7020 CHECK(!b[0], EINVAL);
7022 instr->type = INSTR_JMP_GT_MI;
7024 instr->type = INSTR_JMP_GT_HI;
7025 instr->jmp.ip = NULL; /* Resolved later. */
7027 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7028 instr->jmp.a.n_bits = fa->n_bits;
7029 instr->jmp.a.offset = fa->offset / 8;
7030 instr->jmp.b_val = b_val;
7035 instr_jmp_exec(struct rte_swx_pipeline *p)
7037 struct thread *t = &p->threads[p->thread_id];
7038 struct instruction *ip = t->ip;
7040 TRACE("[Thread %2u] jmp\n", p->thread_id);
7042 thread_ip_set(t, ip->jmp.ip);
7046 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7048 struct thread *t = &p->threads[p->thread_id];
7049 struct instruction *ip = t->ip;
7050 uint32_t header_id = ip->jmp.header_id;
7052 TRACE("[Thread %2u] jmpv\n", p->thread_id);
7054 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7058 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7060 struct thread *t = &p->threads[p->thread_id];
7061 struct instruction *ip = t->ip;
7062 uint32_t header_id = ip->jmp.header_id;
7064 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7066 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7070 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7072 struct thread *t = &p->threads[p->thread_id];
7073 struct instruction *ip = t->ip;
7074 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7076 TRACE("[Thread %2u] jmph\n", p->thread_id);
7078 t->ip = ip_next[t->hit];
7082 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7084 struct thread *t = &p->threads[p->thread_id];
7085 struct instruction *ip = t->ip;
7086 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7088 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7090 t->ip = ip_next[t->hit];
7094 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7096 struct thread *t = &p->threads[p->thread_id];
7097 struct instruction *ip = t->ip;
7099 TRACE("[Thread %2u] jmpa\n", p->thread_id);
7101 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7105 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7107 struct thread *t = &p->threads[p->thread_id];
7108 struct instruction *ip = t->ip;
7110 TRACE("[Thread %2u] jmpna\n", p->thread_id);
7112 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7116 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7118 struct thread *t = &p->threads[p->thread_id];
7119 struct instruction *ip = t->ip;
7121 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7127 instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
7129 struct thread *t = &p->threads[p->thread_id];
7130 struct instruction *ip = t->ip;
7132 TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
7134 JMP_CMP_S(t, ip, ==);
7138 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7140 struct thread *t = &p->threads[p->thread_id];
7141 struct instruction *ip = t->ip;
7143 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7145 JMP_CMP_I(t, ip, ==);
7149 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7151 struct thread *t = &p->threads[p->thread_id];
7152 struct instruction *ip = t->ip;
7154 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7160 instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
7162 struct thread *t = &p->threads[p->thread_id];
7163 struct instruction *ip = t->ip;
7165 TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
7167 JMP_CMP_S(t, ip, !=);
7171 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7173 struct thread *t = &p->threads[p->thread_id];
7174 struct instruction *ip = t->ip;
7176 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7178 JMP_CMP_I(t, ip, !=);
7182 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7184 struct thread *t = &p->threads[p->thread_id];
7185 struct instruction *ip = t->ip;
7187 TRACE("[Thread %2u] jmplt\n", p->thread_id);
7193 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7195 struct thread *t = &p->threads[p->thread_id];
7196 struct instruction *ip = t->ip;
7198 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7200 JMP_CMP_MH(t, ip, <);
7204 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7206 struct thread *t = &p->threads[p->thread_id];
7207 struct instruction *ip = t->ip;
7209 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7211 JMP_CMP_HM(t, ip, <);
7215 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7217 struct thread *t = &p->threads[p->thread_id];
7218 struct instruction *ip = t->ip;
7220 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7222 JMP_CMP_HH(t, ip, <);
7226 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7228 struct thread *t = &p->threads[p->thread_id];
7229 struct instruction *ip = t->ip;
7231 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7233 JMP_CMP_MI(t, ip, <);
7237 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7239 struct thread *t = &p->threads[p->thread_id];
7240 struct instruction *ip = t->ip;
7242 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7244 JMP_CMP_HI(t, ip, <);
7248 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7250 struct thread *t = &p->threads[p->thread_id];
7251 struct instruction *ip = t->ip;
7253 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7259 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7261 struct thread *t = &p->threads[p->thread_id];
7262 struct instruction *ip = t->ip;
7264 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7266 JMP_CMP_MH(t, ip, >);
7270 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7272 struct thread *t = &p->threads[p->thread_id];
7273 struct instruction *ip = t->ip;
7275 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7277 JMP_CMP_HM(t, ip, >);
7281 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7283 struct thread *t = &p->threads[p->thread_id];
7284 struct instruction *ip = t->ip;
7286 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7288 JMP_CMP_HH(t, ip, >);
7292 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7294 struct thread *t = &p->threads[p->thread_id];
7295 struct instruction *ip = t->ip;
7297 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7299 JMP_CMP_MI(t, ip, >);
7303 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7305 struct thread *t = &p->threads[p->thread_id];
7306 struct instruction *ip = t->ip;
7308 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7310 JMP_CMP_HI(t, ip, >);
7317 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7318 struct action *action,
7319 char **tokens __rte_unused,
7321 struct instruction *instr,
7322 struct instruction_data *data __rte_unused)
7324 CHECK(action, EINVAL);
7325 CHECK(n_tokens == 1, EINVAL);
7327 instr->type = INSTR_RETURN;
7332 instr_return_exec(struct rte_swx_pipeline *p)
7334 struct thread *t = &p->threads[p->thread_id];
7336 TRACE("[Thread %2u] return\n", p->thread_id);
7342 instr_translate(struct rte_swx_pipeline *p,
7343 struct action *action,
7345 struct instruction *instr,
7346 struct instruction_data *data)
7348 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7349 int n_tokens = 0, tpos = 0;
7351 /* Parse the instruction string into tokens. */
7355 token = strtok_r(string, " \t\v", &string);
7359 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7360 CHECK_NAME(token, EINVAL);
7362 tokens[n_tokens] = token;
7366 CHECK(n_tokens, EINVAL);
7368 /* Handle the optional instruction label. */
7369 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7370 strcpy(data->label, tokens[0]);
7373 CHECK(n_tokens - tpos, EINVAL);
7376 /* Identify the instruction type. */
7377 if (!strcmp(tokens[tpos], "rx"))
7378 return instr_rx_translate(p,
7385 if (!strcmp(tokens[tpos], "tx"))
7386 return instr_tx_translate(p,
7393 if (!strcmp(tokens[tpos], "drop"))
7394 return instr_drop_translate(p,
7401 if (!strcmp(tokens[tpos], "extract"))
7402 return instr_hdr_extract_translate(p,
7409 if (!strcmp(tokens[tpos], "emit"))
7410 return instr_hdr_emit_translate(p,
7417 if (!strcmp(tokens[tpos], "validate"))
7418 return instr_hdr_validate_translate(p,
7425 if (!strcmp(tokens[tpos], "invalidate"))
7426 return instr_hdr_invalidate_translate(p,
7433 if (!strcmp(tokens[tpos], "mov"))
7434 return instr_mov_translate(p,
7441 if (!strcmp(tokens[tpos], "dma"))
7442 return instr_dma_translate(p,
7449 if (!strcmp(tokens[tpos], "add"))
7450 return instr_alu_add_translate(p,
7457 if (!strcmp(tokens[tpos], "sub"))
7458 return instr_alu_sub_translate(p,
7465 if (!strcmp(tokens[tpos], "ckadd"))
7466 return instr_alu_ckadd_translate(p,
7473 if (!strcmp(tokens[tpos], "cksub"))
7474 return instr_alu_cksub_translate(p,
7481 if (!strcmp(tokens[tpos], "and"))
7482 return instr_alu_and_translate(p,
7489 if (!strcmp(tokens[tpos], "or"))
7490 return instr_alu_or_translate(p,
7497 if (!strcmp(tokens[tpos], "xor"))
7498 return instr_alu_xor_translate(p,
7505 if (!strcmp(tokens[tpos], "shl"))
7506 return instr_alu_shl_translate(p,
7513 if (!strcmp(tokens[tpos], "shr"))
7514 return instr_alu_shr_translate(p,
7521 if (!strcmp(tokens[tpos], "regprefetch"))
7522 return instr_regprefetch_translate(p,
7529 if (!strcmp(tokens[tpos], "regrd"))
7530 return instr_regrd_translate(p,
7537 if (!strcmp(tokens[tpos], "regwr"))
7538 return instr_regwr_translate(p,
7545 if (!strcmp(tokens[tpos], "regadd"))
7546 return instr_regadd_translate(p,
7553 if (!strcmp(tokens[tpos], "metprefetch"))
7554 return instr_metprefetch_translate(p,
7561 if (!strcmp(tokens[tpos], "meter"))
7562 return instr_meter_translate(p,
7569 if (!strcmp(tokens[tpos], "table"))
7570 return instr_table_translate(p,
7577 if (!strcmp(tokens[tpos], "extern"))
7578 return instr_extern_translate(p,
7585 if (!strcmp(tokens[tpos], "jmp"))
7586 return instr_jmp_translate(p,
7593 if (!strcmp(tokens[tpos], "jmpv"))
7594 return instr_jmp_valid_translate(p,
7601 if (!strcmp(tokens[tpos], "jmpnv"))
7602 return instr_jmp_invalid_translate(p,
7609 if (!strcmp(tokens[tpos], "jmph"))
7610 return instr_jmp_hit_translate(p,
7617 if (!strcmp(tokens[tpos], "jmpnh"))
7618 return instr_jmp_miss_translate(p,
7625 if (!strcmp(tokens[tpos], "jmpa"))
7626 return instr_jmp_action_hit_translate(p,
7633 if (!strcmp(tokens[tpos], "jmpna"))
7634 return instr_jmp_action_miss_translate(p,
7641 if (!strcmp(tokens[tpos], "jmpeq"))
7642 return instr_jmp_eq_translate(p,
7649 if (!strcmp(tokens[tpos], "jmpneq"))
7650 return instr_jmp_neq_translate(p,
7657 if (!strcmp(tokens[tpos], "jmplt"))
7658 return instr_jmp_lt_translate(p,
7665 if (!strcmp(tokens[tpos], "jmpgt"))
7666 return instr_jmp_gt_translate(p,
7673 if (!strcmp(tokens[tpos], "return"))
7674 return instr_return_translate(p,
7684 static struct instruction_data *
7685 label_find(struct instruction_data *data, uint32_t n, const char *label)
7689 for (i = 0; i < n; i++)
7690 if (!strcmp(label, data[i].label))
7697 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7699 uint32_t count = 0, i;
7704 for (i = 0; i < n; i++)
7705 if (!strcmp(label, data[i].jmp_label))
7712 instr_label_check(struct instruction_data *instruction_data,
7713 uint32_t n_instructions)
7717 /* Check that all instruction labels are unique. */
7718 for (i = 0; i < n_instructions; i++) {
7719 struct instruction_data *data = &instruction_data[i];
7720 char *label = data->label;
7726 for (j = i + 1; j < n_instructions; j++)
7727 CHECK(strcmp(label, data[j].label), EINVAL);
7730 /* Get users for each instruction label. */
7731 for (i = 0; i < n_instructions; i++) {
7732 struct instruction_data *data = &instruction_data[i];
7733 char *label = data->label;
7735 data->n_users = label_is_used(instruction_data,
7744 instr_jmp_resolve(struct instruction *instructions,
7745 struct instruction_data *instruction_data,
7746 uint32_t n_instructions)
7750 for (i = 0; i < n_instructions; i++) {
7751 struct instruction *instr = &instructions[i];
7752 struct instruction_data *data = &instruction_data[i];
7753 struct instruction_data *found;
7755 if (!instruction_is_jmp(instr))
7758 found = label_find(instruction_data,
7761 CHECK(found, EINVAL);
7763 instr->jmp.ip = &instructions[found - instruction_data];
7770 instr_verify(struct rte_swx_pipeline *p __rte_unused,
7772 struct instruction *instr,
7773 struct instruction_data *data __rte_unused,
7774 uint32_t n_instructions)
7777 enum instruction_type type;
7780 /* Check that the first instruction is rx. */
7781 CHECK(instr[0].type == INSTR_RX, EINVAL);
7783 /* Check that there is at least one tx instruction. */
7784 for (i = 0; i < n_instructions; i++) {
7785 type = instr[i].type;
7787 if (instruction_is_tx(type))
7790 CHECK(i < n_instructions, EINVAL);
7792 /* Check that the last instruction is either tx or unconditional
7795 type = instr[n_instructions - 1].type;
7796 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
7800 enum instruction_type type;
7803 /* Check that there is at least one return or tx instruction. */
7804 for (i = 0; i < n_instructions; i++) {
7805 type = instr[i].type;
7807 if ((type == INSTR_RETURN) || instruction_is_tx(type))
7810 CHECK(i < n_instructions, EINVAL);
7817 instr_pattern_extract_many_detect(struct instruction *instr,
7818 struct instruction_data *data,
7820 uint32_t *n_pattern_instr)
7824 for (i = 0; i < n_instr; i++) {
7825 if (data[i].invalid)
7828 if (instr[i].type != INSTR_HDR_EXTRACT)
7831 if (i == RTE_DIM(instr->io.hdr.header_id))
7834 if (i && data[i].n_users)
7841 *n_pattern_instr = i;
7846 instr_pattern_extract_many_optimize(struct instruction *instr,
7847 struct instruction_data *data,
7852 for (i = 1; i < n_instr; i++) {
7854 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7855 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7856 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7858 data[i].invalid = 1;
7863 instr_pattern_emit_many_tx_detect(struct instruction *instr,
7864 struct instruction_data *data,
7866 uint32_t *n_pattern_instr)
7870 for (i = 0; i < n_instr; i++) {
7871 if (data[i].invalid)
7874 if (instr[i].type != INSTR_HDR_EMIT)
7877 if (i == RTE_DIM(instr->io.hdr.header_id))
7880 if (i && data[i].n_users)
7887 if (!instruction_is_tx(instr[i].type))
7890 if (data[i].n_users)
7895 *n_pattern_instr = i;
7900 instr_pattern_emit_many_tx_optimize(struct instruction *instr,
7901 struct instruction_data *data,
7906 /* Any emit instruction in addition to the first one. */
7907 for (i = 1; i < n_instr - 1; i++) {
7909 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
7910 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
7911 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
7913 data[i].invalid = 1;
7916 /* The TX instruction is the last one in the pattern. */
7918 instr[0].io.io.offset = instr[i].io.io.offset;
7919 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
7920 data[i].invalid = 1;
7924 instr_pattern_dma_many_detect(struct instruction *instr,
7925 struct instruction_data *data,
7927 uint32_t *n_pattern_instr)
7931 for (i = 0; i < n_instr; i++) {
7932 if (data[i].invalid)
7935 if (instr[i].type != INSTR_DMA_HT)
7938 if (i == RTE_DIM(instr->dma.dst.header_id))
7941 if (i && data[i].n_users)
7948 *n_pattern_instr = i;
7953 instr_pattern_dma_many_optimize(struct instruction *instr,
7954 struct instruction_data *data,
7959 for (i = 1; i < n_instr; i++) {
7961 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
7962 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
7963 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
7964 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
7966 data[i].invalid = 1;
7971 instr_optimize(struct instruction *instructions,
7972 struct instruction_data *instruction_data,
7973 uint32_t n_instructions)
7975 uint32_t i, pos = 0;
7977 for (i = 0; i < n_instructions; ) {
7978 struct instruction *instr = &instructions[i];
7979 struct instruction_data *data = &instruction_data[i];
7980 uint32_t n_instr = 0;
7984 detected = instr_pattern_extract_many_detect(instr,
7989 instr_pattern_extract_many_optimize(instr,
7996 /* Emit many + TX. */
7997 detected = instr_pattern_emit_many_tx_detect(instr,
8002 instr_pattern_emit_many_tx_optimize(instr,
8010 detected = instr_pattern_dma_many_detect(instr,
8015 instr_pattern_dma_many_optimize(instr, data, n_instr);
8020 /* No pattern starting at the current instruction. */
8024 /* Eliminate the invalid instructions that have been optimized out. */
8025 for (i = 0; i < n_instructions; i++) {
8026 struct instruction *instr = &instructions[i];
8027 struct instruction_data *data = &instruction_data[i];
8033 memcpy(&instructions[pos], instr, sizeof(*instr));
8034 memcpy(&instruction_data[pos], data, sizeof(*data));
8044 instruction_config(struct rte_swx_pipeline *p,
8046 const char **instructions,
8047 uint32_t n_instructions)
8049 struct instruction *instr = NULL;
8050 struct instruction_data *data = NULL;
8054 CHECK(n_instructions, EINVAL);
8055 CHECK(instructions, EINVAL);
8056 for (i = 0; i < n_instructions; i++)
8057 CHECK_INSTRUCTION(instructions[i], EINVAL);
8059 /* Memory allocation. */
8060 instr = calloc(n_instructions, sizeof(struct instruction));
8066 data = calloc(n_instructions, sizeof(struct instruction_data));
8072 for (i = 0; i < n_instructions; i++) {
8073 char *string = strdup(instructions[i]);
8079 err = instr_translate(p, a, string, &instr[i], &data[i]);
8088 err = instr_label_check(data, n_instructions);
8092 err = instr_verify(p, a, instr, data, n_instructions);
8096 n_instructions = instr_optimize(instr, data, n_instructions);
8098 err = instr_jmp_resolve(instr, data, n_instructions);
8103 a->instructions = instr;
8104 a->n_instructions = n_instructions;
8106 p->instructions = instr;
8107 p->n_instructions = n_instructions;
8119 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8121 static instr_exec_t instruction_table[] = {
8122 [INSTR_RX] = instr_rx_exec,
8123 [INSTR_TX] = instr_tx_exec,
8124 [INSTR_TX_I] = instr_tx_i_exec,
8126 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8127 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8128 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8129 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8130 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8131 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8132 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8133 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8135 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8136 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8137 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8138 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8139 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8140 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8141 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8142 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8143 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8145 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8146 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8148 [INSTR_MOV] = instr_mov_exec,
8149 [INSTR_MOV_S] = instr_mov_s_exec,
8150 [INSTR_MOV_I] = instr_mov_i_exec,
8152 [INSTR_DMA_HT] = instr_dma_ht_exec,
8153 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8154 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8155 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8156 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8157 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8158 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8159 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8161 [INSTR_ALU_ADD] = instr_alu_add_exec,
8162 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8163 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8164 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8165 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8166 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8168 [INSTR_ALU_SUB] = instr_alu_sub_exec,
8169 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8170 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8171 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8172 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8173 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8175 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8176 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8177 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8178 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8180 [INSTR_ALU_AND] = instr_alu_and_exec,
8181 [INSTR_ALU_AND_S] = instr_alu_and_s_exec,
8182 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8184 [INSTR_ALU_OR] = instr_alu_or_exec,
8185 [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
8186 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8188 [INSTR_ALU_XOR] = instr_alu_xor_exec,
8189 [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
8190 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8192 [INSTR_ALU_SHL] = instr_alu_shl_exec,
8193 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8194 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8195 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8196 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8197 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8199 [INSTR_ALU_SHR] = instr_alu_shr_exec,
8200 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8201 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8202 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8203 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8204 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8206 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8207 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8208 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8210 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8211 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8212 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8213 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8214 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8215 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8217 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8218 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8219 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8220 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8221 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8222 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8223 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8224 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8225 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8227 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8228 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8229 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8230 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8231 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8232 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8233 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8234 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8235 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8237 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8238 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8239 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8241 [INSTR_METER_HHM] = instr_meter_hhm_exec,
8242 [INSTR_METER_HHI] = instr_meter_hhi_exec,
8243 [INSTR_METER_HMM] = instr_meter_hmm_exec,
8244 [INSTR_METER_HMI] = instr_meter_hmi_exec,
8245 [INSTR_METER_MHM] = instr_meter_mhm_exec,
8246 [INSTR_METER_MHI] = instr_meter_mhi_exec,
8247 [INSTR_METER_MMM] = instr_meter_mmm_exec,
8248 [INSTR_METER_MMI] = instr_meter_mmi_exec,
8249 [INSTR_METER_IHM] = instr_meter_ihm_exec,
8250 [INSTR_METER_IHI] = instr_meter_ihi_exec,
8251 [INSTR_METER_IMM] = instr_meter_imm_exec,
8252 [INSTR_METER_IMI] = instr_meter_imi_exec,
8254 [INSTR_TABLE] = instr_table_exec,
8255 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8256 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8258 [INSTR_JMP] = instr_jmp_exec,
8259 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8260 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8261 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8262 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8263 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8264 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8266 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8267 [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
8268 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8270 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8271 [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
8272 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8274 [INSTR_JMP_LT] = instr_jmp_lt_exec,
8275 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8276 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8277 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8278 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8279 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8281 [INSTR_JMP_GT] = instr_jmp_gt_exec,
8282 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8283 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8284 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8285 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8286 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8288 [INSTR_RETURN] = instr_return_exec,
8292 instr_exec(struct rte_swx_pipeline *p)
8294 struct thread *t = &p->threads[p->thread_id];
8295 struct instruction *ip = t->ip;
8296 instr_exec_t instr = instruction_table[ip->type];
8304 static struct action *
8305 action_find(struct rte_swx_pipeline *p, const char *name)
8307 struct action *elem;
8312 TAILQ_FOREACH(elem, &p->actions, node)
8313 if (strcmp(elem->name, name) == 0)
8319 static struct action *
8320 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8322 struct action *action = NULL;
8324 TAILQ_FOREACH(action, &p->actions, node)
8325 if (action->id == id)
8331 static struct field *
8332 action_field_find(struct action *a, const char *name)
8334 return a->st ? struct_type_field_find(a->st, name) : NULL;
8337 static struct field *
8338 action_field_parse(struct action *action, const char *name)
8340 if (name[0] != 't' || name[1] != '.')
8343 return action_field_find(action, &name[2]);
8347 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8349 const char *args_struct_type_name,
8350 const char **instructions,
8351 uint32_t n_instructions)
8353 struct struct_type *args_struct_type;
8359 CHECK_NAME(name, EINVAL);
8360 CHECK(!action_find(p, name), EEXIST);
8362 if (args_struct_type_name) {
8363 CHECK_NAME(args_struct_type_name, EINVAL);
8364 args_struct_type = struct_type_find(p, args_struct_type_name);
8365 CHECK(args_struct_type, EINVAL);
8367 args_struct_type = NULL;
8370 /* Node allocation. */
8371 a = calloc(1, sizeof(struct action));
8374 /* Node initialization. */
8375 strcpy(a->name, name);
8376 a->st = args_struct_type;
8377 a->id = p->n_actions;
8379 /* Instruction translation. */
8380 err = instruction_config(p, a, instructions, n_instructions);
8386 /* Node add to tailq. */
8387 TAILQ_INSERT_TAIL(&p->actions, a, node);
8394 action_build(struct rte_swx_pipeline *p)
8396 struct action *action;
8398 p->action_instructions = calloc(p->n_actions,
8399 sizeof(struct instruction *));
8400 CHECK(p->action_instructions, ENOMEM);
8402 TAILQ_FOREACH(action, &p->actions, node)
8403 p->action_instructions[action->id] = action->instructions;
8409 action_build_free(struct rte_swx_pipeline *p)
8411 free(p->action_instructions);
8412 p->action_instructions = NULL;
8416 action_free(struct rte_swx_pipeline *p)
8418 action_build_free(p);
8421 struct action *action;
8423 action = TAILQ_FIRST(&p->actions);
8427 TAILQ_REMOVE(&p->actions, action, node);
8428 free(action->instructions);
8436 static struct table_type *
8437 table_type_find(struct rte_swx_pipeline *p, const char *name)
8439 struct table_type *elem;
8441 TAILQ_FOREACH(elem, &p->table_types, node)
8442 if (strcmp(elem->name, name) == 0)
8448 static struct table_type *
8449 table_type_resolve(struct rte_swx_pipeline *p,
8450 const char *recommended_type_name,
8451 enum rte_swx_table_match_type match_type)
8453 struct table_type *elem;
8455 /* Only consider the recommended type if the match type is correct. */
8456 if (recommended_type_name)
8457 TAILQ_FOREACH(elem, &p->table_types, node)
8458 if (!strcmp(elem->name, recommended_type_name) &&
8459 (elem->match_type == match_type))
8462 /* Ignore the recommended type and get the first element with this match
8465 TAILQ_FOREACH(elem, &p->table_types, node)
8466 if (elem->match_type == match_type)
8472 static struct table *
8473 table_find(struct rte_swx_pipeline *p, const char *name)
8477 TAILQ_FOREACH(elem, &p->tables, node)
8478 if (strcmp(elem->name, name) == 0)
8484 static struct table *
8485 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8487 struct table *table = NULL;
8489 TAILQ_FOREACH(table, &p->tables, node)
8490 if (table->id == id)
8497 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8499 enum rte_swx_table_match_type match_type,
8500 struct rte_swx_table_ops *ops)
8502 struct table_type *elem;
8506 CHECK_NAME(name, EINVAL);
8507 CHECK(!table_type_find(p, name), EEXIST);
8510 CHECK(ops->create, EINVAL);
8511 CHECK(ops->lkp, EINVAL);
8512 CHECK(ops->free, EINVAL);
8514 /* Node allocation. */
8515 elem = calloc(1, sizeof(struct table_type));
8516 CHECK(elem, ENOMEM);
8518 /* Node initialization. */
8519 strcpy(elem->name, name);
8520 elem->match_type = match_type;
8521 memcpy(&elem->ops, ops, sizeof(*ops));
8523 /* Node add to tailq. */
8524 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8529 static enum rte_swx_table_match_type
8530 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8532 uint32_t max_offset_field_id)
8534 uint32_t n_fields_em = 0, i;
8536 for (i = 0; i < n_fields; i++)
8537 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8540 if (n_fields_em == n_fields)
8541 return RTE_SWX_TABLE_MATCH_EXACT;
8543 if ((n_fields_em == n_fields - 1) &&
8544 (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8545 return RTE_SWX_TABLE_MATCH_LPM;
8547 return RTE_SWX_TABLE_MATCH_WILDCARD;
8551 table_match_fields_check(struct rte_swx_pipeline *p,
8552 struct rte_swx_pipeline_table_params *params,
8553 struct header **header,
8554 uint32_t *min_offset_field_id,
8555 uint32_t *max_offset_field_id)
8557 struct header *h0 = NULL;
8558 struct field *hf, *mf;
8559 uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8562 /* Return if no match fields. */
8563 if (!params->n_fields) {
8564 if (params->fields) {
8572 /* Memory allocation. */
8573 offset = calloc(params->n_fields, sizeof(uint32_t));
8579 /* Check that all the match fields belong to either the same header or
8582 hf = header_field_parse(p, params->fields[0].name, &h0);
8583 mf = metadata_field_parse(p, params->fields[0].name);
8589 offset[0] = h0 ? hf->offset : mf->offset;
8591 for (i = 1; i < params->n_fields; i++)
8595 hf = header_field_parse(p, params->fields[i].name, &h);
8596 if (!hf || (h->id != h0->id)) {
8601 offset[i] = hf->offset;
8603 mf = metadata_field_parse(p, params->fields[i].name);
8609 offset[i] = mf->offset;
8612 /* Check that there are no duplicated match fields. */
8613 for (i = 0; i < params->n_fields; i++) {
8616 for (j = 0; j < i; j++)
8617 if (offset[j] == offset[i]) {
8623 /* Find the min and max offset fields. */
8624 min_offset = offset[0];
8625 max_offset = offset[0];
8629 for (i = 1; i < params->n_fields; i++) {
8630 if (offset[i] < min_offset) {
8631 min_offset = offset[i];
8635 if (offset[i] > max_offset) {
8636 max_offset = offset[i];
8645 if (min_offset_field_id)
8646 *min_offset_field_id = min_offset_pos;
8648 if (max_offset_field_id)
8649 *max_offset_field_id = max_offset_pos;
8657 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
8659 struct rte_swx_pipeline_table_params *params,
8660 const char *recommended_table_type_name,
8664 struct table_type *type;
8666 struct action *default_action;
8667 struct header *header = NULL;
8668 uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
8673 CHECK_NAME(name, EINVAL);
8674 CHECK(!table_find(p, name), EEXIST);
8676 CHECK(params, EINVAL);
8679 status = table_match_fields_check(p,
8682 &min_offset_field_id,
8683 &max_offset_field_id);
8687 /* Action checks. */
8688 CHECK(params->n_actions, EINVAL);
8689 CHECK(params->action_names, EINVAL);
8690 for (i = 0; i < params->n_actions; i++) {
8691 const char *action_name = params->action_names[i];
8693 uint32_t action_data_size;
8695 CHECK_NAME(action_name, EINVAL);
8697 a = action_find(p, action_name);
8700 action_data_size = a->st ? a->st->n_bits / 8 : 0;
8701 if (action_data_size > action_data_size_max)
8702 action_data_size_max = action_data_size;
8705 CHECK_NAME(params->default_action_name, EINVAL);
8706 for (i = 0; i < p->n_actions; i++)
8707 if (!strcmp(params->action_names[i],
8708 params->default_action_name))
8710 CHECK(i < params->n_actions, EINVAL);
8711 default_action = action_find(p, params->default_action_name);
8712 CHECK((default_action->st && params->default_action_data) ||
8713 !params->default_action_data, EINVAL);
8715 /* Table type checks. */
8716 if (recommended_table_type_name)
8717 CHECK_NAME(recommended_table_type_name, EINVAL);
8719 if (params->n_fields) {
8720 enum rte_swx_table_match_type match_type;
8722 match_type = table_match_type_resolve(params->fields,
8724 max_offset_field_id);
8725 type = table_type_resolve(p,
8726 recommended_table_type_name,
8728 CHECK(type, EINVAL);
8733 /* Memory allocation. */
8734 t = calloc(1, sizeof(struct table));
8737 t->fields = calloc(params->n_fields, sizeof(struct match_field));
8743 t->actions = calloc(params->n_actions, sizeof(struct action *));
8750 if (action_data_size_max) {
8751 t->default_action_data = calloc(1, action_data_size_max);
8752 if (!t->default_action_data) {
8760 /* Node initialization. */
8761 strcpy(t->name, name);
8762 if (args && args[0])
8763 strcpy(t->args, args);
8766 for (i = 0; i < params->n_fields; i++) {
8767 struct rte_swx_match_field_params *field = ¶ms->fields[i];
8768 struct match_field *f = &t->fields[i];
8770 f->match_type = field->match_type;
8772 header_field_parse(p, field->name, NULL) :
8773 metadata_field_parse(p, field->name);
8775 t->n_fields = params->n_fields;
8778 for (i = 0; i < params->n_actions; i++)
8779 t->actions[i] = action_find(p, params->action_names[i]);
8780 t->default_action = default_action;
8781 if (default_action->st)
8782 memcpy(t->default_action_data,
8783 params->default_action_data,
8784 default_action->st->n_bits / 8);
8785 t->n_actions = params->n_actions;
8786 t->default_action_is_const = params->default_action_is_const;
8787 t->action_data_size_max = action_data_size_max;
8790 t->id = p->n_tables;
8792 /* Node add to tailq. */
8793 TAILQ_INSERT_TAIL(&p->tables, t, node);
8799 static struct rte_swx_table_params *
8800 table_params_get(struct table *table)
8802 struct rte_swx_table_params *params;
8803 struct field *first, *last;
8805 uint32_t key_size, key_offset, action_data_size, i;
8807 /* Memory allocation. */
8808 params = calloc(1, sizeof(struct rte_swx_table_params));
8812 /* Find first (smallest offset) and last (biggest offset) match fields. */
8813 first = table->fields[0].field;
8814 last = table->fields[0].field;
8816 for (i = 0; i < table->n_fields; i++) {
8817 struct field *f = table->fields[i].field;
8819 if (f->offset < first->offset)
8822 if (f->offset > last->offset)
8826 /* Key offset and size. */
8827 key_offset = first->offset / 8;
8828 key_size = (last->offset + last->n_bits - first->offset) / 8;
8830 /* Memory allocation. */
8831 key_mask = calloc(1, key_size);
8838 for (i = 0; i < table->n_fields; i++) {
8839 struct field *f = table->fields[i].field;
8840 uint32_t start = (f->offset - first->offset) / 8;
8841 size_t size = f->n_bits / 8;
8843 memset(&key_mask[start], 0xFF, size);
8846 /* Action data size. */
8847 action_data_size = 0;
8848 for (i = 0; i < table->n_actions; i++) {
8849 struct action *action = table->actions[i];
8850 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
8852 if (ads > action_data_size)
8853 action_data_size = ads;
8857 params->match_type = table->type->match_type;
8858 params->key_size = key_size;
8859 params->key_offset = key_offset;
8860 params->key_mask0 = key_mask;
8861 params->action_data_size = action_data_size;
8862 params->n_keys_max = table->size;
8868 table_params_free(struct rte_swx_table_params *params)
8873 free(params->key_mask0);
8878 table_state_build(struct rte_swx_pipeline *p)
8880 struct table *table;
8882 p->table_state = calloc(p->n_tables,
8883 sizeof(struct rte_swx_table_state));
8884 CHECK(p->table_state, ENOMEM);
8886 TAILQ_FOREACH(table, &p->tables, node) {
8887 struct rte_swx_table_state *ts = &p->table_state[table->id];
8890 struct rte_swx_table_params *params;
8893 params = table_params_get(table);
8894 CHECK(params, ENOMEM);
8896 ts->obj = table->type->ops.create(params,
8901 table_params_free(params);
8902 CHECK(ts->obj, ENODEV);
8905 /* ts->default_action_data. */
8906 if (table->action_data_size_max) {
8907 ts->default_action_data =
8908 malloc(table->action_data_size_max);
8909 CHECK(ts->default_action_data, ENOMEM);
8911 memcpy(ts->default_action_data,
8912 table->default_action_data,
8913 table->action_data_size_max);
8916 /* ts->default_action_id. */
8917 ts->default_action_id = table->default_action->id;
8924 table_state_build_free(struct rte_swx_pipeline *p)
8928 if (!p->table_state)
8931 for (i = 0; i < p->n_tables; i++) {
8932 struct rte_swx_table_state *ts = &p->table_state[i];
8933 struct table *table = table_find_by_id(p, i);
8936 if (table->type && ts->obj)
8937 table->type->ops.free(ts->obj);
8939 /* ts->default_action_data. */
8940 free(ts->default_action_data);
8943 free(p->table_state);
8944 p->table_state = NULL;
8948 table_state_free(struct rte_swx_pipeline *p)
8950 table_state_build_free(p);
8954 table_stub_lkp(void *table __rte_unused,
8955 void *mailbox __rte_unused,
8956 uint8_t **key __rte_unused,
8957 uint64_t *action_id __rte_unused,
8958 uint8_t **action_data __rte_unused,
8962 return 1; /* DONE. */
8966 table_build(struct rte_swx_pipeline *p)
8970 /* Per pipeline: table statistics. */
8971 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
8972 CHECK(p->table_stats, ENOMEM);
8974 for (i = 0; i < p->n_tables; i++) {
8975 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
8976 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
8979 /* Per thread: table runt-time. */
8980 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
8981 struct thread *t = &p->threads[i];
8982 struct table *table;
8984 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
8985 CHECK(t->tables, ENOMEM);
8987 TAILQ_FOREACH(table, &p->tables, node) {
8988 struct table_runtime *r = &t->tables[table->id];
8993 size = table->type->ops.mailbox_size_get();
8996 r->func = table->type->ops.lkp;
9000 r->mailbox = calloc(1, size);
9001 CHECK(r->mailbox, ENOMEM);
9005 r->key = table->header ?
9006 &t->structs[table->header->struct_id] :
9007 &t->structs[p->metadata_struct_id];
9009 r->func = table_stub_lkp;
9018 table_build_free(struct rte_swx_pipeline *p)
9022 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9023 struct thread *t = &p->threads[i];
9029 for (j = 0; j < p->n_tables; j++) {
9030 struct table_runtime *r = &t->tables[j];
9039 if (p->table_stats) {
9040 for (i = 0; i < p->n_tables; i++)
9041 free(p->table_stats[i].n_pkts_action);
9043 free(p->table_stats);
9048 table_free(struct rte_swx_pipeline *p)
9050 table_build_free(p);
9056 elem = TAILQ_FIRST(&p->tables);
9060 TAILQ_REMOVE(&p->tables, elem, node);
9062 free(elem->actions);
9063 free(elem->default_action_data);
9069 struct table_type *elem;
9071 elem = TAILQ_FIRST(&p->table_types);
9075 TAILQ_REMOVE(&p->table_types, elem, node);
9083 static struct regarray *
9084 regarray_find(struct rte_swx_pipeline *p, const char *name)
9086 struct regarray *elem;
9088 TAILQ_FOREACH(elem, &p->regarrays, node)
9089 if (!strcmp(elem->name, name))
9095 static struct regarray *
9096 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9098 struct regarray *elem = NULL;
9100 TAILQ_FOREACH(elem, &p->regarrays, node)
9108 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9117 CHECK_NAME(name, EINVAL);
9118 CHECK(!regarray_find(p, name), EEXIST);
9120 CHECK(size, EINVAL);
9121 size = rte_align32pow2(size);
9123 /* Memory allocation. */
9124 r = calloc(1, sizeof(struct regarray));
9127 /* Node initialization. */
9128 strcpy(r->name, name);
9129 r->init_val = init_val;
9131 r->id = p->n_regarrays;
9133 /* Node add to tailq. */
9134 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9141 regarray_build(struct rte_swx_pipeline *p)
9143 struct regarray *regarray;
9145 if (!p->n_regarrays)
9148 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9149 CHECK(p->regarray_runtime, ENOMEM);
9151 TAILQ_FOREACH(regarray, &p->regarrays, node) {
9152 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9155 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9156 RTE_CACHE_LINE_SIZE,
9158 CHECK(r->regarray, ENOMEM);
9160 if (regarray->init_val)
9161 for (i = 0; i < regarray->size; i++)
9162 r->regarray[i] = regarray->init_val;
9164 r->size_mask = regarray->size - 1;
9171 regarray_build_free(struct rte_swx_pipeline *p)
9175 if (!p->regarray_runtime)
9178 for (i = 0; i < p->n_regarrays; i++) {
9179 struct regarray *regarray = regarray_find_by_id(p, i);
9180 struct regarray_runtime *r = &p->regarray_runtime[i];
9182 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9185 free(p->regarray_runtime);
9186 p->regarray_runtime = NULL;
9190 regarray_free(struct rte_swx_pipeline *p)
9192 regarray_build_free(p);
9195 struct regarray *elem;
9197 elem = TAILQ_FIRST(&p->regarrays);
9201 TAILQ_REMOVE(&p->regarrays, elem, node);
9209 static struct meter_profile *
9210 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9212 struct meter_profile *elem;
9214 TAILQ_FOREACH(elem, &p->meter_profiles, node)
9215 if (!strcmp(elem->name, name))
9221 static struct metarray *
9222 metarray_find(struct rte_swx_pipeline *p, const char *name)
9224 struct metarray *elem;
9226 TAILQ_FOREACH(elem, &p->metarrays, node)
9227 if (!strcmp(elem->name, name))
9233 static struct metarray *
9234 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9236 struct metarray *elem = NULL;
9238 TAILQ_FOREACH(elem, &p->metarrays, node)
9246 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9254 CHECK_NAME(name, EINVAL);
9255 CHECK(!metarray_find(p, name), EEXIST);
9257 CHECK(size, EINVAL);
9258 size = rte_align32pow2(size);
9260 /* Memory allocation. */
9261 m = calloc(1, sizeof(struct metarray));
9264 /* Node initialization. */
9265 strcpy(m->name, name);
9267 m->id = p->n_metarrays;
9269 /* Node add to tailq. */
9270 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9276 struct meter_profile meter_profile_default = {
9285 .cir_bytes_per_period = 1,
9287 .pir_bytes_per_period = 1,
9294 meter_init(struct meter *m)
9296 memset(m, 0, sizeof(struct meter));
9297 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9298 m->profile = &meter_profile_default;
9299 m->color_mask = RTE_COLOR_GREEN;
9301 meter_profile_default.n_users++;
9305 metarray_build(struct rte_swx_pipeline *p)
9309 if (!p->n_metarrays)
9312 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9313 CHECK(p->metarray_runtime, ENOMEM);
9315 TAILQ_FOREACH(m, &p->metarrays, node) {
9316 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9319 r->metarray = env_malloc(m->size * sizeof(struct meter),
9320 RTE_CACHE_LINE_SIZE,
9322 CHECK(r->metarray, ENOMEM);
9324 for (i = 0; i < m->size; i++)
9325 meter_init(&r->metarray[i]);
9327 r->size_mask = m->size - 1;
9334 metarray_build_free(struct rte_swx_pipeline *p)
9338 if (!p->metarray_runtime)
9341 for (i = 0; i < p->n_metarrays; i++) {
9342 struct metarray *m = metarray_find_by_id(p, i);
9343 struct metarray_runtime *r = &p->metarray_runtime[i];
9345 env_free(r->metarray, m->size * sizeof(struct meter));
9348 free(p->metarray_runtime);
9349 p->metarray_runtime = NULL;
9353 metarray_free(struct rte_swx_pipeline *p)
9355 metarray_build_free(p);
9359 struct metarray *elem;
9361 elem = TAILQ_FIRST(&p->metarrays);
9365 TAILQ_REMOVE(&p->metarrays, elem, node);
9369 /* Meter profiles. */
9371 struct meter_profile *elem;
9373 elem = TAILQ_FIRST(&p->meter_profiles);
9377 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9386 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9388 struct rte_swx_pipeline *pipeline;
9390 /* Check input parameters. */
9393 /* Memory allocation. */
9394 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9395 CHECK(pipeline, ENOMEM);
9397 /* Initialization. */
9398 TAILQ_INIT(&pipeline->struct_types);
9399 TAILQ_INIT(&pipeline->port_in_types);
9400 TAILQ_INIT(&pipeline->ports_in);
9401 TAILQ_INIT(&pipeline->port_out_types);
9402 TAILQ_INIT(&pipeline->ports_out);
9403 TAILQ_INIT(&pipeline->extern_types);
9404 TAILQ_INIT(&pipeline->extern_objs);
9405 TAILQ_INIT(&pipeline->extern_funcs);
9406 TAILQ_INIT(&pipeline->headers);
9407 TAILQ_INIT(&pipeline->actions);
9408 TAILQ_INIT(&pipeline->table_types);
9409 TAILQ_INIT(&pipeline->tables);
9410 TAILQ_INIT(&pipeline->regarrays);
9411 TAILQ_INIT(&pipeline->meter_profiles);
9412 TAILQ_INIT(&pipeline->metarrays);
9414 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9415 pipeline->numa_node = numa_node;
9422 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9427 free(p->instructions);
9431 table_state_free(p);
9436 extern_func_free(p);
9446 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9447 const char **instructions,
9448 uint32_t n_instructions)
9453 err = instruction_config(p, NULL, instructions, n_instructions);
9457 /* Thread instruction pointer reset. */
9458 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9459 struct thread *t = &p->threads[i];
9461 thread_ip_reset(p, t);
9468 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9473 CHECK(p->build_done == 0, EEXIST);
9475 status = port_in_build(p);
9479 status = port_out_build(p);
9483 status = struct_build(p);
9487 status = extern_obj_build(p);
9491 status = extern_func_build(p);
9495 status = header_build(p);
9499 status = metadata_build(p);
9503 status = action_build(p);
9507 status = table_build(p);
9511 status = table_state_build(p);
9515 status = regarray_build(p);
9519 status = metarray_build(p);
9527 metarray_build_free(p);
9528 regarray_build_free(p);
9529 table_state_build_free(p);
9530 table_build_free(p);
9531 action_build_free(p);
9532 metadata_build_free(p);
9533 header_build_free(p);
9534 extern_func_build_free(p);
9535 extern_obj_build_free(p);
9536 port_out_build_free(p);
9537 port_in_build_free(p);
9538 struct_build_free(p);
9544 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9548 for (i = 0; i < n_instructions; i++)
9553 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9557 for (i = 0; i < p->n_ports_out; i++) {
9558 struct port_out_runtime *port = &p->out[i];
9561 port->flush(port->obj);
9569 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9570 struct rte_swx_ctl_pipeline_info *pipeline)
9572 struct action *action;
9573 struct table *table;
9574 uint32_t n_actions = 0, n_tables = 0;
9576 if (!p || !pipeline)
9579 TAILQ_FOREACH(action, &p->actions, node)
9582 TAILQ_FOREACH(table, &p->tables, node)
9585 pipeline->n_ports_in = p->n_ports_in;
9586 pipeline->n_ports_out = p->n_ports_out;
9587 pipeline->n_actions = n_actions;
9588 pipeline->n_tables = n_tables;
9589 pipeline->n_regarrays = p->n_regarrays;
9590 pipeline->n_metarrays = p->n_metarrays;
9596 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9598 if (!p || !numa_node)
9601 *numa_node = p->numa_node;
9606 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9608 struct rte_swx_ctl_action_info *action)
9610 struct action *a = NULL;
9612 if (!p || (action_id >= p->n_actions) || !action)
9615 a = action_find_by_id(p, action_id);
9619 strcpy(action->name, a->name);
9620 action->n_args = a->st ? a->st->n_fields : 0;
9625 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
9627 uint32_t action_arg_id,
9628 struct rte_swx_ctl_action_arg_info *action_arg)
9630 struct action *a = NULL;
9631 struct field *arg = NULL;
9633 if (!p || (action_id >= p->n_actions) || !action_arg)
9636 a = action_find_by_id(p, action_id);
9637 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
9640 arg = &a->st->fields[action_arg_id];
9641 strcpy(action_arg->name, arg->name);
9642 action_arg->n_bits = arg->n_bits;
9648 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
9650 struct rte_swx_ctl_table_info *table)
9652 struct table *t = NULL;
9657 t = table_find_by_id(p, table_id);
9661 strcpy(table->name, t->name);
9662 strcpy(table->args, t->args);
9663 table->n_match_fields = t->n_fields;
9664 table->n_actions = t->n_actions;
9665 table->default_action_is_const = t->default_action_is_const;
9666 table->size = t->size;
9671 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
9673 uint32_t match_field_id,
9674 struct rte_swx_ctl_table_match_field_info *match_field)
9677 struct match_field *f;
9679 if (!p || (table_id >= p->n_tables) || !match_field)
9682 t = table_find_by_id(p, table_id);
9683 if (!t || (match_field_id >= t->n_fields))
9686 f = &t->fields[match_field_id];
9687 match_field->match_type = f->match_type;
9688 match_field->is_header = t->header ? 1 : 0;
9689 match_field->n_bits = f->field->n_bits;
9690 match_field->offset = f->field->offset;
9696 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
9698 uint32_t table_action_id,
9699 struct rte_swx_ctl_table_action_info *table_action)
9703 if (!p || (table_id >= p->n_tables) || !table_action)
9706 t = table_find_by_id(p, table_id);
9707 if (!t || (table_action_id >= t->n_actions))
9710 table_action->action_id = t->actions[table_action_id]->id;
9716 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
9718 struct rte_swx_table_ops *table_ops,
9723 if (!p || (table_id >= p->n_tables))
9726 t = table_find_by_id(p, table_id);
9732 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
9742 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
9743 struct rte_swx_table_state **table_state)
9745 if (!p || !table_state || !p->build_done)
9748 *table_state = p->table_state;
9753 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
9754 struct rte_swx_table_state *table_state)
9756 if (!p || !table_state || !p->build_done)
9759 p->table_state = table_state;
9764 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
9766 struct rte_swx_port_in_stats *stats)
9768 struct port_in *port;
9773 port = port_in_find(p, port_id);
9777 port->type->ops.stats_read(port->obj, stats);
9782 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
9784 struct rte_swx_port_out_stats *stats)
9786 struct port_out *port;
9791 port = port_out_find(p, port_id);
9795 port->type->ops.stats_read(port->obj, stats);
9800 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
9801 const char *table_name,
9802 struct rte_swx_table_stats *stats)
9804 struct table *table;
9805 struct table_statistics *table_stats;
9807 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
9810 table = table_find(p, table_name);
9814 table_stats = &p->table_stats[table->id];
9816 memcpy(&stats->n_pkts_action,
9817 &table_stats->n_pkts_action,
9818 p->n_actions * sizeof(uint64_t));
9820 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
9821 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
9827 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
9828 uint32_t regarray_id,
9829 struct rte_swx_ctl_regarray_info *regarray)
9833 if (!p || !regarray)
9836 r = regarray_find_by_id(p, regarray_id);
9840 strcpy(regarray->name, r->name);
9841 regarray->size = r->size;
9846 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
9847 const char *regarray_name,
9848 uint32_t regarray_index,
9851 struct regarray *regarray;
9852 struct regarray_runtime *r;
9854 if (!p || !regarray_name || !value)
9857 regarray = regarray_find(p, regarray_name);
9858 if (!regarray || (regarray_index >= regarray->size))
9861 r = &p->regarray_runtime[regarray->id];
9862 *value = r->regarray[regarray_index];
9867 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
9868 const char *regarray_name,
9869 uint32_t regarray_index,
9872 struct regarray *regarray;
9873 struct regarray_runtime *r;
9875 if (!p || !regarray_name)
9878 regarray = regarray_find(p, regarray_name);
9879 if (!regarray || (regarray_index >= regarray->size))
9882 r = &p->regarray_runtime[regarray->id];
9883 r->regarray[regarray_index] = value;
9888 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
9889 uint32_t metarray_id,
9890 struct rte_swx_ctl_metarray_info *metarray)
9894 if (!p || !metarray)
9897 m = metarray_find_by_id(p, metarray_id);
9901 strcpy(metarray->name, m->name);
9902 metarray->size = m->size;
9907 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
9909 struct rte_meter_trtcm_params *params)
9911 struct meter_profile *mp;
9915 CHECK_NAME(name, EINVAL);
9916 CHECK(params, EINVAL);
9917 CHECK(!meter_profile_find(p, name), EEXIST);
9919 /* Node allocation. */
9920 mp = calloc(1, sizeof(struct meter_profile));
9923 /* Node initialization. */
9924 strcpy(mp->name, name);
9925 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
9926 status = rte_meter_trtcm_profile_config(&mp->profile, params);
9932 /* Node add to tailq. */
9933 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
9939 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
9942 struct meter_profile *mp;
9945 CHECK_NAME(name, EINVAL);
9947 mp = meter_profile_find(p, name);
9949 CHECK(!mp->n_users, EBUSY);
9951 /* Remove node from tailq. */
9952 TAILQ_REMOVE(&p->meter_profiles, mp, node);
9959 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
9960 const char *metarray_name,
9961 uint32_t metarray_index)
9963 struct meter_profile *mp_old;
9964 struct metarray *metarray;
9965 struct metarray_runtime *metarray_runtime;
9969 CHECK_NAME(metarray_name, EINVAL);
9971 metarray = metarray_find(p, metarray_name);
9972 CHECK(metarray, EINVAL);
9973 CHECK(metarray_index < metarray->size, EINVAL);
9975 metarray_runtime = &p->metarray_runtime[metarray->id];
9976 m = &metarray_runtime->metarray[metarray_index];
9977 mp_old = m->profile;
9987 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
9988 const char *metarray_name,
9989 uint32_t metarray_index,
9990 const char *profile_name)
9992 struct meter_profile *mp, *mp_old;
9993 struct metarray *metarray;
9994 struct metarray_runtime *metarray_runtime;
9998 CHECK_NAME(metarray_name, EINVAL);
10000 metarray = metarray_find(p, metarray_name);
10001 CHECK(metarray, EINVAL);
10002 CHECK(metarray_index < metarray->size, EINVAL);
10004 mp = meter_profile_find(p, profile_name);
10007 metarray_runtime = &p->metarray_runtime[metarray->id];
10008 m = &metarray_runtime->metarray[metarray_index];
10009 mp_old = m->profile;
10011 memset(m, 0, sizeof(struct meter));
10012 rte_meter_trtcm_config(&m->m, &mp->profile);
10014 m->color_mask = RTE_COLORS;
10023 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10024 const char *metarray_name,
10025 uint32_t metarray_index,
10026 struct rte_swx_ctl_meter_stats *stats)
10028 struct metarray *metarray;
10029 struct metarray_runtime *metarray_runtime;
10033 CHECK_NAME(metarray_name, EINVAL);
10035 metarray = metarray_find(p, metarray_name);
10036 CHECK(metarray, EINVAL);
10037 CHECK(metarray_index < metarray->size, EINVAL);
10039 CHECK(stats, EINVAL);
10041 metarray_runtime = &p->metarray_runtime[metarray->id];
10042 m = &metarray_runtime->metarray[metarray_index];
10044 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10045 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));