1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
10 #include <arpa/inet.h>
12 #include <rte_common.h>
13 #include <rte_prefetch.h>
14 #include <rte_byteorder.h>
15 #include <rte_cycles.h>
16 #include <rte_meter.h>
18 #include "rte_swx_pipeline.h"
19 #include "rte_swx_ctl.h"
21 #define CHECK(condition, err_code) \
27 #define CHECK_NAME(name, err_code) \
30 (strnlen((name), RTE_SWX_NAME_SIZE) < RTE_SWX_NAME_SIZE), \
33 #define CHECK_INSTRUCTION(instr, err_code) \
36 (strnlen((instr), RTE_SWX_INSTRUCTION_SIZE) < \
37 RTE_SWX_INSTRUCTION_SIZE), \
45 #define TRACE(...) printf(__VA_ARGS__)
53 #define ntoh64(x) rte_be_to_cpu_64(x)
54 #define hton64(x) rte_cpu_to_be_64(x)
56 #ifndef RTE_SWX_PIPELINE_HUGE_PAGES_DISABLE
58 #include <rte_malloc.h>
61 env_malloc(size_t size, size_t alignment, int numa_node)
63 return rte_zmalloc_socket(NULL, size, alignment, numa_node);
67 env_free(void *start, size_t size __rte_unused)
77 env_malloc(size_t size, size_t alignment __rte_unused, int numa_node)
81 if (numa_available() == -1)
84 start = numa_alloc_onnode(size, numa_node);
88 memset(start, 0, size);
93 env_free(void *start, size_t size)
95 if (numa_available() == -1)
98 numa_free(start, size);
107 char name[RTE_SWX_NAME_SIZE];
113 TAILQ_ENTRY(struct_type) node;
114 char name[RTE_SWX_NAME_SIZE];
115 struct field *fields;
120 TAILQ_HEAD(struct_type_tailq, struct_type);
125 struct port_in_type {
126 TAILQ_ENTRY(port_in_type) node;
127 char name[RTE_SWX_NAME_SIZE];
128 struct rte_swx_port_in_ops ops;
131 TAILQ_HEAD(port_in_type_tailq, port_in_type);
134 TAILQ_ENTRY(port_in) node;
135 struct port_in_type *type;
140 TAILQ_HEAD(port_in_tailq, port_in);
142 struct port_in_runtime {
143 rte_swx_port_in_pkt_rx_t pkt_rx;
150 struct port_out_type {
151 TAILQ_ENTRY(port_out_type) node;
152 char name[RTE_SWX_NAME_SIZE];
153 struct rte_swx_port_out_ops ops;
156 TAILQ_HEAD(port_out_type_tailq, port_out_type);
159 TAILQ_ENTRY(port_out) node;
160 struct port_out_type *type;
165 TAILQ_HEAD(port_out_tailq, port_out);
167 struct port_out_runtime {
168 rte_swx_port_out_pkt_tx_t pkt_tx;
169 rte_swx_port_out_flush_t flush;
176 struct extern_type_member_func {
177 TAILQ_ENTRY(extern_type_member_func) node;
178 char name[RTE_SWX_NAME_SIZE];
179 rte_swx_extern_type_member_func_t func;
183 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
186 TAILQ_ENTRY(extern_type) node;
187 char name[RTE_SWX_NAME_SIZE];
188 struct struct_type *mailbox_struct_type;
189 rte_swx_extern_type_constructor_t constructor;
190 rte_swx_extern_type_destructor_t destructor;
191 struct extern_type_member_func_tailq funcs;
195 TAILQ_HEAD(extern_type_tailq, extern_type);
198 TAILQ_ENTRY(extern_obj) node;
199 char name[RTE_SWX_NAME_SIZE];
200 struct extern_type *type;
206 TAILQ_HEAD(extern_obj_tailq, extern_obj);
208 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
209 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
212 struct extern_obj_runtime {
215 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
222 TAILQ_ENTRY(extern_func) node;
223 char name[RTE_SWX_NAME_SIZE];
224 struct struct_type *mailbox_struct_type;
225 rte_swx_extern_func_t func;
230 TAILQ_HEAD(extern_func_tailq, extern_func);
232 struct extern_func_runtime {
234 rte_swx_extern_func_t func;
241 TAILQ_ENTRY(header) node;
242 char name[RTE_SWX_NAME_SIZE];
243 struct struct_type *st;
248 TAILQ_HEAD(header_tailq, header);
250 struct header_runtime {
254 struct header_out_runtime {
264 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
265 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
266 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
267 * when transferred to packet meta-data and in NBO when transferred to packet
271 /* Notation conventions:
272 * -Header field: H = h.header.field (dst/src)
273 * -Meta-data field: M = m.field (dst/src)
274 * -Extern object mailbox field: E = e.field (dst/src)
275 * -Extern function mailbox field: F = f.field (dst/src)
276 * -Table action data field: T = t.field (src only)
277 * -Immediate value: I = 32-bit unsigned value (src only)
280 enum instruction_type {
287 INSTR_TX, /* port_out = M */
288 INSTR_TX_I, /* port_out = I */
290 /* extract h.header */
311 /* validate h.header */
314 /* invalidate h.header */
315 INSTR_HDR_INVALIDATE,
319 * dst = HMEF, src = HMEFTI
321 INSTR_MOV, /* dst = MEF, src = MEFT */
322 INSTR_MOV_MH, /* dst = MEF, src = H */
323 INSTR_MOV_HM, /* dst = H, src = MEFT */
324 INSTR_MOV_HH, /* dst = H, src = H */
325 INSTR_MOV_I, /* dst = HMEF, src = I */
327 /* dma h.header t.field
328 * memcpy(h.header, t.field, sizeof(h.header))
341 * dst = HMEF, src = HMEFTI
343 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
344 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
345 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
346 INSTR_ALU_ADD_HH, /* dst = H, src = H */
347 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
348 INSTR_ALU_ADD_HI, /* dst = H, src = I */
352 * dst = HMEF, src = HMEFTI
354 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
355 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
356 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
357 INSTR_ALU_SUB_HH, /* dst = H, src = H */
358 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
359 INSTR_ALU_SUB_HI, /* dst = H, src = I */
362 * dst = dst '+ src[0:1] '+ src[2:3] + ...
363 * dst = H, src = {H, h.header}
365 INSTR_ALU_CKADD_FIELD, /* src = H */
366 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
367 INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
373 INSTR_ALU_CKSUB_FIELD,
377 * dst = HMEF, src = HMEFTI
379 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
380 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
381 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
382 INSTR_ALU_AND_HH, /* dst = H, src = H */
383 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
387 * dst = HMEF, src = HMEFTI
389 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
390 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
391 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
392 INSTR_ALU_OR_HH, /* dst = H, src = H */
393 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
397 * dst = HMEF, src = HMEFTI
399 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
400 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
401 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
402 INSTR_ALU_XOR_HH, /* dst = H, src = H */
403 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
407 * dst = HMEF, src = HMEFTI
409 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
410 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
411 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
412 INSTR_ALU_SHL_HH, /* dst = H, src = H */
413 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
414 INSTR_ALU_SHL_HI, /* dst = H, src = I */
418 * dst = HMEF, src = HMEFTI
420 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
421 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
422 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
423 INSTR_ALU_SHR_HH, /* dst = H, src = H */
424 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
425 INSTR_ALU_SHR_HI, /* dst = H, src = I */
427 /* regprefetch REGARRAY index
428 * prefetch REGARRAY[index]
431 INSTR_REGPREFETCH_RH, /* index = H */
432 INSTR_REGPREFETCH_RM, /* index = MEFT */
433 INSTR_REGPREFETCH_RI, /* index = I */
435 /* regrd dst REGARRAY index
436 * dst = REGARRAY[index]
437 * dst = HMEF, index = HMEFTI
439 INSTR_REGRD_HRH, /* dst = H, index = H */
440 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
441 INSTR_REGRD_HRI, /* dst = H, index = I */
442 INSTR_REGRD_MRH, /* dst = MEF, index = H */
443 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
444 INSTR_REGRD_MRI, /* dst = MEF, index = I */
446 /* regwr REGARRAY index src
447 * REGARRAY[index] = src
448 * index = HMEFTI, src = HMEFTI
450 INSTR_REGWR_RHH, /* index = H, src = H */
451 INSTR_REGWR_RHM, /* index = H, src = MEFT */
452 INSTR_REGWR_RHI, /* index = H, src = I */
453 INSTR_REGWR_RMH, /* index = MEFT, src = H */
454 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
455 INSTR_REGWR_RMI, /* index = MEFT, src = I */
456 INSTR_REGWR_RIH, /* index = I, src = H */
457 INSTR_REGWR_RIM, /* index = I, src = MEFT */
458 INSTR_REGWR_RII, /* index = I, src = I */
460 /* regadd REGARRAY index src
461 * REGARRAY[index] += src
462 * index = HMEFTI, src = HMEFTI
464 INSTR_REGADD_RHH, /* index = H, src = H */
465 INSTR_REGADD_RHM, /* index = H, src = MEFT */
466 INSTR_REGADD_RHI, /* index = H, src = I */
467 INSTR_REGADD_RMH, /* index = MEFT, src = H */
468 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
469 INSTR_REGADD_RMI, /* index = MEFT, src = I */
470 INSTR_REGADD_RIH, /* index = I, src = H */
471 INSTR_REGADD_RIM, /* index = I, src = MEFT */
472 INSTR_REGADD_RII, /* index = I, src = I */
474 /* metprefetch METARRAY index
475 * prefetch METARRAY[index]
478 INSTR_METPREFETCH_H, /* index = H */
479 INSTR_METPREFETCH_M, /* index = MEFT */
480 INSTR_METPREFETCH_I, /* index = I */
482 /* meter METARRAY index length color_in color_out
483 * color_out = meter(METARRAY[index], length, color_in)
484 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
486 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
487 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
488 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
489 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
490 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
491 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
492 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
493 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
494 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
495 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
496 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
497 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
502 /* extern e.obj.func */
513 /* jmpv LABEL h.header
514 * Jump if header is valid
518 /* jmpnv LABEL h.header
519 * Jump if header is invalid
524 * Jump if table lookup hit
529 * Jump if table lookup miss
536 INSTR_JMP_ACTION_HIT,
538 /* jmpna LABEL ACTION
539 * Jump if action not run
541 INSTR_JMP_ACTION_MISS,
544 * Jump if a is equal to b
545 * a = HMEFT, b = HMEFTI
547 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
548 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
549 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
550 INSTR_JMP_EQ_HH, /* a = H, b = H */
551 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
554 * Jump if a is not equal to b
555 * a = HMEFT, b = HMEFTI
557 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
558 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
559 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
560 INSTR_JMP_NEQ_HH, /* a = H, b = H */
561 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
564 * Jump if a is less than b
565 * a = HMEFT, b = HMEFTI
567 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
568 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
569 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
570 INSTR_JMP_LT_HH, /* a = H, b = H */
571 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
572 INSTR_JMP_LT_HI, /* a = H, b = I */
575 * Jump if a is greater than b
576 * a = HMEFT, b = HMEFTI
578 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
579 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
580 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
581 INSTR_JMP_GT_HH, /* a = H, b = H */
582 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
583 INSTR_JMP_GT_HI, /* a = H, b = I */
591 struct instr_operand {
612 uint8_t header_id[8];
613 uint8_t struct_id[8];
618 struct instr_hdr_validity {
626 struct instr_extern_obj {
631 struct instr_extern_func {
635 struct instr_dst_src {
636 struct instr_operand dst;
638 struct instr_operand src;
643 struct instr_regarray {
648 struct instr_operand idx;
653 struct instr_operand dstsrc;
663 struct instr_operand idx;
667 struct instr_operand length;
670 struct instr_operand color_in;
671 uint32_t color_in_val;
674 struct instr_operand color_out;
679 uint8_t header_id[8];
680 uint8_t struct_id[8];
691 struct instruction *ip;
694 struct instr_operand a;
700 struct instr_operand b;
706 enum instruction_type type;
709 struct instr_hdr_validity valid;
710 struct instr_dst_src mov;
711 struct instr_regarray regarray;
712 struct instr_meter meter;
713 struct instr_dma dma;
714 struct instr_dst_src alu;
715 struct instr_table table;
716 struct instr_extern_obj ext_obj;
717 struct instr_extern_func ext_func;
718 struct instr_jmp jmp;
722 struct instruction_data {
723 char label[RTE_SWX_NAME_SIZE];
724 char jmp_label[RTE_SWX_NAME_SIZE];
725 uint32_t n_users; /* user = jmp instruction to this instruction. */
733 TAILQ_ENTRY(action) node;
734 char name[RTE_SWX_NAME_SIZE];
735 struct struct_type *st;
736 struct instruction *instructions;
737 uint32_t n_instructions;
741 TAILQ_HEAD(action_tailq, action);
747 TAILQ_ENTRY(table_type) node;
748 char name[RTE_SWX_NAME_SIZE];
749 enum rte_swx_table_match_type match_type;
750 struct rte_swx_table_ops ops;
753 TAILQ_HEAD(table_type_tailq, table_type);
756 enum rte_swx_table_match_type match_type;
761 TAILQ_ENTRY(table) node;
762 char name[RTE_SWX_NAME_SIZE];
763 char args[RTE_SWX_NAME_SIZE];
764 struct table_type *type; /* NULL when n_fields == 0. */
767 struct match_field *fields;
769 struct header *header; /* Only valid when n_fields > 0. */
772 struct action **actions;
773 struct action *default_action;
774 uint8_t *default_action_data;
776 int default_action_is_const;
777 uint32_t action_data_size_max;
783 TAILQ_HEAD(table_tailq, table);
785 struct table_runtime {
786 rte_swx_table_lookup_t func;
791 struct table_statistics {
792 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
793 uint64_t *n_pkts_action;
800 TAILQ_ENTRY(regarray) node;
801 char name[RTE_SWX_NAME_SIZE];
807 TAILQ_HEAD(regarray_tailq, regarray);
809 struct regarray_runtime {
817 struct meter_profile {
818 TAILQ_ENTRY(meter_profile) node;
819 char name[RTE_SWX_NAME_SIZE];
820 struct rte_meter_trtcm_params params;
821 struct rte_meter_trtcm_profile profile;
825 TAILQ_HEAD(meter_profile_tailq, meter_profile);
828 TAILQ_ENTRY(metarray) node;
829 char name[RTE_SWX_NAME_SIZE];
834 TAILQ_HEAD(metarray_tailq, metarray);
837 struct rte_meter_trtcm m;
838 struct meter_profile *profile;
839 enum rte_color color_mask;
842 uint64_t n_pkts[RTE_COLORS];
843 uint64_t n_bytes[RTE_COLORS];
846 struct metarray_runtime {
847 struct meter *metarray;
856 struct rte_swx_pkt pkt;
862 /* Packet headers. */
863 struct header_runtime *headers; /* Extracted or generated headers. */
864 struct header_out_runtime *headers_out; /* Emitted headers. */
865 uint8_t *header_storage;
866 uint8_t *header_out_storage;
867 uint64_t valid_headers;
868 uint32_t n_headers_out;
870 /* Packet meta-data. */
874 struct table_runtime *tables;
875 struct rte_swx_table_state *table_state;
877 int hit; /* 0 = Miss, 1 = Hit. */
879 /* Extern objects and functions. */
880 struct extern_obj_runtime *extern_objs;
881 struct extern_func_runtime *extern_funcs;
884 struct instruction *ip;
885 struct instruction *ret;
888 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
889 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
890 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
892 #define HEADER_VALID(thread, header_id) \
893 MASK64_BIT_GET((thread)->valid_headers, header_id)
895 #define ALU(thread, ip, operator) \
897 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
898 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
899 uint64_t dst64 = *dst64_ptr; \
900 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
901 uint64_t dst = dst64 & dst64_mask; \
903 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
904 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
905 uint64_t src64 = *src64_ptr; \
906 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
907 uint64_t src = src64 & src64_mask; \
909 uint64_t result = dst operator src; \
911 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
914 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
916 #define ALU_MH(thread, ip, operator) \
918 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
919 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
920 uint64_t dst64 = *dst64_ptr; \
921 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
922 uint64_t dst = dst64 & dst64_mask; \
924 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
925 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
926 uint64_t src64 = *src64_ptr; \
927 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
929 uint64_t result = dst operator src; \
931 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
934 #define ALU_HM(thread, ip, operator) \
936 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
937 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
938 uint64_t dst64 = *dst64_ptr; \
939 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
940 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
942 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
943 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
944 uint64_t src64 = *src64_ptr; \
945 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
946 uint64_t src = src64 & src64_mask; \
948 uint64_t result = dst operator src; \
949 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
951 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
954 #define ALU_HM_FAST(thread, ip, operator) \
956 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
957 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
958 uint64_t dst64 = *dst64_ptr; \
959 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
960 uint64_t dst = dst64 & dst64_mask; \
962 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
963 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
964 uint64_t src64 = *src64_ptr; \
965 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
966 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
968 uint64_t result = dst operator src; \
970 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
973 #define ALU_HH(thread, ip, operator) \
975 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
976 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
977 uint64_t dst64 = *dst64_ptr; \
978 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
979 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
981 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
982 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
983 uint64_t src64 = *src64_ptr; \
984 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
986 uint64_t result = dst operator src; \
987 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
989 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
992 #define ALU_HH_FAST(thread, ip, operator) \
994 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
995 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
996 uint64_t dst64 = *dst64_ptr; \
997 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
998 uint64_t dst = dst64 & dst64_mask; \
1000 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1001 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1002 uint64_t src64 = *src64_ptr; \
1003 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1005 uint64_t result = dst operator src; \
1007 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1014 #define ALU_HM_FAST ALU
1016 #define ALU_HH_FAST ALU
1020 #define ALU_I(thread, ip, operator) \
1022 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1023 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1024 uint64_t dst64 = *dst64_ptr; \
1025 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1026 uint64_t dst = dst64 & dst64_mask; \
1028 uint64_t src = (ip)->alu.src_val; \
1030 uint64_t result = dst operator src; \
1032 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1035 #define ALU_MI ALU_I
1037 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1039 #define ALU_HI(thread, ip, operator) \
1041 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1042 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1043 uint64_t dst64 = *dst64_ptr; \
1044 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1045 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1047 uint64_t src = (ip)->alu.src_val; \
1049 uint64_t result = dst operator src; \
1050 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1052 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1057 #define ALU_HI ALU_I
1061 #define MOV(thread, ip) \
1063 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1064 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1065 uint64_t dst64 = *dst64_ptr; \
1066 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1068 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1069 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1070 uint64_t src64 = *src64_ptr; \
1071 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1072 uint64_t src = src64 & src64_mask; \
1074 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1077 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1079 #define MOV_MH(thread, ip) \
1081 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1082 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1083 uint64_t dst64 = *dst64_ptr; \
1084 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1086 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1087 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1088 uint64_t src64 = *src64_ptr; \
1089 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1091 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1094 #define MOV_HM(thread, ip) \
1096 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1097 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1098 uint64_t dst64 = *dst64_ptr; \
1099 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1101 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1102 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1103 uint64_t src64 = *src64_ptr; \
1104 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1105 uint64_t src = src64 & src64_mask; \
1107 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1108 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1111 #define MOV_HH(thread, ip) \
1113 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1114 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1115 uint64_t dst64 = *dst64_ptr; \
1116 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1118 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1119 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1120 uint64_t src64 = *src64_ptr; \
1122 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1123 src = src >> (64 - (ip)->mov.dst.n_bits); \
1124 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1135 #define MOV_I(thread, ip) \
1137 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1138 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1139 uint64_t dst64 = *dst64_ptr; \
1140 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1142 uint64_t src = (ip)->mov.src_val; \
1144 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1147 #define JMP_CMP(thread, ip, operator) \
1149 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1150 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1151 uint64_t a64 = *a64_ptr; \
1152 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1153 uint64_t a = a64 & a64_mask; \
1155 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1156 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1157 uint64_t b64 = *b64_ptr; \
1158 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1159 uint64_t b = b64 & b64_mask; \
1161 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1164 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1166 #define JMP_CMP_MH(thread, ip, operator) \
1168 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1169 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1170 uint64_t a64 = *a64_ptr; \
1171 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1172 uint64_t a = a64 & a64_mask; \
1174 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1175 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1176 uint64_t b64 = *b64_ptr; \
1177 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1179 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1182 #define JMP_CMP_HM(thread, ip, operator) \
1184 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1185 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1186 uint64_t a64 = *a64_ptr; \
1187 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1189 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1190 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1191 uint64_t b64 = *b64_ptr; \
1192 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1193 uint64_t b = b64 & b64_mask; \
1195 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1198 #define JMP_CMP_HH(thread, ip, operator) \
1200 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1201 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1202 uint64_t a64 = *a64_ptr; \
1203 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1205 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1206 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1207 uint64_t b64 = *b64_ptr; \
1208 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1210 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1213 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1215 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1216 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1217 uint64_t a64 = *a64_ptr; \
1218 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1220 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1221 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1222 uint64_t b64 = *b64_ptr; \
1223 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1225 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1230 #define JMP_CMP_MH JMP_CMP
1231 #define JMP_CMP_HM JMP_CMP
1232 #define JMP_CMP_HH JMP_CMP
1233 #define JMP_CMP_HH_FAST JMP_CMP
1237 #define JMP_CMP_I(thread, ip, operator) \
1239 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1240 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1241 uint64_t a64 = *a64_ptr; \
1242 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1243 uint64_t a = a64 & a64_mask; \
1245 uint64_t b = (ip)->jmp.b_val; \
1247 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1250 #define JMP_CMP_MI JMP_CMP_I
1252 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1254 #define JMP_CMP_HI(thread, ip, operator) \
1256 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1257 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1258 uint64_t a64 = *a64_ptr; \
1259 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1261 uint64_t b = (ip)->jmp.b_val; \
1263 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1268 #define JMP_CMP_HI JMP_CMP_I
1272 #define METADATA_READ(thread, offset, n_bits) \
1274 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1275 uint64_t m64 = *m64_ptr; \
1276 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1280 #define METADATA_WRITE(thread, offset, n_bits, value) \
1282 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1283 uint64_t m64 = *m64_ptr; \
1284 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1286 uint64_t m_new = value; \
1288 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1291 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1292 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1295 struct rte_swx_pipeline {
1296 struct struct_type_tailq struct_types;
1297 struct port_in_type_tailq port_in_types;
1298 struct port_in_tailq ports_in;
1299 struct port_out_type_tailq port_out_types;
1300 struct port_out_tailq ports_out;
1301 struct extern_type_tailq extern_types;
1302 struct extern_obj_tailq extern_objs;
1303 struct extern_func_tailq extern_funcs;
1304 struct header_tailq headers;
1305 struct struct_type *metadata_st;
1306 uint32_t metadata_struct_id;
1307 struct action_tailq actions;
1308 struct table_type_tailq table_types;
1309 struct table_tailq tables;
1310 struct regarray_tailq regarrays;
1311 struct meter_profile_tailq meter_profiles;
1312 struct metarray_tailq metarrays;
1314 struct port_in_runtime *in;
1315 struct port_out_runtime *out;
1316 struct instruction **action_instructions;
1317 struct rte_swx_table_state *table_state;
1318 struct table_statistics *table_stats;
1319 struct regarray_runtime *regarray_runtime;
1320 struct metarray_runtime *metarray_runtime;
1321 struct instruction *instructions;
1322 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1325 uint32_t n_ports_in;
1326 uint32_t n_ports_out;
1327 uint32_t n_extern_objs;
1328 uint32_t n_extern_funcs;
1331 uint32_t n_regarrays;
1332 uint32_t n_metarrays;
1336 uint32_t n_instructions;
1344 static struct struct_type *
1345 struct_type_find(struct rte_swx_pipeline *p, const char *name)
1347 struct struct_type *elem;
1349 TAILQ_FOREACH(elem, &p->struct_types, node)
1350 if (strcmp(elem->name, name) == 0)
1356 static struct field *
1357 struct_type_field_find(struct struct_type *st, const char *name)
1361 for (i = 0; i < st->n_fields; i++) {
1362 struct field *f = &st->fields[i];
1364 if (strcmp(f->name, name) == 0)
1372 rte_swx_pipeline_struct_type_register(struct rte_swx_pipeline *p,
1374 struct rte_swx_field_params *fields,
1377 struct struct_type *st;
1381 CHECK_NAME(name, EINVAL);
1382 CHECK(fields, EINVAL);
1383 CHECK(n_fields, EINVAL);
1385 for (i = 0; i < n_fields; i++) {
1386 struct rte_swx_field_params *f = &fields[i];
1389 CHECK_NAME(f->name, EINVAL);
1390 CHECK(f->n_bits, EINVAL);
1391 CHECK(f->n_bits <= 64, EINVAL);
1392 CHECK((f->n_bits & 7) == 0, EINVAL);
1394 for (j = 0; j < i; j++) {
1395 struct rte_swx_field_params *f_prev = &fields[j];
1397 CHECK(strcmp(f->name, f_prev->name), EINVAL);
1401 CHECK(!struct_type_find(p, name), EEXIST);
1403 /* Node allocation. */
1404 st = calloc(1, sizeof(struct struct_type));
1407 st->fields = calloc(n_fields, sizeof(struct field));
1413 /* Node initialization. */
1414 strcpy(st->name, name);
1415 for (i = 0; i < n_fields; i++) {
1416 struct field *dst = &st->fields[i];
1417 struct rte_swx_field_params *src = &fields[i];
1419 strcpy(dst->name, src->name);
1420 dst->n_bits = src->n_bits;
1421 dst->offset = st->n_bits;
1423 st->n_bits += src->n_bits;
1425 st->n_fields = n_fields;
1427 /* Node add to tailq. */
1428 TAILQ_INSERT_TAIL(&p->struct_types, st, node);
1434 struct_build(struct rte_swx_pipeline *p)
1438 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1439 struct thread *t = &p->threads[i];
1441 t->structs = calloc(p->n_structs, sizeof(uint8_t *));
1442 CHECK(t->structs, ENOMEM);
1449 struct_build_free(struct rte_swx_pipeline *p)
1453 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
1454 struct thread *t = &p->threads[i];
1462 struct_free(struct rte_swx_pipeline *p)
1464 struct_build_free(p);
1468 struct struct_type *elem;
1470 elem = TAILQ_FIRST(&p->struct_types);
1474 TAILQ_REMOVE(&p->struct_types, elem, node);
1483 static struct port_in_type *
1484 port_in_type_find(struct rte_swx_pipeline *p, const char *name)
1486 struct port_in_type *elem;
1491 TAILQ_FOREACH(elem, &p->port_in_types, node)
1492 if (strcmp(elem->name, name) == 0)
1499 rte_swx_pipeline_port_in_type_register(struct rte_swx_pipeline *p,
1501 struct rte_swx_port_in_ops *ops)
1503 struct port_in_type *elem;
1506 CHECK_NAME(name, EINVAL);
1508 CHECK(ops->create, EINVAL);
1509 CHECK(ops->free, EINVAL);
1510 CHECK(ops->pkt_rx, EINVAL);
1511 CHECK(ops->stats_read, EINVAL);
1513 CHECK(!port_in_type_find(p, name), EEXIST);
1515 /* Node allocation. */
1516 elem = calloc(1, sizeof(struct port_in_type));
1517 CHECK(elem, ENOMEM);
1519 /* Node initialization. */
1520 strcpy(elem->name, name);
1521 memcpy(&elem->ops, ops, sizeof(*ops));
1523 /* Node add to tailq. */
1524 TAILQ_INSERT_TAIL(&p->port_in_types, elem, node);
1529 static struct port_in *
1530 port_in_find(struct rte_swx_pipeline *p, uint32_t port_id)
1532 struct port_in *port;
1534 TAILQ_FOREACH(port, &p->ports_in, node)
1535 if (port->id == port_id)
1542 rte_swx_pipeline_port_in_config(struct rte_swx_pipeline *p,
1544 const char *port_type_name,
1547 struct port_in_type *type = NULL;
1548 struct port_in *port = NULL;
1553 CHECK(!port_in_find(p, port_id), EINVAL);
1555 CHECK_NAME(port_type_name, EINVAL);
1556 type = port_in_type_find(p, port_type_name);
1557 CHECK(type, EINVAL);
1559 obj = type->ops.create(args);
1562 /* Node allocation. */
1563 port = calloc(1, sizeof(struct port_in));
1564 CHECK(port, ENOMEM);
1566 /* Node initialization. */
1571 /* Node add to tailq. */
1572 TAILQ_INSERT_TAIL(&p->ports_in, port, node);
1573 if (p->n_ports_in < port_id + 1)
1574 p->n_ports_in = port_id + 1;
1580 port_in_build(struct rte_swx_pipeline *p)
1582 struct port_in *port;
1585 CHECK(p->n_ports_in, EINVAL);
1586 CHECK(rte_is_power_of_2(p->n_ports_in), EINVAL);
1588 for (i = 0; i < p->n_ports_in; i++)
1589 CHECK(port_in_find(p, i), EINVAL);
1591 p->in = calloc(p->n_ports_in, sizeof(struct port_in_runtime));
1592 CHECK(p->in, ENOMEM);
1594 TAILQ_FOREACH(port, &p->ports_in, node) {
1595 struct port_in_runtime *in = &p->in[port->id];
1597 in->pkt_rx = port->type->ops.pkt_rx;
1598 in->obj = port->obj;
1605 port_in_build_free(struct rte_swx_pipeline *p)
1612 port_in_free(struct rte_swx_pipeline *p)
1614 port_in_build_free(p);
1618 struct port_in *port;
1620 port = TAILQ_FIRST(&p->ports_in);
1624 TAILQ_REMOVE(&p->ports_in, port, node);
1625 port->type->ops.free(port->obj);
1629 /* Input port types. */
1631 struct port_in_type *elem;
1633 elem = TAILQ_FIRST(&p->port_in_types);
1637 TAILQ_REMOVE(&p->port_in_types, elem, node);
1645 static struct port_out_type *
1646 port_out_type_find(struct rte_swx_pipeline *p, const char *name)
1648 struct port_out_type *elem;
1653 TAILQ_FOREACH(elem, &p->port_out_types, node)
1654 if (!strcmp(elem->name, name))
1661 rte_swx_pipeline_port_out_type_register(struct rte_swx_pipeline *p,
1663 struct rte_swx_port_out_ops *ops)
1665 struct port_out_type *elem;
1668 CHECK_NAME(name, EINVAL);
1670 CHECK(ops->create, EINVAL);
1671 CHECK(ops->free, EINVAL);
1672 CHECK(ops->pkt_tx, EINVAL);
1673 CHECK(ops->stats_read, EINVAL);
1675 CHECK(!port_out_type_find(p, name), EEXIST);
1677 /* Node allocation. */
1678 elem = calloc(1, sizeof(struct port_out_type));
1679 CHECK(elem, ENOMEM);
1681 /* Node initialization. */
1682 strcpy(elem->name, name);
1683 memcpy(&elem->ops, ops, sizeof(*ops));
1685 /* Node add to tailq. */
1686 TAILQ_INSERT_TAIL(&p->port_out_types, elem, node);
1691 static struct port_out *
1692 port_out_find(struct rte_swx_pipeline *p, uint32_t port_id)
1694 struct port_out *port;
1696 TAILQ_FOREACH(port, &p->ports_out, node)
1697 if (port->id == port_id)
1704 rte_swx_pipeline_port_out_config(struct rte_swx_pipeline *p,
1706 const char *port_type_name,
1709 struct port_out_type *type = NULL;
1710 struct port_out *port = NULL;
1715 CHECK(!port_out_find(p, port_id), EINVAL);
1717 CHECK_NAME(port_type_name, EINVAL);
1718 type = port_out_type_find(p, port_type_name);
1719 CHECK(type, EINVAL);
1721 obj = type->ops.create(args);
1724 /* Node allocation. */
1725 port = calloc(1, sizeof(struct port_out));
1726 CHECK(port, ENOMEM);
1728 /* Node initialization. */
1733 /* Node add to tailq. */
1734 TAILQ_INSERT_TAIL(&p->ports_out, port, node);
1735 if (p->n_ports_out < port_id + 1)
1736 p->n_ports_out = port_id + 1;
1742 port_out_build(struct rte_swx_pipeline *p)
1744 struct port_out *port;
1747 CHECK(p->n_ports_out, EINVAL);
1749 for (i = 0; i < p->n_ports_out; i++)
1750 CHECK(port_out_find(p, i), EINVAL);
1752 p->out = calloc(p->n_ports_out, sizeof(struct port_out_runtime));
1753 CHECK(p->out, ENOMEM);
1755 TAILQ_FOREACH(port, &p->ports_out, node) {
1756 struct port_out_runtime *out = &p->out[port->id];
1758 out->pkt_tx = port->type->ops.pkt_tx;
1759 out->flush = port->type->ops.flush;
1760 out->obj = port->obj;
1767 port_out_build_free(struct rte_swx_pipeline *p)
1774 port_out_free(struct rte_swx_pipeline *p)
1776 port_out_build_free(p);
1780 struct port_out *port;
1782 port = TAILQ_FIRST(&p->ports_out);
1786 TAILQ_REMOVE(&p->ports_out, port, node);
1787 port->type->ops.free(port->obj);
1791 /* Output port types. */
1793 struct port_out_type *elem;
1795 elem = TAILQ_FIRST(&p->port_out_types);
1799 TAILQ_REMOVE(&p->port_out_types, elem, node);
1807 static struct extern_type *
1808 extern_type_find(struct rte_swx_pipeline *p, const char *name)
1810 struct extern_type *elem;
1812 TAILQ_FOREACH(elem, &p->extern_types, node)
1813 if (strcmp(elem->name, name) == 0)
1819 static struct extern_type_member_func *
1820 extern_type_member_func_find(struct extern_type *type, const char *name)
1822 struct extern_type_member_func *elem;
1824 TAILQ_FOREACH(elem, &type->funcs, node)
1825 if (strcmp(elem->name, name) == 0)
1831 static struct extern_obj *
1832 extern_obj_find(struct rte_swx_pipeline *p, const char *name)
1834 struct extern_obj *elem;
1836 TAILQ_FOREACH(elem, &p->extern_objs, node)
1837 if (strcmp(elem->name, name) == 0)
1843 static struct extern_type_member_func *
1844 extern_obj_member_func_parse(struct rte_swx_pipeline *p,
1846 struct extern_obj **obj)
1848 struct extern_obj *object;
1849 struct extern_type_member_func *func;
1850 char *object_name, *func_name;
1852 if (name[0] != 'e' || name[1] != '.')
1855 object_name = strdup(&name[2]);
1859 func_name = strchr(object_name, '.');
1868 object = extern_obj_find(p, object_name);
1874 func = extern_type_member_func_find(object->type, func_name);
1887 static struct field *
1888 extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
1890 struct extern_obj **object)
1892 struct extern_obj *obj;
1894 char *obj_name, *field_name;
1896 if ((name[0] != 'e') || (name[1] != '.'))
1899 obj_name = strdup(&name[2]);
1903 field_name = strchr(obj_name, '.');
1912 obj = extern_obj_find(p, obj_name);
1918 f = struct_type_field_find(obj->type->mailbox_struct_type, field_name);
1932 rte_swx_pipeline_extern_type_register(struct rte_swx_pipeline *p,
1934 const char *mailbox_struct_type_name,
1935 rte_swx_extern_type_constructor_t constructor,
1936 rte_swx_extern_type_destructor_t destructor)
1938 struct extern_type *elem;
1939 struct struct_type *mailbox_struct_type;
1943 CHECK_NAME(name, EINVAL);
1944 CHECK(!extern_type_find(p, name), EEXIST);
1946 CHECK_NAME(mailbox_struct_type_name, EINVAL);
1947 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
1948 CHECK(mailbox_struct_type, EINVAL);
1950 CHECK(constructor, EINVAL);
1951 CHECK(destructor, EINVAL);
1953 /* Node allocation. */
1954 elem = calloc(1, sizeof(struct extern_type));
1955 CHECK(elem, ENOMEM);
1957 /* Node initialization. */
1958 strcpy(elem->name, name);
1959 elem->mailbox_struct_type = mailbox_struct_type;
1960 elem->constructor = constructor;
1961 elem->destructor = destructor;
1962 TAILQ_INIT(&elem->funcs);
1964 /* Node add to tailq. */
1965 TAILQ_INSERT_TAIL(&p->extern_types, elem, node);
1971 rte_swx_pipeline_extern_type_member_func_register(struct rte_swx_pipeline *p,
1972 const char *extern_type_name,
1974 rte_swx_extern_type_member_func_t member_func)
1976 struct extern_type *type;
1977 struct extern_type_member_func *type_member;
1981 CHECK_NAME(extern_type_name, EINVAL);
1982 type = extern_type_find(p, extern_type_name);
1983 CHECK(type, EINVAL);
1984 CHECK(type->n_funcs < RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX, ENOSPC);
1986 CHECK_NAME(name, EINVAL);
1987 CHECK(!extern_type_member_func_find(type, name), EEXIST);
1989 CHECK(member_func, EINVAL);
1991 /* Node allocation. */
1992 type_member = calloc(1, sizeof(struct extern_type_member_func));
1993 CHECK(type_member, ENOMEM);
1995 /* Node initialization. */
1996 strcpy(type_member->name, name);
1997 type_member->func = member_func;
1998 type_member->id = type->n_funcs;
2000 /* Node add to tailq. */
2001 TAILQ_INSERT_TAIL(&type->funcs, type_member, node);
2008 rte_swx_pipeline_extern_object_config(struct rte_swx_pipeline *p,
2009 const char *extern_type_name,
2013 struct extern_type *type;
2014 struct extern_obj *obj;
2019 CHECK_NAME(extern_type_name, EINVAL);
2020 type = extern_type_find(p, extern_type_name);
2021 CHECK(type, EINVAL);
2023 CHECK_NAME(name, EINVAL);
2024 CHECK(!extern_obj_find(p, name), EEXIST);
2026 /* Node allocation. */
2027 obj = calloc(1, sizeof(struct extern_obj));
2030 /* Object construction. */
2031 obj_handle = type->constructor(args);
2037 /* Node initialization. */
2038 strcpy(obj->name, name);
2040 obj->obj = obj_handle;
2041 obj->struct_id = p->n_structs;
2042 obj->id = p->n_extern_objs;
2044 /* Node add to tailq. */
2045 TAILQ_INSERT_TAIL(&p->extern_objs, obj, node);
2053 extern_obj_build(struct rte_swx_pipeline *p)
2057 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2058 struct thread *t = &p->threads[i];
2059 struct extern_obj *obj;
2061 t->extern_objs = calloc(p->n_extern_objs,
2062 sizeof(struct extern_obj_runtime));
2063 CHECK(t->extern_objs, ENOMEM);
2065 TAILQ_FOREACH(obj, &p->extern_objs, node) {
2066 struct extern_obj_runtime *r =
2067 &t->extern_objs[obj->id];
2068 struct extern_type_member_func *func;
2069 uint32_t mailbox_size =
2070 obj->type->mailbox_struct_type->n_bits / 8;
2074 r->mailbox = calloc(1, mailbox_size);
2075 CHECK(r->mailbox, ENOMEM);
2077 TAILQ_FOREACH(func, &obj->type->funcs, node)
2078 r->funcs[func->id] = func->func;
2080 t->structs[obj->struct_id] = r->mailbox;
2088 extern_obj_build_free(struct rte_swx_pipeline *p)
2092 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2093 struct thread *t = &p->threads[i];
2096 if (!t->extern_objs)
2099 for (j = 0; j < p->n_extern_objs; j++) {
2100 struct extern_obj_runtime *r = &t->extern_objs[j];
2105 free(t->extern_objs);
2106 t->extern_objs = NULL;
2111 extern_obj_free(struct rte_swx_pipeline *p)
2113 extern_obj_build_free(p);
2115 /* Extern objects. */
2117 struct extern_obj *elem;
2119 elem = TAILQ_FIRST(&p->extern_objs);
2123 TAILQ_REMOVE(&p->extern_objs, elem, node);
2125 elem->type->destructor(elem->obj);
2131 struct extern_type *elem;
2133 elem = TAILQ_FIRST(&p->extern_types);
2137 TAILQ_REMOVE(&p->extern_types, elem, node);
2140 struct extern_type_member_func *func;
2142 func = TAILQ_FIRST(&elem->funcs);
2146 TAILQ_REMOVE(&elem->funcs, func, node);
2157 static struct extern_func *
2158 extern_func_find(struct rte_swx_pipeline *p, const char *name)
2160 struct extern_func *elem;
2162 TAILQ_FOREACH(elem, &p->extern_funcs, node)
2163 if (strcmp(elem->name, name) == 0)
2169 static struct extern_func *
2170 extern_func_parse(struct rte_swx_pipeline *p,
2173 if (name[0] != 'f' || name[1] != '.')
2176 return extern_func_find(p, &name[2]);
2179 static struct field *
2180 extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
2182 struct extern_func **function)
2184 struct extern_func *func;
2186 char *func_name, *field_name;
2188 if ((name[0] != 'f') || (name[1] != '.'))
2191 func_name = strdup(&name[2]);
2195 field_name = strchr(func_name, '.');
2204 func = extern_func_find(p, func_name);
2210 f = struct_type_field_find(func->mailbox_struct_type, field_name);
2224 rte_swx_pipeline_extern_func_register(struct rte_swx_pipeline *p,
2226 const char *mailbox_struct_type_name,
2227 rte_swx_extern_func_t func)
2229 struct extern_func *f;
2230 struct struct_type *mailbox_struct_type;
2234 CHECK_NAME(name, EINVAL);
2235 CHECK(!extern_func_find(p, name), EEXIST);
2237 CHECK_NAME(mailbox_struct_type_name, EINVAL);
2238 mailbox_struct_type = struct_type_find(p, mailbox_struct_type_name);
2239 CHECK(mailbox_struct_type, EINVAL);
2241 CHECK(func, EINVAL);
2243 /* Node allocation. */
2244 f = calloc(1, sizeof(struct extern_func));
2245 CHECK(func, ENOMEM);
2247 /* Node initialization. */
2248 strcpy(f->name, name);
2249 f->mailbox_struct_type = mailbox_struct_type;
2251 f->struct_id = p->n_structs;
2252 f->id = p->n_extern_funcs;
2254 /* Node add to tailq. */
2255 TAILQ_INSERT_TAIL(&p->extern_funcs, f, node);
2256 p->n_extern_funcs++;
2263 extern_func_build(struct rte_swx_pipeline *p)
2267 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2268 struct thread *t = &p->threads[i];
2269 struct extern_func *func;
2271 /* Memory allocation. */
2272 t->extern_funcs = calloc(p->n_extern_funcs,
2273 sizeof(struct extern_func_runtime));
2274 CHECK(t->extern_funcs, ENOMEM);
2276 /* Extern function. */
2277 TAILQ_FOREACH(func, &p->extern_funcs, node) {
2278 struct extern_func_runtime *r =
2279 &t->extern_funcs[func->id];
2280 uint32_t mailbox_size =
2281 func->mailbox_struct_type->n_bits / 8;
2283 r->func = func->func;
2285 r->mailbox = calloc(1, mailbox_size);
2286 CHECK(r->mailbox, ENOMEM);
2288 t->structs[func->struct_id] = r->mailbox;
2296 extern_func_build_free(struct rte_swx_pipeline *p)
2300 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2301 struct thread *t = &p->threads[i];
2304 if (!t->extern_funcs)
2307 for (j = 0; j < p->n_extern_funcs; j++) {
2308 struct extern_func_runtime *r = &t->extern_funcs[j];
2313 free(t->extern_funcs);
2314 t->extern_funcs = NULL;
2319 extern_func_free(struct rte_swx_pipeline *p)
2321 extern_func_build_free(p);
2324 struct extern_func *elem;
2326 elem = TAILQ_FIRST(&p->extern_funcs);
2330 TAILQ_REMOVE(&p->extern_funcs, elem, node);
2338 static struct header *
2339 header_find(struct rte_swx_pipeline *p, const char *name)
2341 struct header *elem;
2343 TAILQ_FOREACH(elem, &p->headers, node)
2344 if (strcmp(elem->name, name) == 0)
2350 static struct header *
2351 header_parse(struct rte_swx_pipeline *p,
2354 if (name[0] != 'h' || name[1] != '.')
2357 return header_find(p, &name[2]);
2360 static struct field *
2361 header_field_parse(struct rte_swx_pipeline *p,
2363 struct header **header)
2367 char *header_name, *field_name;
2369 if ((name[0] != 'h') || (name[1] != '.'))
2372 header_name = strdup(&name[2]);
2376 field_name = strchr(header_name, '.');
2385 h = header_find(p, header_name);
2391 f = struct_type_field_find(h->st, field_name);
2405 rte_swx_pipeline_packet_header_register(struct rte_swx_pipeline *p,
2407 const char *struct_type_name)
2409 struct struct_type *st;
2411 size_t n_headers_max;
2414 CHECK_NAME(name, EINVAL);
2415 CHECK_NAME(struct_type_name, EINVAL);
2417 CHECK(!header_find(p, name), EEXIST);
2419 st = struct_type_find(p, struct_type_name);
2422 n_headers_max = RTE_SIZEOF_FIELD(struct thread, valid_headers) * 8;
2423 CHECK(p->n_headers < n_headers_max, ENOSPC);
2425 /* Node allocation. */
2426 h = calloc(1, sizeof(struct header));
2429 /* Node initialization. */
2430 strcpy(h->name, name);
2432 h->struct_id = p->n_structs;
2433 h->id = p->n_headers;
2435 /* Node add to tailq. */
2436 TAILQ_INSERT_TAIL(&p->headers, h, node);
2444 header_build(struct rte_swx_pipeline *p)
2447 uint32_t n_bytes = 0, i;
2449 TAILQ_FOREACH(h, &p->headers, node) {
2450 n_bytes += h->st->n_bits / 8;
2453 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2454 struct thread *t = &p->threads[i];
2455 uint32_t offset = 0;
2457 t->headers = calloc(p->n_headers,
2458 sizeof(struct header_runtime));
2459 CHECK(t->headers, ENOMEM);
2461 t->headers_out = calloc(p->n_headers,
2462 sizeof(struct header_out_runtime));
2463 CHECK(t->headers_out, ENOMEM);
2465 t->header_storage = calloc(1, n_bytes);
2466 CHECK(t->header_storage, ENOMEM);
2468 t->header_out_storage = calloc(1, n_bytes);
2469 CHECK(t->header_out_storage, ENOMEM);
2471 TAILQ_FOREACH(h, &p->headers, node) {
2472 uint8_t *header_storage;
2474 header_storage = &t->header_storage[offset];
2475 offset += h->st->n_bits / 8;
2477 t->headers[h->id].ptr0 = header_storage;
2478 t->structs[h->struct_id] = header_storage;
2486 header_build_free(struct rte_swx_pipeline *p)
2490 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2491 struct thread *t = &p->threads[i];
2493 free(t->headers_out);
2494 t->headers_out = NULL;
2499 free(t->header_out_storage);
2500 t->header_out_storage = NULL;
2502 free(t->header_storage);
2503 t->header_storage = NULL;
2508 header_free(struct rte_swx_pipeline *p)
2510 header_build_free(p);
2513 struct header *elem;
2515 elem = TAILQ_FIRST(&p->headers);
2519 TAILQ_REMOVE(&p->headers, elem, node);
2527 static struct field *
2528 metadata_field_parse(struct rte_swx_pipeline *p, const char *name)
2530 if (!p->metadata_st)
2533 if (name[0] != 'm' || name[1] != '.')
2536 return struct_type_field_find(p->metadata_st, &name[2]);
2540 rte_swx_pipeline_packet_metadata_register(struct rte_swx_pipeline *p,
2541 const char *struct_type_name)
2543 struct struct_type *st = NULL;
2547 CHECK_NAME(struct_type_name, EINVAL);
2548 st = struct_type_find(p, struct_type_name);
2550 CHECK(!p->metadata_st, EINVAL);
2552 p->metadata_st = st;
2553 p->metadata_struct_id = p->n_structs;
2561 metadata_build(struct rte_swx_pipeline *p)
2563 uint32_t n_bytes = p->metadata_st->n_bits / 8;
2566 /* Thread-level initialization. */
2567 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2568 struct thread *t = &p->threads[i];
2571 metadata = calloc(1, n_bytes);
2572 CHECK(metadata, ENOMEM);
2574 t->metadata = metadata;
2575 t->structs[p->metadata_struct_id] = metadata;
2582 metadata_build_free(struct rte_swx_pipeline *p)
2586 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
2587 struct thread *t = &p->threads[i];
2595 metadata_free(struct rte_swx_pipeline *p)
2597 metadata_build_free(p);
2604 instruction_is_tx(enum instruction_type type)
2617 instruction_is_jmp(struct instruction *instr)
2619 switch (instr->type) {
2621 case INSTR_JMP_VALID:
2622 case INSTR_JMP_INVALID:
2624 case INSTR_JMP_MISS:
2625 case INSTR_JMP_ACTION_HIT:
2626 case INSTR_JMP_ACTION_MISS:
2628 case INSTR_JMP_EQ_MH:
2629 case INSTR_JMP_EQ_HM:
2630 case INSTR_JMP_EQ_HH:
2631 case INSTR_JMP_EQ_I:
2633 case INSTR_JMP_NEQ_MH:
2634 case INSTR_JMP_NEQ_HM:
2635 case INSTR_JMP_NEQ_HH:
2636 case INSTR_JMP_NEQ_I:
2638 case INSTR_JMP_LT_MH:
2639 case INSTR_JMP_LT_HM:
2640 case INSTR_JMP_LT_HH:
2641 case INSTR_JMP_LT_MI:
2642 case INSTR_JMP_LT_HI:
2644 case INSTR_JMP_GT_MH:
2645 case INSTR_JMP_GT_HM:
2646 case INSTR_JMP_GT_HH:
2647 case INSTR_JMP_GT_MI:
2648 case INSTR_JMP_GT_HI:
2656 static struct field *
2657 action_field_parse(struct action *action, const char *name);
2659 static struct field *
2660 struct_field_parse(struct rte_swx_pipeline *p,
2661 struct action *action,
2663 uint32_t *struct_id)
2670 struct header *header;
2672 f = header_field_parse(p, name, &header);
2676 *struct_id = header->struct_id;
2682 f = metadata_field_parse(p, name);
2686 *struct_id = p->metadata_struct_id;
2695 f = action_field_parse(action, name);
2705 struct extern_obj *obj;
2707 f = extern_obj_mailbox_field_parse(p, name, &obj);
2711 *struct_id = obj->struct_id;
2717 struct extern_func *func;
2719 f = extern_func_mailbox_field_parse(p, name, &func);
2723 *struct_id = func->struct_id;
2733 pipeline_port_inc(struct rte_swx_pipeline *p)
2735 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
2739 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
2741 t->ip = p->instructions;
2745 thread_ip_set(struct thread *t, struct instruction *ip)
2751 thread_ip_action_call(struct rte_swx_pipeline *p,
2756 t->ip = p->action_instructions[action_id];
2760 thread_ip_inc(struct rte_swx_pipeline *p);
2763 thread_ip_inc(struct rte_swx_pipeline *p)
2765 struct thread *t = &p->threads[p->thread_id];
2771 thread_ip_inc_cond(struct thread *t, int cond)
2777 thread_yield(struct rte_swx_pipeline *p)
2779 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2783 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
2785 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
2792 instr_rx_translate(struct rte_swx_pipeline *p,
2793 struct action *action,
2796 struct instruction *instr,
2797 struct instruction_data *data __rte_unused)
2801 CHECK(!action, EINVAL);
2802 CHECK(n_tokens == 2, EINVAL);
2804 f = metadata_field_parse(p, tokens[1]);
2807 instr->type = INSTR_RX;
2808 instr->io.io.offset = f->offset / 8;
2809 instr->io.io.n_bits = f->n_bits;
2814 instr_rx_exec(struct rte_swx_pipeline *p);
2817 instr_rx_exec(struct rte_swx_pipeline *p)
2819 struct thread *t = &p->threads[p->thread_id];
2820 struct instruction *ip = t->ip;
2821 struct port_in_runtime *port = &p->in[p->port_id];
2822 struct rte_swx_pkt *pkt = &t->pkt;
2826 pkt_received = port->pkt_rx(port->obj, pkt);
2827 t->ptr = &pkt->pkt[pkt->offset];
2828 rte_prefetch0(t->ptr);
2830 TRACE("[Thread %2u] rx %s from port %u\n",
2832 pkt_received ? "1 pkt" : "0 pkts",
2836 t->valid_headers = 0;
2837 t->n_headers_out = 0;
2840 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
2843 t->table_state = p->table_state;
2846 pipeline_port_inc(p);
2847 thread_ip_inc_cond(t, pkt_received);
2855 instr_tx_translate(struct rte_swx_pipeline *p,
2856 struct action *action __rte_unused,
2859 struct instruction *instr,
2860 struct instruction_data *data __rte_unused)
2862 char *port = tokens[1];
2866 CHECK(n_tokens == 2, EINVAL);
2868 f = metadata_field_parse(p, port);
2870 instr->type = INSTR_TX;
2871 instr->io.io.offset = f->offset / 8;
2872 instr->io.io.n_bits = f->n_bits;
2877 port_val = strtoul(port, &port, 0);
2878 CHECK(!port[0], EINVAL);
2880 instr->type = INSTR_TX_I;
2881 instr->io.io.val = port_val;
2886 instr_drop_translate(struct rte_swx_pipeline *p,
2887 struct action *action __rte_unused,
2888 char **tokens __rte_unused,
2890 struct instruction *instr,
2891 struct instruction_data *data __rte_unused)
2893 CHECK(n_tokens == 1, EINVAL);
2896 instr->type = INSTR_TX_I;
2897 instr->io.io.val = p->n_ports_out - 1;
2902 emit_handler(struct thread *t)
2904 struct header_out_runtime *h0 = &t->headers_out[0];
2905 struct header_out_runtime *h1 = &t->headers_out[1];
2906 uint32_t offset = 0, i;
2908 /* No header change or header decapsulation. */
2909 if ((t->n_headers_out == 1) &&
2910 (h0->ptr + h0->n_bytes == t->ptr)) {
2911 TRACE("Emit handler: no header change or header decap.\n");
2913 t->pkt.offset -= h0->n_bytes;
2914 t->pkt.length += h0->n_bytes;
2919 /* Header encapsulation (optionally, with prior header decasulation). */
2920 if ((t->n_headers_out == 2) &&
2921 (h1->ptr + h1->n_bytes == t->ptr) &&
2922 (h0->ptr == h0->ptr0)) {
2925 TRACE("Emit handler: header encapsulation.\n");
2927 offset = h0->n_bytes + h1->n_bytes;
2928 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
2929 t->pkt.offset -= offset;
2930 t->pkt.length += offset;
2935 /* Header insertion. */
2938 /* Header extraction. */
2941 /* For any other case. */
2942 TRACE("Emit handler: complex case.\n");
2944 for (i = 0; i < t->n_headers_out; i++) {
2945 struct header_out_runtime *h = &t->headers_out[i];
2947 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
2948 offset += h->n_bytes;
2952 memcpy(t->ptr - offset, t->header_out_storage, offset);
2953 t->pkt.offset -= offset;
2954 t->pkt.length += offset;
2959 instr_tx_exec(struct rte_swx_pipeline *p);
2962 instr_tx_exec(struct rte_swx_pipeline *p)
2964 struct thread *t = &p->threads[p->thread_id];
2965 struct instruction *ip = t->ip;
2966 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2967 struct port_out_runtime *port = &p->out[port_id];
2968 struct rte_swx_pkt *pkt = &t->pkt;
2970 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
2978 port->pkt_tx(port->obj, pkt);
2981 thread_ip_reset(p, t);
2986 instr_tx_i_exec(struct rte_swx_pipeline *p)
2988 struct thread *t = &p->threads[p->thread_id];
2989 struct instruction *ip = t->ip;
2990 uint64_t port_id = ip->io.io.val;
2991 struct port_out_runtime *port = &p->out[port_id];
2992 struct rte_swx_pkt *pkt = &t->pkt;
2994 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
3002 port->pkt_tx(port->obj, pkt);
3005 thread_ip_reset(p, t);
3013 instr_hdr_extract_translate(struct rte_swx_pipeline *p,
3014 struct action *action,
3017 struct instruction *instr,
3018 struct instruction_data *data __rte_unused)
3022 CHECK(!action, EINVAL);
3023 CHECK(n_tokens == 2, EINVAL);
3025 h = header_parse(p, tokens[1]);
3028 instr->type = INSTR_HDR_EXTRACT;
3029 instr->io.hdr.header_id[0] = h->id;
3030 instr->io.hdr.struct_id[0] = h->struct_id;
3031 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3036 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract);
3039 __instr_hdr_extract_exec(struct rte_swx_pipeline *p, uint32_t n_extract)
3041 struct thread *t = &p->threads[p->thread_id];
3042 struct instruction *ip = t->ip;
3043 uint64_t valid_headers = t->valid_headers;
3044 uint8_t *ptr = t->ptr;
3045 uint32_t offset = t->pkt.offset;
3046 uint32_t length = t->pkt.length;
3049 for (i = 0; i < n_extract; i++) {
3050 uint32_t header_id = ip->io.hdr.header_id[i];
3051 uint32_t struct_id = ip->io.hdr.struct_id[i];
3052 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3054 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
3060 t->structs[struct_id] = ptr;
3061 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3070 t->valid_headers = valid_headers;
3073 t->pkt.offset = offset;
3074 t->pkt.length = length;
3079 instr_hdr_extract_exec(struct rte_swx_pipeline *p)
3081 __instr_hdr_extract_exec(p, 1);
3088 instr_hdr_extract2_exec(struct rte_swx_pipeline *p)
3090 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3093 __instr_hdr_extract_exec(p, 2);
3100 instr_hdr_extract3_exec(struct rte_swx_pipeline *p)
3102 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3105 __instr_hdr_extract_exec(p, 3);
3112 instr_hdr_extract4_exec(struct rte_swx_pipeline *p)
3114 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3117 __instr_hdr_extract_exec(p, 4);
3124 instr_hdr_extract5_exec(struct rte_swx_pipeline *p)
3126 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3129 __instr_hdr_extract_exec(p, 5);
3136 instr_hdr_extract6_exec(struct rte_swx_pipeline *p)
3138 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3141 __instr_hdr_extract_exec(p, 6);
3148 instr_hdr_extract7_exec(struct rte_swx_pipeline *p)
3150 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3153 __instr_hdr_extract_exec(p, 7);
3160 instr_hdr_extract8_exec(struct rte_swx_pipeline *p)
3162 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3165 __instr_hdr_extract_exec(p, 8);
3175 instr_hdr_emit_translate(struct rte_swx_pipeline *p,
3176 struct action *action __rte_unused,
3179 struct instruction *instr,
3180 struct instruction_data *data __rte_unused)
3184 CHECK(n_tokens == 2, EINVAL);
3186 h = header_parse(p, tokens[1]);
3189 instr->type = INSTR_HDR_EMIT;
3190 instr->io.hdr.header_id[0] = h->id;
3191 instr->io.hdr.struct_id[0] = h->struct_id;
3192 instr->io.hdr.n_bytes[0] = h->st->n_bits / 8;
3197 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit);
3200 __instr_hdr_emit_exec(struct rte_swx_pipeline *p, uint32_t n_emit)
3202 struct thread *t = &p->threads[p->thread_id];
3203 struct instruction *ip = t->ip;
3204 uint64_t valid_headers = t->valid_headers;
3205 uint32_t n_headers_out = t->n_headers_out;
3206 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
3207 uint8_t *ho_ptr = NULL;
3208 uint32_t ho_nbytes = 0, first = 1, i;
3210 for (i = 0; i < n_emit; i++) {
3211 uint32_t header_id = ip->io.hdr.header_id[i];
3212 uint32_t struct_id = ip->io.hdr.struct_id[i];
3213 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
3215 struct header_runtime *hi = &t->headers[header_id];
3216 uint8_t *hi_ptr = t->structs[struct_id];
3218 if (!MASK64_BIT_GET(valid_headers, header_id))
3221 TRACE("[Thread %2u]: emit header %u\n",
3229 if (!t->n_headers_out) {
3230 ho = &t->headers_out[0];
3232 ho->ptr0 = hi->ptr0;
3236 ho_nbytes = n_bytes;
3243 ho_nbytes = ho->n_bytes;
3247 if (ho_ptr + ho_nbytes == hi_ptr) {
3248 ho_nbytes += n_bytes;
3250 ho->n_bytes = ho_nbytes;
3253 ho->ptr0 = hi->ptr0;
3257 ho_nbytes = n_bytes;
3263 ho->n_bytes = ho_nbytes;
3264 t->n_headers_out = n_headers_out;
3268 instr_hdr_emit_exec(struct rte_swx_pipeline *p)
3270 __instr_hdr_emit_exec(p, 1);
3277 instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p)
3279 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3282 __instr_hdr_emit_exec(p, 1);
3287 instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p)
3289 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3292 __instr_hdr_emit_exec(p, 2);
3297 instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p)
3299 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3302 __instr_hdr_emit_exec(p, 3);
3307 instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p)
3309 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3312 __instr_hdr_emit_exec(p, 4);
3317 instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p)
3319 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3322 __instr_hdr_emit_exec(p, 5);
3327 instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p)
3329 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3332 __instr_hdr_emit_exec(p, 6);
3337 instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p)
3339 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3342 __instr_hdr_emit_exec(p, 7);
3347 instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p)
3349 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n",
3352 __instr_hdr_emit_exec(p, 8);
3360 instr_hdr_validate_translate(struct rte_swx_pipeline *p,
3361 struct action *action __rte_unused,
3364 struct instruction *instr,
3365 struct instruction_data *data __rte_unused)
3369 CHECK(n_tokens == 2, EINVAL);
3371 h = header_parse(p, tokens[1]);
3374 instr->type = INSTR_HDR_VALIDATE;
3375 instr->valid.header_id = h->id;
3380 instr_hdr_validate_exec(struct rte_swx_pipeline *p)
3382 struct thread *t = &p->threads[p->thread_id];
3383 struct instruction *ip = t->ip;
3384 uint32_t header_id = ip->valid.header_id;
3386 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
3389 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
3399 instr_hdr_invalidate_translate(struct rte_swx_pipeline *p,
3400 struct action *action __rte_unused,
3403 struct instruction *instr,
3404 struct instruction_data *data __rte_unused)
3408 CHECK(n_tokens == 2, EINVAL);
3410 h = header_parse(p, tokens[1]);
3413 instr->type = INSTR_HDR_INVALIDATE;
3414 instr->valid.header_id = h->id;
3419 instr_hdr_invalidate_exec(struct rte_swx_pipeline *p)
3421 struct thread *t = &p->threads[p->thread_id];
3422 struct instruction *ip = t->ip;
3423 uint32_t header_id = ip->valid.header_id;
3425 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
3428 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
3437 static struct table *
3438 table_find(struct rte_swx_pipeline *p, const char *name);
3441 instr_table_translate(struct rte_swx_pipeline *p,
3442 struct action *action,
3445 struct instruction *instr,
3446 struct instruction_data *data __rte_unused)
3450 CHECK(!action, EINVAL);
3451 CHECK(n_tokens == 2, EINVAL);
3453 t = table_find(p, tokens[1]);
3456 instr->type = INSTR_TABLE;
3457 instr->table.table_id = t->id;
3462 instr_table_exec(struct rte_swx_pipeline *p)
3464 struct thread *t = &p->threads[p->thread_id];
3465 struct instruction *ip = t->ip;
3466 uint32_t table_id = ip->table.table_id;
3467 struct rte_swx_table_state *ts = &t->table_state[table_id];
3468 struct table_runtime *table = &t->tables[table_id];
3469 struct table_statistics *stats = &p->table_stats[table_id];
3470 uint64_t action_id, n_pkts_hit, n_pkts_action;
3471 uint8_t *action_data;
3475 done = table->func(ts->obj,
3483 TRACE("[Thread %2u] table %u (not finalized)\n",
3491 action_id = hit ? action_id : ts->default_action_id;
3492 action_data = hit ? action_data : ts->default_action_data;
3493 n_pkts_hit = stats->n_pkts_hit[hit];
3494 n_pkts_action = stats->n_pkts_action[action_id];
3496 TRACE("[Thread %2u] table %u (%s, action %u)\n",
3499 hit ? "hit" : "miss",
3500 (uint32_t)action_id);
3502 t->action_id = action_id;
3503 t->structs[0] = action_data;
3505 stats->n_pkts_hit[hit] = n_pkts_hit + 1;
3506 stats->n_pkts_action[action_id] = n_pkts_action + 1;
3509 thread_ip_action_call(p, t, action_id);
3516 instr_extern_translate(struct rte_swx_pipeline *p,
3517 struct action *action __rte_unused,
3520 struct instruction *instr,
3521 struct instruction_data *data __rte_unused)
3523 char *token = tokens[1];
3525 CHECK(n_tokens == 2, EINVAL);
3527 if (token[0] == 'e') {
3528 struct extern_obj *obj;
3529 struct extern_type_member_func *func;
3531 func = extern_obj_member_func_parse(p, token, &obj);
3532 CHECK(func, EINVAL);
3534 instr->type = INSTR_EXTERN_OBJ;
3535 instr->ext_obj.ext_obj_id = obj->id;
3536 instr->ext_obj.func_id = func->id;
3541 if (token[0] == 'f') {
3542 struct extern_func *func;
3544 func = extern_func_parse(p, token);
3545 CHECK(func, EINVAL);
3547 instr->type = INSTR_EXTERN_FUNC;
3548 instr->ext_func.ext_func_id = func->id;
3557 instr_extern_obj_exec(struct rte_swx_pipeline *p)
3559 struct thread *t = &p->threads[p->thread_id];
3560 struct instruction *ip = t->ip;
3561 uint32_t obj_id = ip->ext_obj.ext_obj_id;
3562 uint32_t func_id = ip->ext_obj.func_id;
3563 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
3564 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
3566 TRACE("[Thread %2u] extern obj %u member func %u\n",
3571 /* Extern object member function execute. */
3572 uint32_t done = func(obj->obj, obj->mailbox);
3575 thread_ip_inc_cond(t, done);
3576 thread_yield_cond(p, done ^ 1);
3580 instr_extern_func_exec(struct rte_swx_pipeline *p)
3582 struct thread *t = &p->threads[p->thread_id];
3583 struct instruction *ip = t->ip;
3584 uint32_t ext_func_id = ip->ext_func.ext_func_id;
3585 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
3586 rte_swx_extern_func_t func = ext_func->func;
3588 TRACE("[Thread %2u] extern func %u\n",
3592 /* Extern function execute. */
3593 uint32_t done = func(ext_func->mailbox);
3596 thread_ip_inc_cond(t, done);
3597 thread_yield_cond(p, done ^ 1);
3604 instr_mov_translate(struct rte_swx_pipeline *p,
3605 struct action *action,
3608 struct instruction *instr,
3609 struct instruction_data *data __rte_unused)
3611 char *dst = tokens[1], *src = tokens[2];
3612 struct field *fdst, *fsrc;
3614 uint32_t dst_struct_id, src_struct_id;
3616 CHECK(n_tokens == 3, EINVAL);
3618 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3619 CHECK(fdst, EINVAL);
3621 /* MOV, MOV_MH, MOV_HM or MOV_HH. */
3622 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3624 instr->type = INSTR_MOV;
3625 if (dst[0] != 'h' && src[0] == 'h')
3626 instr->type = INSTR_MOV_MH;
3627 if (dst[0] == 'h' && src[0] != 'h')
3628 instr->type = INSTR_MOV_HM;
3629 if (dst[0] == 'h' && src[0] == 'h')
3630 instr->type = INSTR_MOV_HH;
3632 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3633 instr->mov.dst.n_bits = fdst->n_bits;
3634 instr->mov.dst.offset = fdst->offset / 8;
3635 instr->mov.src.struct_id = (uint8_t)src_struct_id;
3636 instr->mov.src.n_bits = fsrc->n_bits;
3637 instr->mov.src.offset = fsrc->offset / 8;
3642 src_val = strtoull(src, &src, 0);
3643 CHECK(!src[0], EINVAL);
3646 src_val = hton64(src_val) >> (64 - fdst->n_bits);
3648 instr->type = INSTR_MOV_I;
3649 instr->mov.dst.struct_id = (uint8_t)dst_struct_id;
3650 instr->mov.dst.n_bits = fdst->n_bits;
3651 instr->mov.dst.offset = fdst->offset / 8;
3652 instr->mov.src_val = src_val;
3657 instr_mov_exec(struct rte_swx_pipeline *p)
3659 struct thread *t = &p->threads[p->thread_id];
3660 struct instruction *ip = t->ip;
3662 TRACE("[Thread %2u] mov\n",
3672 instr_mov_mh_exec(struct rte_swx_pipeline *p)
3674 struct thread *t = &p->threads[p->thread_id];
3675 struct instruction *ip = t->ip;
3677 TRACE("[Thread %2u] mov (mh)\n",
3687 instr_mov_hm_exec(struct rte_swx_pipeline *p)
3689 struct thread *t = &p->threads[p->thread_id];
3690 struct instruction *ip = t->ip;
3692 TRACE("[Thread %2u] mov (hm)\n",
3702 instr_mov_hh_exec(struct rte_swx_pipeline *p)
3704 struct thread *t = &p->threads[p->thread_id];
3705 struct instruction *ip = t->ip;
3707 TRACE("[Thread %2u] mov (hh)\n",
3717 instr_mov_i_exec(struct rte_swx_pipeline *p)
3719 struct thread *t = &p->threads[p->thread_id];
3720 struct instruction *ip = t->ip;
3722 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n",
3736 instr_dma_translate(struct rte_swx_pipeline *p,
3737 struct action *action,
3740 struct instruction *instr,
3741 struct instruction_data *data __rte_unused)
3743 char *dst = tokens[1];
3744 char *src = tokens[2];
3748 CHECK(action, EINVAL);
3749 CHECK(n_tokens == 3, EINVAL);
3751 h = header_parse(p, dst);
3754 tf = action_field_parse(action, src);
3757 instr->type = INSTR_DMA_HT;
3758 instr->dma.dst.header_id[0] = h->id;
3759 instr->dma.dst.struct_id[0] = h->struct_id;
3760 instr->dma.n_bytes[0] = h->st->n_bits / 8;
3761 instr->dma.src.offset[0] = tf->offset / 8;
3767 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma);
3770 __instr_dma_ht_exec(struct rte_swx_pipeline *p, uint32_t n_dma)
3772 struct thread *t = &p->threads[p->thread_id];
3773 struct instruction *ip = t->ip;
3774 uint8_t *action_data = t->structs[0];
3775 uint64_t valid_headers = t->valid_headers;
3778 for (i = 0; i < n_dma; i++) {
3779 uint32_t header_id = ip->dma.dst.header_id[i];
3780 uint32_t struct_id = ip->dma.dst.struct_id[i];
3781 uint32_t offset = ip->dma.src.offset[i];
3782 uint32_t n_bytes = ip->dma.n_bytes[i];
3784 struct header_runtime *h = &t->headers[header_id];
3785 uint8_t *h_ptr0 = h->ptr0;
3786 uint8_t *h_ptr = t->structs[struct_id];
3788 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
3790 void *src = &action_data[offset];
3792 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
3795 memcpy(dst, src, n_bytes);
3796 t->structs[struct_id] = dst;
3797 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
3800 t->valid_headers = valid_headers;
3804 instr_dma_ht_exec(struct rte_swx_pipeline *p)
3806 __instr_dma_ht_exec(p, 1);
3813 instr_dma_ht2_exec(struct rte_swx_pipeline *p)
3815 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n",
3818 __instr_dma_ht_exec(p, 2);
3825 instr_dma_ht3_exec(struct rte_swx_pipeline *p)
3827 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n",
3830 __instr_dma_ht_exec(p, 3);
3837 instr_dma_ht4_exec(struct rte_swx_pipeline *p)
3839 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n",
3842 __instr_dma_ht_exec(p, 4);
3849 instr_dma_ht5_exec(struct rte_swx_pipeline *p)
3851 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n",
3854 __instr_dma_ht_exec(p, 5);
3861 instr_dma_ht6_exec(struct rte_swx_pipeline *p)
3863 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n",
3866 __instr_dma_ht_exec(p, 6);
3873 instr_dma_ht7_exec(struct rte_swx_pipeline *p)
3875 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n",
3878 __instr_dma_ht_exec(p, 7);
3885 instr_dma_ht8_exec(struct rte_swx_pipeline *p)
3887 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n",
3890 __instr_dma_ht_exec(p, 8);
3900 instr_alu_add_translate(struct rte_swx_pipeline *p,
3901 struct action *action,
3904 struct instruction *instr,
3905 struct instruction_data *data __rte_unused)
3907 char *dst = tokens[1], *src = tokens[2];
3908 struct field *fdst, *fsrc;
3910 uint32_t dst_struct_id, src_struct_id;
3912 CHECK(n_tokens == 3, EINVAL);
3914 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3915 CHECK(fdst, EINVAL);
3917 /* ADD, ADD_HM, ADD_MH, ADD_HH. */
3918 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3920 instr->type = INSTR_ALU_ADD;
3921 if (dst[0] == 'h' && src[0] != 'h')
3922 instr->type = INSTR_ALU_ADD_HM;
3923 if (dst[0] != 'h' && src[0] == 'h')
3924 instr->type = INSTR_ALU_ADD_MH;
3925 if (dst[0] == 'h' && src[0] == 'h')
3926 instr->type = INSTR_ALU_ADD_HH;
3928 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3929 instr->alu.dst.n_bits = fdst->n_bits;
3930 instr->alu.dst.offset = fdst->offset / 8;
3931 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3932 instr->alu.src.n_bits = fsrc->n_bits;
3933 instr->alu.src.offset = fsrc->offset / 8;
3937 /* ADD_MI, ADD_HI. */
3938 src_val = strtoull(src, &src, 0);
3939 CHECK(!src[0], EINVAL);
3941 instr->type = INSTR_ALU_ADD_MI;
3943 instr->type = INSTR_ALU_ADD_HI;
3945 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3946 instr->alu.dst.n_bits = fdst->n_bits;
3947 instr->alu.dst.offset = fdst->offset / 8;
3948 instr->alu.src_val = src_val;
3953 instr_alu_sub_translate(struct rte_swx_pipeline *p,
3954 struct action *action,
3957 struct instruction *instr,
3958 struct instruction_data *data __rte_unused)
3960 char *dst = tokens[1], *src = tokens[2];
3961 struct field *fdst, *fsrc;
3963 uint32_t dst_struct_id, src_struct_id;
3965 CHECK(n_tokens == 3, EINVAL);
3967 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
3968 CHECK(fdst, EINVAL);
3970 /* SUB, SUB_HM, SUB_MH, SUB_HH. */
3971 fsrc = struct_field_parse(p, action, src, &src_struct_id);
3973 instr->type = INSTR_ALU_SUB;
3974 if (dst[0] == 'h' && src[0] != 'h')
3975 instr->type = INSTR_ALU_SUB_HM;
3976 if (dst[0] != 'h' && src[0] == 'h')
3977 instr->type = INSTR_ALU_SUB_MH;
3978 if (dst[0] == 'h' && src[0] == 'h')
3979 instr->type = INSTR_ALU_SUB_HH;
3981 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3982 instr->alu.dst.n_bits = fdst->n_bits;
3983 instr->alu.dst.offset = fdst->offset / 8;
3984 instr->alu.src.struct_id = (uint8_t)src_struct_id;
3985 instr->alu.src.n_bits = fsrc->n_bits;
3986 instr->alu.src.offset = fsrc->offset / 8;
3990 /* SUB_MI, SUB_HI. */
3991 src_val = strtoull(src, &src, 0);
3992 CHECK(!src[0], EINVAL);
3994 instr->type = INSTR_ALU_SUB_MI;
3996 instr->type = INSTR_ALU_SUB_HI;
3998 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
3999 instr->alu.dst.n_bits = fdst->n_bits;
4000 instr->alu.dst.offset = fdst->offset / 8;
4001 instr->alu.src_val = src_val;
4006 instr_alu_ckadd_translate(struct rte_swx_pipeline *p,
4007 struct action *action __rte_unused,
4010 struct instruction *instr,
4011 struct instruction_data *data __rte_unused)
4013 char *dst = tokens[1], *src = tokens[2];
4014 struct header *hdst, *hsrc;
4015 struct field *fdst, *fsrc;
4017 CHECK(n_tokens == 3, EINVAL);
4019 fdst = header_field_parse(p, dst, &hdst);
4020 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4023 fsrc = header_field_parse(p, src, &hsrc);
4025 instr->type = INSTR_ALU_CKADD_FIELD;
4026 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4027 instr->alu.dst.n_bits = fdst->n_bits;
4028 instr->alu.dst.offset = fdst->offset / 8;
4029 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4030 instr->alu.src.n_bits = fsrc->n_bits;
4031 instr->alu.src.offset = fsrc->offset / 8;
4035 /* CKADD_STRUCT, CKADD_STRUCT20. */
4036 hsrc = header_parse(p, src);
4037 CHECK(hsrc, EINVAL);
4039 instr->type = INSTR_ALU_CKADD_STRUCT;
4040 if ((hsrc->st->n_bits / 8) == 20)
4041 instr->type = INSTR_ALU_CKADD_STRUCT20;
4043 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4044 instr->alu.dst.n_bits = fdst->n_bits;
4045 instr->alu.dst.offset = fdst->offset / 8;
4046 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4047 instr->alu.src.n_bits = hsrc->st->n_bits;
4048 instr->alu.src.offset = 0; /* Unused. */
4053 instr_alu_cksub_translate(struct rte_swx_pipeline *p,
4054 struct action *action __rte_unused,
4057 struct instruction *instr,
4058 struct instruction_data *data __rte_unused)
4060 char *dst = tokens[1], *src = tokens[2];
4061 struct header *hdst, *hsrc;
4062 struct field *fdst, *fsrc;
4064 CHECK(n_tokens == 3, EINVAL);
4066 fdst = header_field_parse(p, dst, &hdst);
4067 CHECK(fdst && (fdst->n_bits == 16), EINVAL);
4069 fsrc = header_field_parse(p, src, &hsrc);
4070 CHECK(fsrc, EINVAL);
4072 instr->type = INSTR_ALU_CKSUB_FIELD;
4073 instr->alu.dst.struct_id = (uint8_t)hdst->struct_id;
4074 instr->alu.dst.n_bits = fdst->n_bits;
4075 instr->alu.dst.offset = fdst->offset / 8;
4076 instr->alu.src.struct_id = (uint8_t)hsrc->struct_id;
4077 instr->alu.src.n_bits = fsrc->n_bits;
4078 instr->alu.src.offset = fsrc->offset / 8;
4083 instr_alu_shl_translate(struct rte_swx_pipeline *p,
4084 struct action *action,
4087 struct instruction *instr,
4088 struct instruction_data *data __rte_unused)
4090 char *dst = tokens[1], *src = tokens[2];
4091 struct field *fdst, *fsrc;
4093 uint32_t dst_struct_id, src_struct_id;
4095 CHECK(n_tokens == 3, EINVAL);
4097 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4098 CHECK(fdst, EINVAL);
4100 /* SHL, SHL_HM, SHL_MH, SHL_HH. */
4101 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4103 instr->type = INSTR_ALU_SHL;
4104 if (dst[0] == 'h' && src[0] != 'h')
4105 instr->type = INSTR_ALU_SHL_HM;
4106 if (dst[0] != 'h' && src[0] == 'h')
4107 instr->type = INSTR_ALU_SHL_MH;
4108 if (dst[0] == 'h' && src[0] == 'h')
4109 instr->type = INSTR_ALU_SHL_HH;
4111 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4112 instr->alu.dst.n_bits = fdst->n_bits;
4113 instr->alu.dst.offset = fdst->offset / 8;
4114 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4115 instr->alu.src.n_bits = fsrc->n_bits;
4116 instr->alu.src.offset = fsrc->offset / 8;
4120 /* SHL_MI, SHL_HI. */
4121 src_val = strtoull(src, &src, 0);
4122 CHECK(!src[0], EINVAL);
4124 instr->type = INSTR_ALU_SHL_MI;
4126 instr->type = INSTR_ALU_SHL_HI;
4128 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4129 instr->alu.dst.n_bits = fdst->n_bits;
4130 instr->alu.dst.offset = fdst->offset / 8;
4131 instr->alu.src_val = src_val;
4136 instr_alu_shr_translate(struct rte_swx_pipeline *p,
4137 struct action *action,
4140 struct instruction *instr,
4141 struct instruction_data *data __rte_unused)
4143 char *dst = tokens[1], *src = tokens[2];
4144 struct field *fdst, *fsrc;
4146 uint32_t dst_struct_id, src_struct_id;
4148 CHECK(n_tokens == 3, EINVAL);
4150 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4151 CHECK(fdst, EINVAL);
4153 /* SHR, SHR_HM, SHR_MH, SHR_HH. */
4154 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4156 instr->type = INSTR_ALU_SHR;
4157 if (dst[0] == 'h' && src[0] != 'h')
4158 instr->type = INSTR_ALU_SHR_HM;
4159 if (dst[0] != 'h' && src[0] == 'h')
4160 instr->type = INSTR_ALU_SHR_MH;
4161 if (dst[0] == 'h' && src[0] == 'h')
4162 instr->type = INSTR_ALU_SHR_HH;
4164 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4165 instr->alu.dst.n_bits = fdst->n_bits;
4166 instr->alu.dst.offset = fdst->offset / 8;
4167 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4168 instr->alu.src.n_bits = fsrc->n_bits;
4169 instr->alu.src.offset = fsrc->offset / 8;
4173 /* SHR_MI, SHR_HI. */
4174 src_val = strtoull(src, &src, 0);
4175 CHECK(!src[0], EINVAL);
4177 instr->type = INSTR_ALU_SHR_MI;
4179 instr->type = INSTR_ALU_SHR_HI;
4181 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4182 instr->alu.dst.n_bits = fdst->n_bits;
4183 instr->alu.dst.offset = fdst->offset / 8;
4184 instr->alu.src_val = src_val;
4189 instr_alu_and_translate(struct rte_swx_pipeline *p,
4190 struct action *action,
4193 struct instruction *instr,
4194 struct instruction_data *data __rte_unused)
4196 char *dst = tokens[1], *src = tokens[2];
4197 struct field *fdst, *fsrc;
4199 uint32_t dst_struct_id, src_struct_id;
4201 CHECK(n_tokens == 3, EINVAL);
4203 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4204 CHECK(fdst, EINVAL);
4206 /* AND, AND_MH, AND_HM, AND_HH. */
4207 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4209 instr->type = INSTR_ALU_AND;
4210 if (dst[0] != 'h' && src[0] == 'h')
4211 instr->type = INSTR_ALU_AND_MH;
4212 if (dst[0] == 'h' && src[0] != 'h')
4213 instr->type = INSTR_ALU_AND_HM;
4214 if (dst[0] == 'h' && src[0] == 'h')
4215 instr->type = INSTR_ALU_AND_HH;
4217 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4218 instr->alu.dst.n_bits = fdst->n_bits;
4219 instr->alu.dst.offset = fdst->offset / 8;
4220 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4221 instr->alu.src.n_bits = fsrc->n_bits;
4222 instr->alu.src.offset = fsrc->offset / 8;
4227 src_val = strtoull(src, &src, 0);
4228 CHECK(!src[0], EINVAL);
4231 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4233 instr->type = INSTR_ALU_AND_I;
4234 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4235 instr->alu.dst.n_bits = fdst->n_bits;
4236 instr->alu.dst.offset = fdst->offset / 8;
4237 instr->alu.src_val = src_val;
4242 instr_alu_or_translate(struct rte_swx_pipeline *p,
4243 struct action *action,
4246 struct instruction *instr,
4247 struct instruction_data *data __rte_unused)
4249 char *dst = tokens[1], *src = tokens[2];
4250 struct field *fdst, *fsrc;
4252 uint32_t dst_struct_id, src_struct_id;
4254 CHECK(n_tokens == 3, EINVAL);
4256 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4257 CHECK(fdst, EINVAL);
4259 /* OR, OR_MH, OR_HM, OR_HH. */
4260 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4262 instr->type = INSTR_ALU_OR;
4263 if (dst[0] != 'h' && src[0] == 'h')
4264 instr->type = INSTR_ALU_OR_MH;
4265 if (dst[0] == 'h' && src[0] != 'h')
4266 instr->type = INSTR_ALU_OR_HM;
4267 if (dst[0] == 'h' && src[0] == 'h')
4268 instr->type = INSTR_ALU_OR_HH;
4270 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4271 instr->alu.dst.n_bits = fdst->n_bits;
4272 instr->alu.dst.offset = fdst->offset / 8;
4273 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4274 instr->alu.src.n_bits = fsrc->n_bits;
4275 instr->alu.src.offset = fsrc->offset / 8;
4280 src_val = strtoull(src, &src, 0);
4281 CHECK(!src[0], EINVAL);
4284 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4286 instr->type = INSTR_ALU_OR_I;
4287 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4288 instr->alu.dst.n_bits = fdst->n_bits;
4289 instr->alu.dst.offset = fdst->offset / 8;
4290 instr->alu.src_val = src_val;
4295 instr_alu_xor_translate(struct rte_swx_pipeline *p,
4296 struct action *action,
4299 struct instruction *instr,
4300 struct instruction_data *data __rte_unused)
4302 char *dst = tokens[1], *src = tokens[2];
4303 struct field *fdst, *fsrc;
4305 uint32_t dst_struct_id, src_struct_id;
4307 CHECK(n_tokens == 3, EINVAL);
4309 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
4310 CHECK(fdst, EINVAL);
4312 /* XOR, XOR_MH, XOR_HM, XOR_HH. */
4313 fsrc = struct_field_parse(p, action, src, &src_struct_id);
4315 instr->type = INSTR_ALU_XOR;
4316 if (dst[0] != 'h' && src[0] == 'h')
4317 instr->type = INSTR_ALU_XOR_MH;
4318 if (dst[0] == 'h' && src[0] != 'h')
4319 instr->type = INSTR_ALU_XOR_HM;
4320 if (dst[0] == 'h' && src[0] == 'h')
4321 instr->type = INSTR_ALU_XOR_HH;
4323 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4324 instr->alu.dst.n_bits = fdst->n_bits;
4325 instr->alu.dst.offset = fdst->offset / 8;
4326 instr->alu.src.struct_id = (uint8_t)src_struct_id;
4327 instr->alu.src.n_bits = fsrc->n_bits;
4328 instr->alu.src.offset = fsrc->offset / 8;
4333 src_val = strtoull(src, &src, 0);
4334 CHECK(!src[0], EINVAL);
4337 src_val = hton64(src_val) >> (64 - fdst->n_bits);
4339 instr->type = INSTR_ALU_XOR_I;
4340 instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
4341 instr->alu.dst.n_bits = fdst->n_bits;
4342 instr->alu.dst.offset = fdst->offset / 8;
4343 instr->alu.src_val = src_val;
4348 instr_alu_add_exec(struct rte_swx_pipeline *p)
4350 struct thread *t = &p->threads[p->thread_id];
4351 struct instruction *ip = t->ip;
4353 TRACE("[Thread %2u] add\n", p->thread_id);
4363 instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
4365 struct thread *t = &p->threads[p->thread_id];
4366 struct instruction *ip = t->ip;
4368 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
4378 instr_alu_add_hm_exec(struct rte_swx_pipeline *p)
4380 struct thread *t = &p->threads[p->thread_id];
4381 struct instruction *ip = t->ip;
4383 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
4393 instr_alu_add_hh_exec(struct rte_swx_pipeline *p)
4395 struct thread *t = &p->threads[p->thread_id];
4396 struct instruction *ip = t->ip;
4398 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
4408 instr_alu_add_mi_exec(struct rte_swx_pipeline *p)
4410 struct thread *t = &p->threads[p->thread_id];
4411 struct instruction *ip = t->ip;
4413 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
4423 instr_alu_add_hi_exec(struct rte_swx_pipeline *p)
4425 struct thread *t = &p->threads[p->thread_id];
4426 struct instruction *ip = t->ip;
4428 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
4438 instr_alu_sub_exec(struct rte_swx_pipeline *p)
4440 struct thread *t = &p->threads[p->thread_id];
4441 struct instruction *ip = t->ip;
4443 TRACE("[Thread %2u] sub\n", p->thread_id);
4453 instr_alu_sub_mh_exec(struct rte_swx_pipeline *p)
4455 struct thread *t = &p->threads[p->thread_id];
4456 struct instruction *ip = t->ip;
4458 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
4468 instr_alu_sub_hm_exec(struct rte_swx_pipeline *p)
4470 struct thread *t = &p->threads[p->thread_id];
4471 struct instruction *ip = t->ip;
4473 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
4483 instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
4485 struct thread *t = &p->threads[p->thread_id];
4486 struct instruction *ip = t->ip;
4488 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
4498 instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
4500 struct thread *t = &p->threads[p->thread_id];
4501 struct instruction *ip = t->ip;
4503 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
4513 instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
4515 struct thread *t = &p->threads[p->thread_id];
4516 struct instruction *ip = t->ip;
4518 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
4528 instr_alu_shl_exec(struct rte_swx_pipeline *p)
4530 struct thread *t = &p->threads[p->thread_id];
4531 struct instruction *ip = t->ip;
4533 TRACE("[Thread %2u] shl\n", p->thread_id);
4543 instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
4545 struct thread *t = &p->threads[p->thread_id];
4546 struct instruction *ip = t->ip;
4548 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
4558 instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
4560 struct thread *t = &p->threads[p->thread_id];
4561 struct instruction *ip = t->ip;
4563 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
4573 instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
4575 struct thread *t = &p->threads[p->thread_id];
4576 struct instruction *ip = t->ip;
4578 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
4588 instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
4590 struct thread *t = &p->threads[p->thread_id];
4591 struct instruction *ip = t->ip;
4593 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
4603 instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
4605 struct thread *t = &p->threads[p->thread_id];
4606 struct instruction *ip = t->ip;
4608 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
4618 instr_alu_shr_exec(struct rte_swx_pipeline *p)
4620 struct thread *t = &p->threads[p->thread_id];
4621 struct instruction *ip = t->ip;
4623 TRACE("[Thread %2u] shr\n", p->thread_id);
4633 instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
4635 struct thread *t = &p->threads[p->thread_id];
4636 struct instruction *ip = t->ip;
4638 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
4648 instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
4650 struct thread *t = &p->threads[p->thread_id];
4651 struct instruction *ip = t->ip;
4653 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
4663 instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
4665 struct thread *t = &p->threads[p->thread_id];
4666 struct instruction *ip = t->ip;
4668 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
4678 instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
4680 struct thread *t = &p->threads[p->thread_id];
4681 struct instruction *ip = t->ip;
4683 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
4693 instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
4695 struct thread *t = &p->threads[p->thread_id];
4696 struct instruction *ip = t->ip;
4698 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
4708 instr_alu_and_exec(struct rte_swx_pipeline *p)
4710 struct thread *t = &p->threads[p->thread_id];
4711 struct instruction *ip = t->ip;
4713 TRACE("[Thread %2u] and\n", p->thread_id);
4723 instr_alu_and_mh_exec(struct rte_swx_pipeline *p)
4725 struct thread *t = &p->threads[p->thread_id];
4726 struct instruction *ip = t->ip;
4728 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
4738 instr_alu_and_hm_exec(struct rte_swx_pipeline *p)
4740 struct thread *t = &p->threads[p->thread_id];
4741 struct instruction *ip = t->ip;
4743 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
4746 ALU_HM_FAST(t, ip, &);
4753 instr_alu_and_hh_exec(struct rte_swx_pipeline *p)
4755 struct thread *t = &p->threads[p->thread_id];
4756 struct instruction *ip = t->ip;
4758 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
4761 ALU_HH_FAST(t, ip, &);
4768 instr_alu_and_i_exec(struct rte_swx_pipeline *p)
4770 struct thread *t = &p->threads[p->thread_id];
4771 struct instruction *ip = t->ip;
4773 TRACE("[Thread %2u] and (i)\n", p->thread_id);
4783 instr_alu_or_exec(struct rte_swx_pipeline *p)
4785 struct thread *t = &p->threads[p->thread_id];
4786 struct instruction *ip = t->ip;
4788 TRACE("[Thread %2u] or\n", p->thread_id);
4798 instr_alu_or_mh_exec(struct rte_swx_pipeline *p)
4800 struct thread *t = &p->threads[p->thread_id];
4801 struct instruction *ip = t->ip;
4803 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
4813 instr_alu_or_hm_exec(struct rte_swx_pipeline *p)
4815 struct thread *t = &p->threads[p->thread_id];
4816 struct instruction *ip = t->ip;
4818 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
4821 ALU_HM_FAST(t, ip, |);
4828 instr_alu_or_hh_exec(struct rte_swx_pipeline *p)
4830 struct thread *t = &p->threads[p->thread_id];
4831 struct instruction *ip = t->ip;
4833 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
4836 ALU_HH_FAST(t, ip, |);
4843 instr_alu_or_i_exec(struct rte_swx_pipeline *p)
4845 struct thread *t = &p->threads[p->thread_id];
4846 struct instruction *ip = t->ip;
4848 TRACE("[Thread %2u] or (i)\n", p->thread_id);
4858 instr_alu_xor_exec(struct rte_swx_pipeline *p)
4860 struct thread *t = &p->threads[p->thread_id];
4861 struct instruction *ip = t->ip;
4863 TRACE("[Thread %2u] xor\n", p->thread_id);
4873 instr_alu_xor_mh_exec(struct rte_swx_pipeline *p)
4875 struct thread *t = &p->threads[p->thread_id];
4876 struct instruction *ip = t->ip;
4878 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
4888 instr_alu_xor_hm_exec(struct rte_swx_pipeline *p)
4890 struct thread *t = &p->threads[p->thread_id];
4891 struct instruction *ip = t->ip;
4893 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
4896 ALU_HM_FAST(t, ip, ^);
4903 instr_alu_xor_hh_exec(struct rte_swx_pipeline *p)
4905 struct thread *t = &p->threads[p->thread_id];
4906 struct instruction *ip = t->ip;
4908 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
4911 ALU_HH_FAST(t, ip, ^);
4918 instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
4920 struct thread *t = &p->threads[p->thread_id];
4921 struct instruction *ip = t->ip;
4923 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
4933 instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
4935 struct thread *t = &p->threads[p->thread_id];
4936 struct instruction *ip = t->ip;
4937 uint8_t *dst_struct, *src_struct;
4938 uint16_t *dst16_ptr, dst;
4939 uint64_t *src64_ptr, src64, src64_mask, src;
4942 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
4945 dst_struct = t->structs[ip->alu.dst.struct_id];
4946 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
4949 src_struct = t->structs[ip->alu.src.struct_id];
4950 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
4952 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
4953 src = src64 & src64_mask;
4958 /* The first input (r) is a 16-bit number. The second and the third
4959 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
4960 * three numbers (output r) is a 34-bit number.
4962 r += (src >> 32) + (src & 0xFFFFFFFF);
4964 /* The first input is a 16-bit number. The second input is an 18-bit
4965 * number. In the worst case scenario, the sum of the two numbers is a
4968 r = (r & 0xFFFF) + (r >> 16);
4970 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
4971 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
4973 r = (r & 0xFFFF) + (r >> 16);
4975 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
4976 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
4977 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
4978 * therefore the output r is always a 16-bit number.
4980 r = (r & 0xFFFF) + (r >> 16);
4985 *dst16_ptr = (uint16_t)r;
4992 instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
4994 struct thread *t = &p->threads[p->thread_id];
4995 struct instruction *ip = t->ip;
4996 uint8_t *dst_struct, *src_struct;
4997 uint16_t *dst16_ptr, dst;
4998 uint64_t *src64_ptr, src64, src64_mask, src;
5001 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
5004 dst_struct = t->structs[ip->alu.dst.struct_id];
5005 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5008 src_struct = t->structs[ip->alu.src.struct_id];
5009 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
5011 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
5012 src = src64 & src64_mask;
5017 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
5018 * the following sequence of operations in 2's complement arithmetic:
5019 * a '- b = (a - b) % 0xFFFF.
5021 * In order to prevent an underflow for the below subtraction, in which
5022 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
5023 * minuend), we first add a multiple of the 0xFFFF modulus to the
5024 * minuend. The number we add to the minuend needs to be a 34-bit number
5025 * or higher, so for readability reasons we picked the 36-bit multiple.
5026 * We are effectively turning the 16-bit minuend into a 36-bit number:
5027 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
5029 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
5031 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
5032 * result (the output r) is a 36-bit number.
5034 r -= (src >> 32) + (src & 0xFFFFFFFF);
5036 /* The first input is a 16-bit number. The second input is a 20-bit
5037 * number. Their sum is a 21-bit number.
5039 r = (r & 0xFFFF) + (r >> 16);
5041 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5042 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
5044 r = (r & 0xFFFF) + (r >> 16);
5046 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5047 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5048 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5049 * generated, therefore the output r is always a 16-bit number.
5051 r = (r & 0xFFFF) + (r >> 16);
5056 *dst16_ptr = (uint16_t)r;
5063 instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
5065 struct thread *t = &p->threads[p->thread_id];
5066 struct instruction *ip = t->ip;
5067 uint8_t *dst_struct, *src_struct;
5068 uint16_t *dst16_ptr;
5069 uint32_t *src32_ptr;
5072 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
5075 dst_struct = t->structs[ip->alu.dst.struct_id];
5076 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5078 src_struct = t->structs[ip->alu.src.struct_id];
5079 src32_ptr = (uint32_t *)&src_struct[0];
5081 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
5082 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
5083 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
5084 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
5085 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
5087 /* The first input is a 16-bit number. The second input is a 19-bit
5088 * number. Their sum is a 20-bit number.
5090 r0 = (r0 & 0xFFFF) + (r0 >> 16);
5092 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5093 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
5095 r0 = (r0 & 0xFFFF) + (r0 >> 16);
5097 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5098 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5099 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
5100 * generated, therefore the output r is always a 16-bit number.
5102 r0 = (r0 & 0xFFFF) + (r0 >> 16);
5105 r0 = r0 ? r0 : 0xFFFF;
5107 *dst16_ptr = (uint16_t)r0;
5114 instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
5116 struct thread *t = &p->threads[p->thread_id];
5117 struct instruction *ip = t->ip;
5118 uint8_t *dst_struct, *src_struct;
5119 uint16_t *dst16_ptr;
5120 uint32_t *src32_ptr;
5124 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
5127 dst_struct = t->structs[ip->alu.dst.struct_id];
5128 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
5130 src_struct = t->structs[ip->alu.src.struct_id];
5131 src32_ptr = (uint32_t *)&src_struct[0];
5133 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
5134 * Therefore, in the worst case scenario, a 35-bit number is added to a
5135 * 16-bit number (the input r), so the output r is 36-bit number.
5137 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
5140 /* The first input is a 16-bit number. The second input is a 20-bit
5141 * number. Their sum is a 21-bit number.
5143 r = (r & 0xFFFF) + (r >> 16);
5145 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
5146 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
5148 r = (r & 0xFFFF) + (r >> 16);
5150 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
5151 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
5152 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
5153 * generated, therefore the output r is always a 16-bit number.
5155 r = (r & 0xFFFF) + (r >> 16);
5160 *dst16_ptr = (uint16_t)r;
5169 static struct regarray *
5170 regarray_find(struct rte_swx_pipeline *p, const char *name);
5173 instr_regprefetch_translate(struct rte_swx_pipeline *p,
5174 struct action *action,
5177 struct instruction *instr,
5178 struct instruction_data *data __rte_unused)
5180 char *regarray = tokens[1], *idx = tokens[2];
5183 uint32_t idx_struct_id, idx_val;
5185 CHECK(n_tokens == 3, EINVAL);
5187 r = regarray_find(p, regarray);
5190 /* REGPREFETCH_RH, REGPREFETCH_RM. */
5191 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5193 instr->type = INSTR_REGPREFETCH_RM;
5195 instr->type = INSTR_REGPREFETCH_RH;
5197 instr->regarray.regarray_id = r->id;
5198 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5199 instr->regarray.idx.n_bits = fidx->n_bits;
5200 instr->regarray.idx.offset = fidx->offset / 8;
5201 instr->regarray.dstsrc_val = 0; /* Unused. */
5205 /* REGPREFETCH_RI. */
5206 idx_val = strtoul(idx, &idx, 0);
5207 CHECK(!idx[0], EINVAL);
5209 instr->type = INSTR_REGPREFETCH_RI;
5210 instr->regarray.regarray_id = r->id;
5211 instr->regarray.idx_val = idx_val;
5212 instr->regarray.dstsrc_val = 0; /* Unused. */
5217 instr_regrd_translate(struct rte_swx_pipeline *p,
5218 struct action *action,
5221 struct instruction *instr,
5222 struct instruction_data *data __rte_unused)
5224 char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
5226 struct field *fdst, *fidx;
5227 uint32_t dst_struct_id, idx_struct_id, idx_val;
5229 CHECK(n_tokens == 4, EINVAL);
5231 r = regarray_find(p, regarray);
5234 fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
5235 CHECK(fdst, EINVAL);
5237 /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
5238 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5240 instr->type = INSTR_REGRD_MRM;
5241 if (dst[0] == 'h' && idx[0] != 'h')
5242 instr->type = INSTR_REGRD_HRM;
5243 if (dst[0] != 'h' && idx[0] == 'h')
5244 instr->type = INSTR_REGRD_MRH;
5245 if (dst[0] == 'h' && idx[0] == 'h')
5246 instr->type = INSTR_REGRD_HRH;
5248 instr->regarray.regarray_id = r->id;
5249 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5250 instr->regarray.idx.n_bits = fidx->n_bits;
5251 instr->regarray.idx.offset = fidx->offset / 8;
5252 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5253 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5254 instr->regarray.dstsrc.offset = fdst->offset / 8;
5258 /* REGRD_MRI, REGRD_HRI. */
5259 idx_val = strtoul(idx, &idx, 0);
5260 CHECK(!idx[0], EINVAL);
5262 instr->type = INSTR_REGRD_MRI;
5264 instr->type = INSTR_REGRD_HRI;
5266 instr->regarray.regarray_id = r->id;
5267 instr->regarray.idx_val = idx_val;
5268 instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
5269 instr->regarray.dstsrc.n_bits = fdst->n_bits;
5270 instr->regarray.dstsrc.offset = fdst->offset / 8;
5275 instr_regwr_translate(struct rte_swx_pipeline *p,
5276 struct action *action,
5279 struct instruction *instr,
5280 struct instruction_data *data __rte_unused)
5282 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5284 struct field *fidx, *fsrc;
5286 uint32_t idx_struct_id, idx_val, src_struct_id;
5288 CHECK(n_tokens == 4, EINVAL);
5290 r = regarray_find(p, regarray);
5293 /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
5294 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5295 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5297 instr->type = INSTR_REGWR_RMM;
5298 if (idx[0] == 'h' && src[0] != 'h')
5299 instr->type = INSTR_REGWR_RHM;
5300 if (idx[0] != 'h' && src[0] == 'h')
5301 instr->type = INSTR_REGWR_RMH;
5302 if (idx[0] == 'h' && src[0] == 'h')
5303 instr->type = INSTR_REGWR_RHH;
5305 instr->regarray.regarray_id = r->id;
5306 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5307 instr->regarray.idx.n_bits = fidx->n_bits;
5308 instr->regarray.idx.offset = fidx->offset / 8;
5309 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5310 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5311 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5315 /* REGWR_RHI, REGWR_RMI. */
5316 if (fidx && !fsrc) {
5317 src_val = strtoull(src, &src, 0);
5318 CHECK(!src[0], EINVAL);
5320 instr->type = INSTR_REGWR_RMI;
5322 instr->type = INSTR_REGWR_RHI;
5324 instr->regarray.regarray_id = r->id;
5325 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5326 instr->regarray.idx.n_bits = fidx->n_bits;
5327 instr->regarray.idx.offset = fidx->offset / 8;
5328 instr->regarray.dstsrc_val = src_val;
5332 /* REGWR_RIH, REGWR_RIM. */
5333 if (!fidx && fsrc) {
5334 idx_val = strtoul(idx, &idx, 0);
5335 CHECK(!idx[0], EINVAL);
5337 instr->type = INSTR_REGWR_RIM;
5339 instr->type = INSTR_REGWR_RIH;
5341 instr->regarray.regarray_id = r->id;
5342 instr->regarray.idx_val = idx_val;
5343 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5344 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5345 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5350 src_val = strtoull(src, &src, 0);
5351 CHECK(!src[0], EINVAL);
5353 idx_val = strtoul(idx, &idx, 0);
5354 CHECK(!idx[0], EINVAL);
5356 instr->type = INSTR_REGWR_RII;
5357 instr->regarray.idx_val = idx_val;
5358 instr->regarray.dstsrc_val = src_val;
5364 instr_regadd_translate(struct rte_swx_pipeline *p,
5365 struct action *action,
5368 struct instruction *instr,
5369 struct instruction_data *data __rte_unused)
5371 char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
5373 struct field *fidx, *fsrc;
5375 uint32_t idx_struct_id, idx_val, src_struct_id;
5377 CHECK(n_tokens == 4, EINVAL);
5379 r = regarray_find(p, regarray);
5382 /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
5383 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
5384 fsrc = struct_field_parse(p, action, src, &src_struct_id);
5386 instr->type = INSTR_REGADD_RMM;
5387 if (idx[0] == 'h' && src[0] != 'h')
5388 instr->type = INSTR_REGADD_RHM;
5389 if (idx[0] != 'h' && src[0] == 'h')
5390 instr->type = INSTR_REGADD_RMH;
5391 if (idx[0] == 'h' && src[0] == 'h')
5392 instr->type = INSTR_REGADD_RHH;
5394 instr->regarray.regarray_id = r->id;
5395 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5396 instr->regarray.idx.n_bits = fidx->n_bits;
5397 instr->regarray.idx.offset = fidx->offset / 8;
5398 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5399 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5400 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5404 /* REGADD_RHI, REGADD_RMI. */
5405 if (fidx && !fsrc) {
5406 src_val = strtoull(src, &src, 0);
5407 CHECK(!src[0], EINVAL);
5409 instr->type = INSTR_REGADD_RMI;
5411 instr->type = INSTR_REGADD_RHI;
5413 instr->regarray.regarray_id = r->id;
5414 instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
5415 instr->regarray.idx.n_bits = fidx->n_bits;
5416 instr->regarray.idx.offset = fidx->offset / 8;
5417 instr->regarray.dstsrc_val = src_val;
5421 /* REGADD_RIH, REGADD_RIM. */
5422 if (!fidx && fsrc) {
5423 idx_val = strtoul(idx, &idx, 0);
5424 CHECK(!idx[0], EINVAL);
5426 instr->type = INSTR_REGADD_RIM;
5428 instr->type = INSTR_REGADD_RIH;
5430 instr->regarray.regarray_id = r->id;
5431 instr->regarray.idx_val = idx_val;
5432 instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
5433 instr->regarray.dstsrc.n_bits = fsrc->n_bits;
5434 instr->regarray.dstsrc.offset = fsrc->offset / 8;
5439 src_val = strtoull(src, &src, 0);
5440 CHECK(!src[0], EINVAL);
5442 idx_val = strtoul(idx, &idx, 0);
5443 CHECK(!idx[0], EINVAL);
5445 instr->type = INSTR_REGADD_RII;
5446 instr->regarray.idx_val = idx_val;
5447 instr->regarray.dstsrc_val = src_val;
5451 static inline uint64_t *
5452 instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
5454 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5458 static inline uint64_t
5459 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5461 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5463 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5464 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5465 uint64_t idx64 = *idx64_ptr;
5466 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
5467 uint64_t idx = idx64 & idx64_mask & r->size_mask;
5472 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5474 static inline uint64_t
5475 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
5477 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5479 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
5480 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
5481 uint64_t idx64 = *idx64_ptr;
5482 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
5489 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
5493 static inline uint64_t
5494 instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
5496 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
5498 uint64_t idx = ip->regarray.idx_val & r->size_mask;
5503 static inline uint64_t
5504 instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
5506 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5507 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5508 uint64_t src64 = *src64_ptr;
5509 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5510 uint64_t src = src64 & src64_mask;
5515 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5517 static inline uint64_t
5518 instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
5520 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
5521 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
5522 uint64_t src64 = *src64_ptr;
5523 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
5530 #define instr_regarray_src_nbo instr_regarray_src_hbo
5535 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5537 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5538 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5539 uint64_t dst64 = *dst64_ptr;
5540 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5542 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5546 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
5549 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
5551 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
5552 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
5553 uint64_t dst64 = *dst64_ptr;
5554 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
5556 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
5557 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
5562 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
5567 instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
5569 struct thread *t = &p->threads[p->thread_id];
5570 struct instruction *ip = t->ip;
5571 uint64_t *regarray, idx;
5573 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
5576 regarray = instr_regarray_regarray(p, ip);
5577 idx = instr_regarray_idx_nbo(p, t, ip);
5578 rte_prefetch0(®array[idx]);
5585 instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
5587 struct thread *t = &p->threads[p->thread_id];
5588 struct instruction *ip = t->ip;
5589 uint64_t *regarray, idx;
5591 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
5594 regarray = instr_regarray_regarray(p, ip);
5595 idx = instr_regarray_idx_hbo(p, t, ip);
5596 rte_prefetch0(®array[idx]);
5603 instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
5605 struct thread *t = &p->threads[p->thread_id];
5606 struct instruction *ip = t->ip;
5607 uint64_t *regarray, idx;
5609 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
5612 regarray = instr_regarray_regarray(p, ip);
5613 idx = instr_regarray_idx_imm(p, ip);
5614 rte_prefetch0(®array[idx]);
5621 instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
5623 struct thread *t = &p->threads[p->thread_id];
5624 struct instruction *ip = t->ip;
5625 uint64_t *regarray, idx;
5627 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
5630 regarray = instr_regarray_regarray(p, ip);
5631 idx = instr_regarray_idx_nbo(p, t, ip);
5632 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5639 instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
5641 struct thread *t = &p->threads[p->thread_id];
5642 struct instruction *ip = t->ip;
5643 uint64_t *regarray, idx;
5645 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
5648 regarray = instr_regarray_regarray(p, ip);
5649 idx = instr_regarray_idx_hbo(p, t, ip);
5650 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5657 instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
5659 struct thread *t = &p->threads[p->thread_id];
5660 struct instruction *ip = t->ip;
5661 uint64_t *regarray, idx;
5663 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
5666 regarray = instr_regarray_regarray(p, ip);
5667 idx = instr_regarray_idx_nbo(p, t, ip);
5668 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5675 instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
5677 struct thread *t = &p->threads[p->thread_id];
5678 struct instruction *ip = t->ip;
5679 uint64_t *regarray, idx;
5682 regarray = instr_regarray_regarray(p, ip);
5683 idx = instr_regarray_idx_hbo(p, t, ip);
5684 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5691 instr_regrd_hri_exec(struct rte_swx_pipeline *p)
5693 struct thread *t = &p->threads[p->thread_id];
5694 struct instruction *ip = t->ip;
5695 uint64_t *regarray, idx;
5697 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
5700 regarray = instr_regarray_regarray(p, ip);
5701 idx = instr_regarray_idx_imm(p, ip);
5702 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
5709 instr_regrd_mri_exec(struct rte_swx_pipeline *p)
5711 struct thread *t = &p->threads[p->thread_id];
5712 struct instruction *ip = t->ip;
5713 uint64_t *regarray, idx;
5715 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
5718 regarray = instr_regarray_regarray(p, ip);
5719 idx = instr_regarray_idx_imm(p, ip);
5720 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
5727 instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
5729 struct thread *t = &p->threads[p->thread_id];
5730 struct instruction *ip = t->ip;
5731 uint64_t *regarray, idx, src;
5733 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
5736 regarray = instr_regarray_regarray(p, ip);
5737 idx = instr_regarray_idx_nbo(p, t, ip);
5738 src = instr_regarray_src_nbo(t, ip);
5739 regarray[idx] = src;
5746 instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
5748 struct thread *t = &p->threads[p->thread_id];
5749 struct instruction *ip = t->ip;
5750 uint64_t *regarray, idx, src;
5752 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
5755 regarray = instr_regarray_regarray(p, ip);
5756 idx = instr_regarray_idx_nbo(p, t, ip);
5757 src = instr_regarray_src_hbo(t, ip);
5758 regarray[idx] = src;
5765 instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
5767 struct thread *t = &p->threads[p->thread_id];
5768 struct instruction *ip = t->ip;
5769 uint64_t *regarray, idx, src;
5771 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
5774 regarray = instr_regarray_regarray(p, ip);
5775 idx = instr_regarray_idx_hbo(p, t, ip);
5776 src = instr_regarray_src_nbo(t, ip);
5777 regarray[idx] = src;
5784 instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
5786 struct thread *t = &p->threads[p->thread_id];
5787 struct instruction *ip = t->ip;
5788 uint64_t *regarray, idx, src;
5790 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
5793 regarray = instr_regarray_regarray(p, ip);
5794 idx = instr_regarray_idx_hbo(p, t, ip);
5795 src = instr_regarray_src_hbo(t, ip);
5796 regarray[idx] = src;
5803 instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
5805 struct thread *t = &p->threads[p->thread_id];
5806 struct instruction *ip = t->ip;
5807 uint64_t *regarray, idx, src;
5809 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
5812 regarray = instr_regarray_regarray(p, ip);
5813 idx = instr_regarray_idx_nbo(p, t, ip);
5814 src = ip->regarray.dstsrc_val;
5815 regarray[idx] = src;
5822 instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
5824 struct thread *t = &p->threads[p->thread_id];
5825 struct instruction *ip = t->ip;
5826 uint64_t *regarray, idx, src;
5828 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
5831 regarray = instr_regarray_regarray(p, ip);
5832 idx = instr_regarray_idx_hbo(p, t, ip);
5833 src = ip->regarray.dstsrc_val;
5834 regarray[idx] = src;
5841 instr_regwr_rih_exec(struct rte_swx_pipeline *p)
5843 struct thread *t = &p->threads[p->thread_id];
5844 struct instruction *ip = t->ip;
5845 uint64_t *regarray, idx, src;
5847 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
5850 regarray = instr_regarray_regarray(p, ip);
5851 idx = instr_regarray_idx_imm(p, ip);
5852 src = instr_regarray_src_nbo(t, ip);
5853 regarray[idx] = src;
5860 instr_regwr_rim_exec(struct rte_swx_pipeline *p)
5862 struct thread *t = &p->threads[p->thread_id];
5863 struct instruction *ip = t->ip;
5864 uint64_t *regarray, idx, src;
5866 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
5869 regarray = instr_regarray_regarray(p, ip);
5870 idx = instr_regarray_idx_imm(p, ip);
5871 src = instr_regarray_src_hbo(t, ip);
5872 regarray[idx] = src;
5879 instr_regwr_rii_exec(struct rte_swx_pipeline *p)
5881 struct thread *t = &p->threads[p->thread_id];
5882 struct instruction *ip = t->ip;
5883 uint64_t *regarray, idx, src;
5885 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
5888 regarray = instr_regarray_regarray(p, ip);
5889 idx = instr_regarray_idx_imm(p, ip);
5890 src = ip->regarray.dstsrc_val;
5891 regarray[idx] = src;
5898 instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
5900 struct thread *t = &p->threads[p->thread_id];
5901 struct instruction *ip = t->ip;
5902 uint64_t *regarray, idx, src;
5904 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
5907 regarray = instr_regarray_regarray(p, ip);
5908 idx = instr_regarray_idx_nbo(p, t, ip);
5909 src = instr_regarray_src_nbo(t, ip);
5910 regarray[idx] += src;
5917 instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
5919 struct thread *t = &p->threads[p->thread_id];
5920 struct instruction *ip = t->ip;
5921 uint64_t *regarray, idx, src;
5923 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
5926 regarray = instr_regarray_regarray(p, ip);
5927 idx = instr_regarray_idx_nbo(p, t, ip);
5928 src = instr_regarray_src_hbo(t, ip);
5929 regarray[idx] += src;
5936 instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
5938 struct thread *t = &p->threads[p->thread_id];
5939 struct instruction *ip = t->ip;
5940 uint64_t *regarray, idx, src;
5942 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
5945 regarray = instr_regarray_regarray(p, ip);
5946 idx = instr_regarray_idx_hbo(p, t, ip);
5947 src = instr_regarray_src_nbo(t, ip);
5948 regarray[idx] += src;
5955 instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
5957 struct thread *t = &p->threads[p->thread_id];
5958 struct instruction *ip = t->ip;
5959 uint64_t *regarray, idx, src;
5961 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
5964 regarray = instr_regarray_regarray(p, ip);
5965 idx = instr_regarray_idx_hbo(p, t, ip);
5966 src = instr_regarray_src_hbo(t, ip);
5967 regarray[idx] += src;
5974 instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
5976 struct thread *t = &p->threads[p->thread_id];
5977 struct instruction *ip = t->ip;
5978 uint64_t *regarray, idx, src;
5980 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
5983 regarray = instr_regarray_regarray(p, ip);
5984 idx = instr_regarray_idx_nbo(p, t, ip);
5985 src = ip->regarray.dstsrc_val;
5986 regarray[idx] += src;
5993 instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
5995 struct thread *t = &p->threads[p->thread_id];
5996 struct instruction *ip = t->ip;
5997 uint64_t *regarray, idx, src;
5999 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
6002 regarray = instr_regarray_regarray(p, ip);
6003 idx = instr_regarray_idx_hbo(p, t, ip);
6004 src = ip->regarray.dstsrc_val;
6005 regarray[idx] += src;
6012 instr_regadd_rih_exec(struct rte_swx_pipeline *p)
6014 struct thread *t = &p->threads[p->thread_id];
6015 struct instruction *ip = t->ip;
6016 uint64_t *regarray, idx, src;
6018 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
6021 regarray = instr_regarray_regarray(p, ip);
6022 idx = instr_regarray_idx_imm(p, ip);
6023 src = instr_regarray_src_nbo(t, ip);
6024 regarray[idx] += src;
6031 instr_regadd_rim_exec(struct rte_swx_pipeline *p)
6033 struct thread *t = &p->threads[p->thread_id];
6034 struct instruction *ip = t->ip;
6035 uint64_t *regarray, idx, src;
6037 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
6040 regarray = instr_regarray_regarray(p, ip);
6041 idx = instr_regarray_idx_imm(p, ip);
6042 src = instr_regarray_src_hbo(t, ip);
6043 regarray[idx] += src;
6050 instr_regadd_rii_exec(struct rte_swx_pipeline *p)
6052 struct thread *t = &p->threads[p->thread_id];
6053 struct instruction *ip = t->ip;
6054 uint64_t *regarray, idx, src;
6056 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
6059 regarray = instr_regarray_regarray(p, ip);
6060 idx = instr_regarray_idx_imm(p, ip);
6061 src = ip->regarray.dstsrc_val;
6062 regarray[idx] += src;
6071 static struct metarray *
6072 metarray_find(struct rte_swx_pipeline *p, const char *name);
6075 instr_metprefetch_translate(struct rte_swx_pipeline *p,
6076 struct action *action,
6079 struct instruction *instr,
6080 struct instruction_data *data __rte_unused)
6082 char *metarray = tokens[1], *idx = tokens[2];
6085 uint32_t idx_struct_id, idx_val;
6087 CHECK(n_tokens == 3, EINVAL);
6089 m = metarray_find(p, metarray);
6092 /* METPREFETCH_H, METPREFETCH_M. */
6093 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6095 instr->type = INSTR_METPREFETCH_M;
6097 instr->type = INSTR_METPREFETCH_H;
6099 instr->meter.metarray_id = m->id;
6100 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6101 instr->meter.idx.n_bits = fidx->n_bits;
6102 instr->meter.idx.offset = fidx->offset / 8;
6106 /* METPREFETCH_I. */
6107 idx_val = strtoul(idx, &idx, 0);
6108 CHECK(!idx[0], EINVAL);
6110 instr->type = INSTR_METPREFETCH_I;
6111 instr->meter.metarray_id = m->id;
6112 instr->meter.idx_val = idx_val;
6117 instr_meter_translate(struct rte_swx_pipeline *p,
6118 struct action *action,
6121 struct instruction *instr,
6122 struct instruction_data *data __rte_unused)
6124 char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
6125 char *color_in = tokens[4], *color_out = tokens[5];
6127 struct field *fidx, *flength, *fcin, *fcout;
6128 uint32_t idx_struct_id, length_struct_id;
6129 uint32_t color_in_struct_id, color_out_struct_id;
6131 CHECK(n_tokens == 6, EINVAL);
6133 m = metarray_find(p, metarray);
6136 fidx = struct_field_parse(p, action, idx, &idx_struct_id);
6138 flength = struct_field_parse(p, action, length, &length_struct_id);
6139 CHECK(flength, EINVAL);
6141 fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
6143 fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
6144 CHECK(fcout, EINVAL);
6146 /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
6148 instr->type = INSTR_METER_MMM;
6149 if (idx[0] == 'h' && length[0] == 'h')
6150 instr->type = INSTR_METER_HHM;
6151 if (idx[0] == 'h' && length[0] != 'h')
6152 instr->type = INSTR_METER_HMM;
6153 if (idx[0] != 'h' && length[0] == 'h')
6154 instr->type = INSTR_METER_MHM;
6156 instr->meter.metarray_id = m->id;
6158 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6159 instr->meter.idx.n_bits = fidx->n_bits;
6160 instr->meter.idx.offset = fidx->offset / 8;
6162 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6163 instr->meter.length.n_bits = flength->n_bits;
6164 instr->meter.length.offset = flength->offset / 8;
6166 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6167 instr->meter.color_in.n_bits = fcin->n_bits;
6168 instr->meter.color_in.offset = fcin->offset / 8;
6170 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6171 instr->meter.color_out.n_bits = fcout->n_bits;
6172 instr->meter.color_out.offset = fcout->offset / 8;
6177 /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
6178 if (fidx && !fcin) {
6179 uint32_t color_in_val = strtoul(color_in, &color_in, 0);
6180 CHECK(!color_in[0], EINVAL);
6182 instr->type = INSTR_METER_MMI;
6183 if (idx[0] == 'h' && length[0] == 'h')
6184 instr->type = INSTR_METER_HHI;
6185 if (idx[0] == 'h' && length[0] != 'h')
6186 instr->type = INSTR_METER_HMI;
6187 if (idx[0] != 'h' && length[0] == 'h')
6188 instr->type = INSTR_METER_MHI;
6190 instr->meter.metarray_id = m->id;
6192 instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
6193 instr->meter.idx.n_bits = fidx->n_bits;
6194 instr->meter.idx.offset = fidx->offset / 8;
6196 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6197 instr->meter.length.n_bits = flength->n_bits;
6198 instr->meter.length.offset = flength->offset / 8;
6200 instr->meter.color_in_val = color_in_val;
6202 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6203 instr->meter.color_out.n_bits = fcout->n_bits;
6204 instr->meter.color_out.offset = fcout->offset / 8;
6209 /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
6210 if (!fidx && fcin) {
6213 idx_val = strtoul(idx, &idx, 0);
6214 CHECK(!idx[0], EINVAL);
6216 instr->type = INSTR_METER_IMM;
6217 if (length[0] == 'h')
6218 instr->type = INSTR_METER_IHM;
6220 instr->meter.metarray_id = m->id;
6222 instr->meter.idx_val = idx_val;
6224 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6225 instr->meter.length.n_bits = flength->n_bits;
6226 instr->meter.length.offset = flength->offset / 8;
6228 instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
6229 instr->meter.color_in.n_bits = fcin->n_bits;
6230 instr->meter.color_in.offset = fcin->offset / 8;
6232 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6233 instr->meter.color_out.n_bits = fcout->n_bits;
6234 instr->meter.color_out.offset = fcout->offset / 8;
6239 /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
6240 if (!fidx && !fcin) {
6241 uint32_t idx_val, color_in_val;
6243 idx_val = strtoul(idx, &idx, 0);
6244 CHECK(!idx[0], EINVAL);
6246 color_in_val = strtoul(color_in, &color_in, 0);
6247 CHECK(!color_in[0], EINVAL);
6249 instr->type = INSTR_METER_IMI;
6250 if (length[0] == 'h')
6251 instr->type = INSTR_METER_IHI;
6253 instr->meter.metarray_id = m->id;
6255 instr->meter.idx_val = idx_val;
6257 instr->meter.length.struct_id = (uint8_t)length_struct_id;
6258 instr->meter.length.n_bits = flength->n_bits;
6259 instr->meter.length.offset = flength->offset / 8;
6261 instr->meter.color_in_val = color_in_val;
6263 instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
6264 instr->meter.color_out.n_bits = fcout->n_bits;
6265 instr->meter.color_out.offset = fcout->offset / 8;
6273 static inline struct meter *
6274 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6276 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6278 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6279 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6280 uint64_t idx64 = *idx64_ptr;
6281 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
6282 uint64_t idx = idx64 & idx64_mask & r->size_mask;
6284 return &r->metarray[idx];
6287 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6289 static inline struct meter *
6290 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
6292 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6294 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
6295 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
6296 uint64_t idx64 = *idx64_ptr;
6297 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
6299 return &r->metarray[idx];
6304 #define instr_meter_idx_nbo instr_meter_idx_hbo
6308 static inline struct meter *
6309 instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
6311 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
6313 uint64_t idx = ip->meter.idx_val & r->size_mask;
6315 return &r->metarray[idx];
6318 static inline uint32_t
6319 instr_meter_length_hbo(struct thread *t, struct instruction *ip)
6321 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6322 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6323 uint64_t src64 = *src64_ptr;
6324 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
6325 uint64_t src = src64 & src64_mask;
6327 return (uint32_t)src;
6330 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
6332 static inline uint32_t
6333 instr_meter_length_nbo(struct thread *t, struct instruction *ip)
6335 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
6336 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
6337 uint64_t src64 = *src64_ptr;
6338 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
6340 return (uint32_t)src;
6345 #define instr_meter_length_nbo instr_meter_length_hbo
6349 static inline enum rte_color
6350 instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
6352 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
6353 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
6354 uint64_t src64 = *src64_ptr;
6355 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
6356 uint64_t src = src64 & src64_mask;
6358 return (enum rte_color)src;
6362 instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
6364 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
6365 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
6366 uint64_t dst64 = *dst64_ptr;
6367 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
6369 uint64_t src = (uint64_t)color_out;
6371 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
6375 instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
6377 struct thread *t = &p->threads[p->thread_id];
6378 struct instruction *ip = t->ip;
6381 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
6384 m = instr_meter_idx_nbo(p, t, ip);
6392 instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
6394 struct thread *t = &p->threads[p->thread_id];
6395 struct instruction *ip = t->ip;
6398 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
6401 m = instr_meter_idx_hbo(p, t, ip);
6409 instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
6411 struct thread *t = &p->threads[p->thread_id];
6412 struct instruction *ip = t->ip;
6415 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
6418 m = instr_meter_idx_imm(p, ip);
6426 instr_meter_hhm_exec(struct rte_swx_pipeline *p)
6428 struct thread *t = &p->threads[p->thread_id];
6429 struct instruction *ip = t->ip;
6431 uint64_t time, n_pkts, n_bytes;
6433 enum rte_color color_in, color_out;
6435 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
6438 m = instr_meter_idx_nbo(p, t, ip);
6439 rte_prefetch0(m->n_pkts);
6440 time = rte_get_tsc_cycles();
6441 length = instr_meter_length_nbo(t, ip);
6442 color_in = instr_meter_color_in_hbo(t, ip);
6444 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6445 &m->profile->profile,
6450 color_out &= m->color_mask;
6452 n_pkts = m->n_pkts[color_out];
6453 n_bytes = m->n_bytes[color_out];
6455 instr_meter_color_out_hbo_set(t, ip, color_out);
6457 m->n_pkts[color_out] = n_pkts + 1;
6458 m->n_bytes[color_out] = n_bytes + length;
6465 instr_meter_hhi_exec(struct rte_swx_pipeline *p)
6467 struct thread *t = &p->threads[p->thread_id];
6468 struct instruction *ip = t->ip;
6470 uint64_t time, n_pkts, n_bytes;
6472 enum rte_color color_in, color_out;
6474 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
6477 m = instr_meter_idx_nbo(p, t, ip);
6478 rte_prefetch0(m->n_pkts);
6479 time = rte_get_tsc_cycles();
6480 length = instr_meter_length_nbo(t, ip);
6481 color_in = (enum rte_color)ip->meter.color_in_val;
6483 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6484 &m->profile->profile,
6489 color_out &= m->color_mask;
6491 n_pkts = m->n_pkts[color_out];
6492 n_bytes = m->n_bytes[color_out];
6494 instr_meter_color_out_hbo_set(t, ip, color_out);
6496 m->n_pkts[color_out] = n_pkts + 1;
6497 m->n_bytes[color_out] = n_bytes + length;
6504 instr_meter_hmm_exec(struct rte_swx_pipeline *p)
6506 struct thread *t = &p->threads[p->thread_id];
6507 struct instruction *ip = t->ip;
6509 uint64_t time, n_pkts, n_bytes;
6511 enum rte_color color_in, color_out;
6513 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
6516 m = instr_meter_idx_nbo(p, t, ip);
6517 rte_prefetch0(m->n_pkts);
6518 time = rte_get_tsc_cycles();
6519 length = instr_meter_length_hbo(t, ip);
6520 color_in = instr_meter_color_in_hbo(t, ip);
6522 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6523 &m->profile->profile,
6528 color_out &= m->color_mask;
6530 n_pkts = m->n_pkts[color_out];
6531 n_bytes = m->n_bytes[color_out];
6533 instr_meter_color_out_hbo_set(t, ip, color_out);
6535 m->n_pkts[color_out] = n_pkts + 1;
6536 m->n_bytes[color_out] = n_bytes + length;
6542 instr_meter_hmi_exec(struct rte_swx_pipeline *p)
6544 struct thread *t = &p->threads[p->thread_id];
6545 struct instruction *ip = t->ip;
6547 uint64_t time, n_pkts, n_bytes;
6549 enum rte_color color_in, color_out;
6551 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
6554 m = instr_meter_idx_nbo(p, t, ip);
6555 rte_prefetch0(m->n_pkts);
6556 time = rte_get_tsc_cycles();
6557 length = instr_meter_length_hbo(t, ip);
6558 color_in = (enum rte_color)ip->meter.color_in_val;
6560 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6561 &m->profile->profile,
6566 color_out &= m->color_mask;
6568 n_pkts = m->n_pkts[color_out];
6569 n_bytes = m->n_bytes[color_out];
6571 instr_meter_color_out_hbo_set(t, ip, color_out);
6573 m->n_pkts[color_out] = n_pkts + 1;
6574 m->n_bytes[color_out] = n_bytes + length;
6581 instr_meter_mhm_exec(struct rte_swx_pipeline *p)
6583 struct thread *t = &p->threads[p->thread_id];
6584 struct instruction *ip = t->ip;
6586 uint64_t time, n_pkts, n_bytes;
6588 enum rte_color color_in, color_out;
6590 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
6593 m = instr_meter_idx_hbo(p, t, ip);
6594 rte_prefetch0(m->n_pkts);
6595 time = rte_get_tsc_cycles();
6596 length = instr_meter_length_nbo(t, ip);
6597 color_in = instr_meter_color_in_hbo(t, ip);
6599 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6600 &m->profile->profile,
6605 color_out &= m->color_mask;
6607 n_pkts = m->n_pkts[color_out];
6608 n_bytes = m->n_bytes[color_out];
6610 instr_meter_color_out_hbo_set(t, ip, color_out);
6612 m->n_pkts[color_out] = n_pkts + 1;
6613 m->n_bytes[color_out] = n_bytes + length;
6620 instr_meter_mhi_exec(struct rte_swx_pipeline *p)
6622 struct thread *t = &p->threads[p->thread_id];
6623 struct instruction *ip = t->ip;
6625 uint64_t time, n_pkts, n_bytes;
6627 enum rte_color color_in, color_out;
6629 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
6632 m = instr_meter_idx_hbo(p, t, ip);
6633 rte_prefetch0(m->n_pkts);
6634 time = rte_get_tsc_cycles();
6635 length = instr_meter_length_nbo(t, ip);
6636 color_in = (enum rte_color)ip->meter.color_in_val;
6638 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6639 &m->profile->profile,
6644 color_out &= m->color_mask;
6646 n_pkts = m->n_pkts[color_out];
6647 n_bytes = m->n_bytes[color_out];
6649 instr_meter_color_out_hbo_set(t, ip, color_out);
6651 m->n_pkts[color_out] = n_pkts + 1;
6652 m->n_bytes[color_out] = n_bytes + length;
6659 instr_meter_mmm_exec(struct rte_swx_pipeline *p)
6661 struct thread *t = &p->threads[p->thread_id];
6662 struct instruction *ip = t->ip;
6664 uint64_t time, n_pkts, n_bytes;
6666 enum rte_color color_in, color_out;
6668 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
6671 m = instr_meter_idx_hbo(p, t, ip);
6672 rte_prefetch0(m->n_pkts);
6673 time = rte_get_tsc_cycles();
6674 length = instr_meter_length_hbo(t, ip);
6675 color_in = instr_meter_color_in_hbo(t, ip);
6677 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6678 &m->profile->profile,
6683 color_out &= m->color_mask;
6685 n_pkts = m->n_pkts[color_out];
6686 n_bytes = m->n_bytes[color_out];
6688 instr_meter_color_out_hbo_set(t, ip, color_out);
6690 m->n_pkts[color_out] = n_pkts + 1;
6691 m->n_bytes[color_out] = n_bytes + length;
6698 instr_meter_mmi_exec(struct rte_swx_pipeline *p)
6700 struct thread *t = &p->threads[p->thread_id];
6701 struct instruction *ip = t->ip;
6703 uint64_t time, n_pkts, n_bytes;
6705 enum rte_color color_in, color_out;
6707 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
6710 m = instr_meter_idx_hbo(p, t, ip);
6711 rte_prefetch0(m->n_pkts);
6712 time = rte_get_tsc_cycles();
6713 length = instr_meter_length_hbo(t, ip);
6714 color_in = (enum rte_color)ip->meter.color_in_val;
6716 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6717 &m->profile->profile,
6722 color_out &= m->color_mask;
6724 n_pkts = m->n_pkts[color_out];
6725 n_bytes = m->n_bytes[color_out];
6727 instr_meter_color_out_hbo_set(t, ip, color_out);
6729 m->n_pkts[color_out] = n_pkts + 1;
6730 m->n_bytes[color_out] = n_bytes + length;
6737 instr_meter_ihm_exec(struct rte_swx_pipeline *p)
6739 struct thread *t = &p->threads[p->thread_id];
6740 struct instruction *ip = t->ip;
6742 uint64_t time, n_pkts, n_bytes;
6744 enum rte_color color_in, color_out;
6746 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
6749 m = instr_meter_idx_imm(p, ip);
6750 rte_prefetch0(m->n_pkts);
6751 time = rte_get_tsc_cycles();
6752 length = instr_meter_length_nbo(t, ip);
6753 color_in = instr_meter_color_in_hbo(t, ip);
6755 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6756 &m->profile->profile,
6761 color_out &= m->color_mask;
6763 n_pkts = m->n_pkts[color_out];
6764 n_bytes = m->n_bytes[color_out];
6766 instr_meter_color_out_hbo_set(t, ip, color_out);
6768 m->n_pkts[color_out] = n_pkts + 1;
6769 m->n_bytes[color_out] = n_bytes + length;
6776 instr_meter_ihi_exec(struct rte_swx_pipeline *p)
6778 struct thread *t = &p->threads[p->thread_id];
6779 struct instruction *ip = t->ip;
6781 uint64_t time, n_pkts, n_bytes;
6783 enum rte_color color_in, color_out;
6785 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
6788 m = instr_meter_idx_imm(p, ip);
6789 rte_prefetch0(m->n_pkts);
6790 time = rte_get_tsc_cycles();
6791 length = instr_meter_length_nbo(t, ip);
6792 color_in = (enum rte_color)ip->meter.color_in_val;
6794 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6795 &m->profile->profile,
6800 color_out &= m->color_mask;
6802 n_pkts = m->n_pkts[color_out];
6803 n_bytes = m->n_bytes[color_out];
6805 instr_meter_color_out_hbo_set(t, ip, color_out);
6807 m->n_pkts[color_out] = n_pkts + 1;
6808 m->n_bytes[color_out] = n_bytes + length;
6815 instr_meter_imm_exec(struct rte_swx_pipeline *p)
6817 struct thread *t = &p->threads[p->thread_id];
6818 struct instruction *ip = t->ip;
6820 uint64_t time, n_pkts, n_bytes;
6822 enum rte_color color_in, color_out;
6824 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
6827 m = instr_meter_idx_imm(p, ip);
6828 rte_prefetch0(m->n_pkts);
6829 time = rte_get_tsc_cycles();
6830 length = instr_meter_length_hbo(t, ip);
6831 color_in = instr_meter_color_in_hbo(t, ip);
6833 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6834 &m->profile->profile,
6839 color_out &= m->color_mask;
6841 n_pkts = m->n_pkts[color_out];
6842 n_bytes = m->n_bytes[color_out];
6844 instr_meter_color_out_hbo_set(t, ip, color_out);
6846 m->n_pkts[color_out] = n_pkts + 1;
6847 m->n_bytes[color_out] = n_bytes + length;
6853 instr_meter_imi_exec(struct rte_swx_pipeline *p)
6855 struct thread *t = &p->threads[p->thread_id];
6856 struct instruction *ip = t->ip;
6858 uint64_t time, n_pkts, n_bytes;
6860 enum rte_color color_in, color_out;
6862 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
6865 m = instr_meter_idx_imm(p, ip);
6866 rte_prefetch0(m->n_pkts);
6867 time = rte_get_tsc_cycles();
6868 length = instr_meter_length_hbo(t, ip);
6869 color_in = (enum rte_color)ip->meter.color_in_val;
6871 color_out = rte_meter_trtcm_color_aware_check(&m->m,
6872 &m->profile->profile,
6877 color_out &= m->color_mask;
6879 n_pkts = m->n_pkts[color_out];
6880 n_bytes = m->n_bytes[color_out];
6882 instr_meter_color_out_hbo_set(t, ip, color_out);
6884 m->n_pkts[color_out] = n_pkts + 1;
6885 m->n_bytes[color_out] = n_bytes + length;
6894 static struct action *
6895 action_find(struct rte_swx_pipeline *p, const char *name);
6898 instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
6899 struct action *action __rte_unused,
6902 struct instruction *instr,
6903 struct instruction_data *data)
6905 CHECK(n_tokens == 2, EINVAL);
6907 strcpy(data->jmp_label, tokens[1]);
6909 instr->type = INSTR_JMP;
6910 instr->jmp.ip = NULL; /* Resolved later. */
6915 instr_jmp_valid_translate(struct rte_swx_pipeline *p,
6916 struct action *action __rte_unused,
6919 struct instruction *instr,
6920 struct instruction_data *data)
6924 CHECK(n_tokens == 3, EINVAL);
6926 strcpy(data->jmp_label, tokens[1]);
6928 h = header_parse(p, tokens[2]);
6931 instr->type = INSTR_JMP_VALID;
6932 instr->jmp.ip = NULL; /* Resolved later. */
6933 instr->jmp.header_id = h->id;
6938 instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
6939 struct action *action __rte_unused,
6942 struct instruction *instr,
6943 struct instruction_data *data)
6947 CHECK(n_tokens == 3, EINVAL);
6949 strcpy(data->jmp_label, tokens[1]);
6951 h = header_parse(p, tokens[2]);
6954 instr->type = INSTR_JMP_INVALID;
6955 instr->jmp.ip = NULL; /* Resolved later. */
6956 instr->jmp.header_id = h->id;
6961 instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
6962 struct action *action,
6965 struct instruction *instr,
6966 struct instruction_data *data)
6968 CHECK(!action, EINVAL);
6969 CHECK(n_tokens == 2, EINVAL);
6971 strcpy(data->jmp_label, tokens[1]);
6973 instr->type = INSTR_JMP_HIT;
6974 instr->jmp.ip = NULL; /* Resolved later. */
6979 instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
6980 struct action *action,
6983 struct instruction *instr,
6984 struct instruction_data *data)
6986 CHECK(!action, EINVAL);
6987 CHECK(n_tokens == 2, EINVAL);
6989 strcpy(data->jmp_label, tokens[1]);
6991 instr->type = INSTR_JMP_MISS;
6992 instr->jmp.ip = NULL; /* Resolved later. */
6997 instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
6998 struct action *action,
7001 struct instruction *instr,
7002 struct instruction_data *data)
7006 CHECK(!action, EINVAL);
7007 CHECK(n_tokens == 3, EINVAL);
7009 strcpy(data->jmp_label, tokens[1]);
7011 a = action_find(p, tokens[2]);
7014 instr->type = INSTR_JMP_ACTION_HIT;
7015 instr->jmp.ip = NULL; /* Resolved later. */
7016 instr->jmp.action_id = a->id;
7021 instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
7022 struct action *action,
7025 struct instruction *instr,
7026 struct instruction_data *data)
7030 CHECK(!action, EINVAL);
7031 CHECK(n_tokens == 3, EINVAL);
7033 strcpy(data->jmp_label, tokens[1]);
7035 a = action_find(p, tokens[2]);
7038 instr->type = INSTR_JMP_ACTION_MISS;
7039 instr->jmp.ip = NULL; /* Resolved later. */
7040 instr->jmp.action_id = a->id;
7045 instr_jmp_eq_translate(struct rte_swx_pipeline *p,
7046 struct action *action,
7049 struct instruction *instr,
7050 struct instruction_data *data)
7052 char *a = tokens[2], *b = tokens[3];
7053 struct field *fa, *fb;
7055 uint32_t a_struct_id, b_struct_id;
7057 CHECK(n_tokens == 4, EINVAL);
7059 strcpy(data->jmp_label, tokens[1]);
7061 fa = struct_field_parse(p, action, a, &a_struct_id);
7064 /* JMP_EQ, JMP_EQ_MH, JMP_EQ_HM, JMP_EQ_HH. */
7065 fb = struct_field_parse(p, action, b, &b_struct_id);
7067 instr->type = INSTR_JMP_EQ;
7068 if (a[0] != 'h' && b[0] == 'h')
7069 instr->type = INSTR_JMP_EQ_MH;
7070 if (a[0] == 'h' && b[0] != 'h')
7071 instr->type = INSTR_JMP_EQ_HM;
7072 if (a[0] == 'h' && b[0] == 'h')
7073 instr->type = INSTR_JMP_EQ_HH;
7074 instr->jmp.ip = NULL; /* Resolved later. */
7076 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7077 instr->jmp.a.n_bits = fa->n_bits;
7078 instr->jmp.a.offset = fa->offset / 8;
7079 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7080 instr->jmp.b.n_bits = fb->n_bits;
7081 instr->jmp.b.offset = fb->offset / 8;
7086 b_val = strtoull(b, &b, 0);
7087 CHECK(!b[0], EINVAL);
7090 b_val = hton64(b_val) >> (64 - fa->n_bits);
7092 instr->type = INSTR_JMP_EQ_I;
7093 instr->jmp.ip = NULL; /* Resolved later. */
7094 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7095 instr->jmp.a.n_bits = fa->n_bits;
7096 instr->jmp.a.offset = fa->offset / 8;
7097 instr->jmp.b_val = b_val;
7102 instr_jmp_neq_translate(struct rte_swx_pipeline *p,
7103 struct action *action,
7106 struct instruction *instr,
7107 struct instruction_data *data)
7109 char *a = tokens[2], *b = tokens[3];
7110 struct field *fa, *fb;
7112 uint32_t a_struct_id, b_struct_id;
7114 CHECK(n_tokens == 4, EINVAL);
7116 strcpy(data->jmp_label, tokens[1]);
7118 fa = struct_field_parse(p, action, a, &a_struct_id);
7121 /* JMP_NEQ, JMP_NEQ_MH, JMP_NEQ_HM, JMP_NEQ_HH. */
7122 fb = struct_field_parse(p, action, b, &b_struct_id);
7124 instr->type = INSTR_JMP_NEQ;
7125 if (a[0] != 'h' && b[0] == 'h')
7126 instr->type = INSTR_JMP_NEQ_MH;
7127 if (a[0] == 'h' && b[0] != 'h')
7128 instr->type = INSTR_JMP_NEQ_HM;
7129 if (a[0] == 'h' && b[0] == 'h')
7130 instr->type = INSTR_JMP_NEQ_HH;
7131 instr->jmp.ip = NULL; /* Resolved later. */
7133 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7134 instr->jmp.a.n_bits = fa->n_bits;
7135 instr->jmp.a.offset = fa->offset / 8;
7136 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7137 instr->jmp.b.n_bits = fb->n_bits;
7138 instr->jmp.b.offset = fb->offset / 8;
7143 b_val = strtoull(b, &b, 0);
7144 CHECK(!b[0], EINVAL);
7147 b_val = hton64(b_val) >> (64 - fa->n_bits);
7149 instr->type = INSTR_JMP_NEQ_I;
7150 instr->jmp.ip = NULL; /* Resolved later. */
7151 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7152 instr->jmp.a.n_bits = fa->n_bits;
7153 instr->jmp.a.offset = fa->offset / 8;
7154 instr->jmp.b_val = b_val;
7159 instr_jmp_lt_translate(struct rte_swx_pipeline *p,
7160 struct action *action,
7163 struct instruction *instr,
7164 struct instruction_data *data)
7166 char *a = tokens[2], *b = tokens[3];
7167 struct field *fa, *fb;
7169 uint32_t a_struct_id, b_struct_id;
7171 CHECK(n_tokens == 4, EINVAL);
7173 strcpy(data->jmp_label, tokens[1]);
7175 fa = struct_field_parse(p, action, a, &a_struct_id);
7178 /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
7179 fb = struct_field_parse(p, action, b, &b_struct_id);
7181 instr->type = INSTR_JMP_LT;
7182 if (a[0] == 'h' && b[0] != 'h')
7183 instr->type = INSTR_JMP_LT_HM;
7184 if (a[0] != 'h' && b[0] == 'h')
7185 instr->type = INSTR_JMP_LT_MH;
7186 if (a[0] == 'h' && b[0] == 'h')
7187 instr->type = INSTR_JMP_LT_HH;
7188 instr->jmp.ip = NULL; /* Resolved later. */
7190 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7191 instr->jmp.a.n_bits = fa->n_bits;
7192 instr->jmp.a.offset = fa->offset / 8;
7193 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7194 instr->jmp.b.n_bits = fb->n_bits;
7195 instr->jmp.b.offset = fb->offset / 8;
7199 /* JMP_LT_MI, JMP_LT_HI. */
7200 b_val = strtoull(b, &b, 0);
7201 CHECK(!b[0], EINVAL);
7203 instr->type = INSTR_JMP_LT_MI;
7205 instr->type = INSTR_JMP_LT_HI;
7206 instr->jmp.ip = NULL; /* Resolved later. */
7208 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7209 instr->jmp.a.n_bits = fa->n_bits;
7210 instr->jmp.a.offset = fa->offset / 8;
7211 instr->jmp.b_val = b_val;
7216 instr_jmp_gt_translate(struct rte_swx_pipeline *p,
7217 struct action *action,
7220 struct instruction *instr,
7221 struct instruction_data *data)
7223 char *a = tokens[2], *b = tokens[3];
7224 struct field *fa, *fb;
7226 uint32_t a_struct_id, b_struct_id;
7228 CHECK(n_tokens == 4, EINVAL);
7230 strcpy(data->jmp_label, tokens[1]);
7232 fa = struct_field_parse(p, action, a, &a_struct_id);
7235 /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
7236 fb = struct_field_parse(p, action, b, &b_struct_id);
7238 instr->type = INSTR_JMP_GT;
7239 if (a[0] == 'h' && b[0] != 'h')
7240 instr->type = INSTR_JMP_GT_HM;
7241 if (a[0] != 'h' && b[0] == 'h')
7242 instr->type = INSTR_JMP_GT_MH;
7243 if (a[0] == 'h' && b[0] == 'h')
7244 instr->type = INSTR_JMP_GT_HH;
7245 instr->jmp.ip = NULL; /* Resolved later. */
7247 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7248 instr->jmp.a.n_bits = fa->n_bits;
7249 instr->jmp.a.offset = fa->offset / 8;
7250 instr->jmp.b.struct_id = (uint8_t)b_struct_id;
7251 instr->jmp.b.n_bits = fb->n_bits;
7252 instr->jmp.b.offset = fb->offset / 8;
7256 /* JMP_GT_MI, JMP_GT_HI. */
7257 b_val = strtoull(b, &b, 0);
7258 CHECK(!b[0], EINVAL);
7260 instr->type = INSTR_JMP_GT_MI;
7262 instr->type = INSTR_JMP_GT_HI;
7263 instr->jmp.ip = NULL; /* Resolved later. */
7265 instr->jmp.a.struct_id = (uint8_t)a_struct_id;
7266 instr->jmp.a.n_bits = fa->n_bits;
7267 instr->jmp.a.offset = fa->offset / 8;
7268 instr->jmp.b_val = b_val;
7273 instr_jmp_exec(struct rte_swx_pipeline *p)
7275 struct thread *t = &p->threads[p->thread_id];
7276 struct instruction *ip = t->ip;
7278 TRACE("[Thread %2u] jmp\n", p->thread_id);
7280 thread_ip_set(t, ip->jmp.ip);
7284 instr_jmp_valid_exec(struct rte_swx_pipeline *p)
7286 struct thread *t = &p->threads[p->thread_id];
7287 struct instruction *ip = t->ip;
7288 uint32_t header_id = ip->jmp.header_id;
7290 TRACE("[Thread %2u] jmpv\n", p->thread_id);
7292 t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
7296 instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
7298 struct thread *t = &p->threads[p->thread_id];
7299 struct instruction *ip = t->ip;
7300 uint32_t header_id = ip->jmp.header_id;
7302 TRACE("[Thread %2u] jmpnv\n", p->thread_id);
7304 t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
7308 instr_jmp_hit_exec(struct rte_swx_pipeline *p)
7310 struct thread *t = &p->threads[p->thread_id];
7311 struct instruction *ip = t->ip;
7312 struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
7314 TRACE("[Thread %2u] jmph\n", p->thread_id);
7316 t->ip = ip_next[t->hit];
7320 instr_jmp_miss_exec(struct rte_swx_pipeline *p)
7322 struct thread *t = &p->threads[p->thread_id];
7323 struct instruction *ip = t->ip;
7324 struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
7326 TRACE("[Thread %2u] jmpnh\n", p->thread_id);
7328 t->ip = ip_next[t->hit];
7332 instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
7334 struct thread *t = &p->threads[p->thread_id];
7335 struct instruction *ip = t->ip;
7337 TRACE("[Thread %2u] jmpa\n", p->thread_id);
7339 t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
7343 instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
7345 struct thread *t = &p->threads[p->thread_id];
7346 struct instruction *ip = t->ip;
7348 TRACE("[Thread %2u] jmpna\n", p->thread_id);
7350 t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
7354 instr_jmp_eq_exec(struct rte_swx_pipeline *p)
7356 struct thread *t = &p->threads[p->thread_id];
7357 struct instruction *ip = t->ip;
7359 TRACE("[Thread %2u] jmpeq\n", p->thread_id);
7365 instr_jmp_eq_mh_exec(struct rte_swx_pipeline *p)
7367 struct thread *t = &p->threads[p->thread_id];
7368 struct instruction *ip = t->ip;
7370 TRACE("[Thread %2u] jmpeq (mh)\n", p->thread_id);
7372 JMP_CMP_MH(t, ip, ==);
7376 instr_jmp_eq_hm_exec(struct rte_swx_pipeline *p)
7378 struct thread *t = &p->threads[p->thread_id];
7379 struct instruction *ip = t->ip;
7381 TRACE("[Thread %2u] jmpeq (hm)\n", p->thread_id);
7383 JMP_CMP_HM(t, ip, ==);
7387 instr_jmp_eq_hh_exec(struct rte_swx_pipeline *p)
7389 struct thread *t = &p->threads[p->thread_id];
7390 struct instruction *ip = t->ip;
7392 TRACE("[Thread %2u] jmpeq (hh)\n", p->thread_id);
7394 JMP_CMP_HH_FAST(t, ip, ==);
7398 instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
7400 struct thread *t = &p->threads[p->thread_id];
7401 struct instruction *ip = t->ip;
7403 TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
7405 JMP_CMP_I(t, ip, ==);
7409 instr_jmp_neq_exec(struct rte_swx_pipeline *p)
7411 struct thread *t = &p->threads[p->thread_id];
7412 struct instruction *ip = t->ip;
7414 TRACE("[Thread %2u] jmpneq\n", p->thread_id);
7420 instr_jmp_neq_mh_exec(struct rte_swx_pipeline *p)
7422 struct thread *t = &p->threads[p->thread_id];
7423 struct instruction *ip = t->ip;
7425 TRACE("[Thread %2u] jmpneq (mh)\n", p->thread_id);
7427 JMP_CMP_MH(t, ip, !=);
7431 instr_jmp_neq_hm_exec(struct rte_swx_pipeline *p)
7433 struct thread *t = &p->threads[p->thread_id];
7434 struct instruction *ip = t->ip;
7436 TRACE("[Thread %2u] jmpneq (hm)\n", p->thread_id);
7438 JMP_CMP_HM(t, ip, !=);
7442 instr_jmp_neq_hh_exec(struct rte_swx_pipeline *p)
7444 struct thread *t = &p->threads[p->thread_id];
7445 struct instruction *ip = t->ip;
7447 TRACE("[Thread %2u] jmpneq (hh)\n", p->thread_id);
7449 JMP_CMP_HH_FAST(t, ip, !=);
7453 instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
7455 struct thread *t = &p->threads[p->thread_id];
7456 struct instruction *ip = t->ip;
7458 TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
7460 JMP_CMP_I(t, ip, !=);
7464 instr_jmp_lt_exec(struct rte_swx_pipeline *p)
7466 struct thread *t = &p->threads[p->thread_id];
7467 struct instruction *ip = t->ip;
7469 TRACE("[Thread %2u] jmplt\n", p->thread_id);
7475 instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
7477 struct thread *t = &p->threads[p->thread_id];
7478 struct instruction *ip = t->ip;
7480 TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
7482 JMP_CMP_MH(t, ip, <);
7486 instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
7488 struct thread *t = &p->threads[p->thread_id];
7489 struct instruction *ip = t->ip;
7491 TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
7493 JMP_CMP_HM(t, ip, <);
7497 instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
7499 struct thread *t = &p->threads[p->thread_id];
7500 struct instruction *ip = t->ip;
7502 TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
7504 JMP_CMP_HH(t, ip, <);
7508 instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
7510 struct thread *t = &p->threads[p->thread_id];
7511 struct instruction *ip = t->ip;
7513 TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
7515 JMP_CMP_MI(t, ip, <);
7519 instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
7521 struct thread *t = &p->threads[p->thread_id];
7522 struct instruction *ip = t->ip;
7524 TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
7526 JMP_CMP_HI(t, ip, <);
7530 instr_jmp_gt_exec(struct rte_swx_pipeline *p)
7532 struct thread *t = &p->threads[p->thread_id];
7533 struct instruction *ip = t->ip;
7535 TRACE("[Thread %2u] jmpgt\n", p->thread_id);
7541 instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
7543 struct thread *t = &p->threads[p->thread_id];
7544 struct instruction *ip = t->ip;
7546 TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
7548 JMP_CMP_MH(t, ip, >);
7552 instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
7554 struct thread *t = &p->threads[p->thread_id];
7555 struct instruction *ip = t->ip;
7557 TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
7559 JMP_CMP_HM(t, ip, >);
7563 instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
7565 struct thread *t = &p->threads[p->thread_id];
7566 struct instruction *ip = t->ip;
7568 TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
7570 JMP_CMP_HH(t, ip, >);
7574 instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
7576 struct thread *t = &p->threads[p->thread_id];
7577 struct instruction *ip = t->ip;
7579 TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
7581 JMP_CMP_MI(t, ip, >);
7585 instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
7587 struct thread *t = &p->threads[p->thread_id];
7588 struct instruction *ip = t->ip;
7590 TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
7592 JMP_CMP_HI(t, ip, >);
7599 instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
7600 struct action *action,
7601 char **tokens __rte_unused,
7603 struct instruction *instr,
7604 struct instruction_data *data __rte_unused)
7606 CHECK(action, EINVAL);
7607 CHECK(n_tokens == 1, EINVAL);
7609 instr->type = INSTR_RETURN;
7614 instr_return_exec(struct rte_swx_pipeline *p)
7616 struct thread *t = &p->threads[p->thread_id];
7618 TRACE("[Thread %2u] return\n", p->thread_id);
7624 instr_translate(struct rte_swx_pipeline *p,
7625 struct action *action,
7627 struct instruction *instr,
7628 struct instruction_data *data)
7630 char *tokens[RTE_SWX_INSTRUCTION_TOKENS_MAX];
7631 int n_tokens = 0, tpos = 0;
7633 /* Parse the instruction string into tokens. */
7637 token = strtok_r(string, " \t\v", &string);
7641 CHECK(n_tokens < RTE_SWX_INSTRUCTION_TOKENS_MAX, EINVAL);
7642 CHECK_NAME(token, EINVAL);
7644 tokens[n_tokens] = token;
7648 CHECK(n_tokens, EINVAL);
7650 /* Handle the optional instruction label. */
7651 if ((n_tokens >= 2) && !strcmp(tokens[1], ":")) {
7652 strcpy(data->label, tokens[0]);
7655 CHECK(n_tokens - tpos, EINVAL);
7658 /* Identify the instruction type. */
7659 if (!strcmp(tokens[tpos], "rx"))
7660 return instr_rx_translate(p,
7667 if (!strcmp(tokens[tpos], "tx"))
7668 return instr_tx_translate(p,
7675 if (!strcmp(tokens[tpos], "drop"))
7676 return instr_drop_translate(p,
7683 if (!strcmp(tokens[tpos], "extract"))
7684 return instr_hdr_extract_translate(p,
7691 if (!strcmp(tokens[tpos], "emit"))
7692 return instr_hdr_emit_translate(p,
7699 if (!strcmp(tokens[tpos], "validate"))
7700 return instr_hdr_validate_translate(p,
7707 if (!strcmp(tokens[tpos], "invalidate"))
7708 return instr_hdr_invalidate_translate(p,
7715 if (!strcmp(tokens[tpos], "mov"))
7716 return instr_mov_translate(p,
7723 if (!strcmp(tokens[tpos], "dma"))
7724 return instr_dma_translate(p,
7731 if (!strcmp(tokens[tpos], "add"))
7732 return instr_alu_add_translate(p,
7739 if (!strcmp(tokens[tpos], "sub"))
7740 return instr_alu_sub_translate(p,
7747 if (!strcmp(tokens[tpos], "ckadd"))
7748 return instr_alu_ckadd_translate(p,
7755 if (!strcmp(tokens[tpos], "cksub"))
7756 return instr_alu_cksub_translate(p,
7763 if (!strcmp(tokens[tpos], "and"))
7764 return instr_alu_and_translate(p,
7771 if (!strcmp(tokens[tpos], "or"))
7772 return instr_alu_or_translate(p,
7779 if (!strcmp(tokens[tpos], "xor"))
7780 return instr_alu_xor_translate(p,
7787 if (!strcmp(tokens[tpos], "shl"))
7788 return instr_alu_shl_translate(p,
7795 if (!strcmp(tokens[tpos], "shr"))
7796 return instr_alu_shr_translate(p,
7803 if (!strcmp(tokens[tpos], "regprefetch"))
7804 return instr_regprefetch_translate(p,
7811 if (!strcmp(tokens[tpos], "regrd"))
7812 return instr_regrd_translate(p,
7819 if (!strcmp(tokens[tpos], "regwr"))
7820 return instr_regwr_translate(p,
7827 if (!strcmp(tokens[tpos], "regadd"))
7828 return instr_regadd_translate(p,
7835 if (!strcmp(tokens[tpos], "metprefetch"))
7836 return instr_metprefetch_translate(p,
7843 if (!strcmp(tokens[tpos], "meter"))
7844 return instr_meter_translate(p,
7851 if (!strcmp(tokens[tpos], "table"))
7852 return instr_table_translate(p,
7859 if (!strcmp(tokens[tpos], "extern"))
7860 return instr_extern_translate(p,
7867 if (!strcmp(tokens[tpos], "jmp"))
7868 return instr_jmp_translate(p,
7875 if (!strcmp(tokens[tpos], "jmpv"))
7876 return instr_jmp_valid_translate(p,
7883 if (!strcmp(tokens[tpos], "jmpnv"))
7884 return instr_jmp_invalid_translate(p,
7891 if (!strcmp(tokens[tpos], "jmph"))
7892 return instr_jmp_hit_translate(p,
7899 if (!strcmp(tokens[tpos], "jmpnh"))
7900 return instr_jmp_miss_translate(p,
7907 if (!strcmp(tokens[tpos], "jmpa"))
7908 return instr_jmp_action_hit_translate(p,
7915 if (!strcmp(tokens[tpos], "jmpna"))
7916 return instr_jmp_action_miss_translate(p,
7923 if (!strcmp(tokens[tpos], "jmpeq"))
7924 return instr_jmp_eq_translate(p,
7931 if (!strcmp(tokens[tpos], "jmpneq"))
7932 return instr_jmp_neq_translate(p,
7939 if (!strcmp(tokens[tpos], "jmplt"))
7940 return instr_jmp_lt_translate(p,
7947 if (!strcmp(tokens[tpos], "jmpgt"))
7948 return instr_jmp_gt_translate(p,
7955 if (!strcmp(tokens[tpos], "return"))
7956 return instr_return_translate(p,
7966 static struct instruction_data *
7967 label_find(struct instruction_data *data, uint32_t n, const char *label)
7971 for (i = 0; i < n; i++)
7972 if (!strcmp(label, data[i].label))
7979 label_is_used(struct instruction_data *data, uint32_t n, const char *label)
7981 uint32_t count = 0, i;
7986 for (i = 0; i < n; i++)
7987 if (!strcmp(label, data[i].jmp_label))
7994 instr_label_check(struct instruction_data *instruction_data,
7995 uint32_t n_instructions)
7999 /* Check that all instruction labels are unique. */
8000 for (i = 0; i < n_instructions; i++) {
8001 struct instruction_data *data = &instruction_data[i];
8002 char *label = data->label;
8008 for (j = i + 1; j < n_instructions; j++)
8009 CHECK(strcmp(label, data[j].label), EINVAL);
8012 /* Get users for each instruction label. */
8013 for (i = 0; i < n_instructions; i++) {
8014 struct instruction_data *data = &instruction_data[i];
8015 char *label = data->label;
8017 data->n_users = label_is_used(instruction_data,
8026 instr_jmp_resolve(struct instruction *instructions,
8027 struct instruction_data *instruction_data,
8028 uint32_t n_instructions)
8032 for (i = 0; i < n_instructions; i++) {
8033 struct instruction *instr = &instructions[i];
8034 struct instruction_data *data = &instruction_data[i];
8035 struct instruction_data *found;
8037 if (!instruction_is_jmp(instr))
8040 found = label_find(instruction_data,
8043 CHECK(found, EINVAL);
8045 instr->jmp.ip = &instructions[found - instruction_data];
8052 instr_verify(struct rte_swx_pipeline *p __rte_unused,
8054 struct instruction *instr,
8055 struct instruction_data *data __rte_unused,
8056 uint32_t n_instructions)
8059 enum instruction_type type;
8062 /* Check that the first instruction is rx. */
8063 CHECK(instr[0].type == INSTR_RX, EINVAL);
8065 /* Check that there is at least one tx instruction. */
8066 for (i = 0; i < n_instructions; i++) {
8067 type = instr[i].type;
8069 if (instruction_is_tx(type))
8072 CHECK(i < n_instructions, EINVAL);
8074 /* Check that the last instruction is either tx or unconditional
8077 type = instr[n_instructions - 1].type;
8078 CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
8082 enum instruction_type type;
8085 /* Check that there is at least one return or tx instruction. */
8086 for (i = 0; i < n_instructions; i++) {
8087 type = instr[i].type;
8089 if ((type == INSTR_RETURN) || instruction_is_tx(type))
8092 CHECK(i < n_instructions, EINVAL);
8099 instr_compact(struct instruction *instructions,
8100 struct instruction_data *instruction_data,
8101 uint32_t n_instructions)
8103 uint32_t i, pos = 0;
8105 /* Eliminate the invalid instructions that have been optimized out. */
8106 for (i = 0; i < n_instructions; i++) {
8107 struct instruction *instr = &instructions[i];
8108 struct instruction_data *data = &instruction_data[i];
8114 memcpy(&instructions[pos], instr, sizeof(*instr));
8115 memcpy(&instruction_data[pos], data, sizeof(*data));
8125 instr_pattern_extract_many_search(struct instruction *instr,
8126 struct instruction_data *data,
8128 uint32_t *n_pattern_instr)
8132 for (i = 0; i < n_instr; i++) {
8133 if (data[i].invalid)
8136 if (instr[i].type != INSTR_HDR_EXTRACT)
8139 if (i == RTE_DIM(instr->io.hdr.header_id))
8142 if (i && data[i].n_users)
8149 *n_pattern_instr = i;
8154 instr_pattern_extract_many_replace(struct instruction *instr,
8155 struct instruction_data *data,
8160 for (i = 1; i < n_instr; i++) {
8162 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8163 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8164 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8166 data[i].invalid = 1;
8171 instr_pattern_extract_many_optimize(struct instruction *instructions,
8172 struct instruction_data *instruction_data,
8173 uint32_t n_instructions)
8177 for (i = 0; i < n_instructions; ) {
8178 struct instruction *instr = &instructions[i];
8179 struct instruction_data *data = &instruction_data[i];
8180 uint32_t n_instr = 0;
8184 detected = instr_pattern_extract_many_search(instr,
8189 instr_pattern_extract_many_replace(instr,
8196 /* No pattern starting at the current instruction. */
8200 /* Eliminate the invalid instructions that have been optimized out. */
8201 n_instructions = instr_compact(instructions,
8205 return n_instructions;
8209 instr_pattern_emit_many_tx_search(struct instruction *instr,
8210 struct instruction_data *data,
8212 uint32_t *n_pattern_instr)
8216 for (i = 0; i < n_instr; i++) {
8217 if (data[i].invalid)
8220 if (instr[i].type != INSTR_HDR_EMIT)
8223 if (i == RTE_DIM(instr->io.hdr.header_id))
8226 if (i && data[i].n_users)
8233 if (!instruction_is_tx(instr[i].type))
8236 if (data[i].n_users)
8241 *n_pattern_instr = i;
8246 instr_pattern_emit_many_tx_replace(struct instruction *instr,
8247 struct instruction_data *data,
8252 /* Any emit instruction in addition to the first one. */
8253 for (i = 1; i < n_instr - 1; i++) {
8255 instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
8256 instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
8257 instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
8259 data[i].invalid = 1;
8262 /* The TX instruction is the last one in the pattern. */
8264 instr[0].io.io.offset = instr[i].io.io.offset;
8265 instr[0].io.io.n_bits = instr[i].io.io.n_bits;
8266 data[i].invalid = 1;
8270 instr_pattern_emit_many_tx_optimize(struct instruction *instructions,
8271 struct instruction_data *instruction_data,
8272 uint32_t n_instructions)
8276 for (i = 0; i < n_instructions; ) {
8277 struct instruction *instr = &instructions[i];
8278 struct instruction_data *data = &instruction_data[i];
8279 uint32_t n_instr = 0;
8282 /* Emit many + TX. */
8283 detected = instr_pattern_emit_many_tx_search(instr,
8288 instr_pattern_emit_many_tx_replace(instr,
8295 /* No pattern starting at the current instruction. */
8299 /* Eliminate the invalid instructions that have been optimized out. */
8300 n_instructions = instr_compact(instructions,
8304 return n_instructions;
8308 instr_pattern_dma_many_search(struct instruction *instr,
8309 struct instruction_data *data,
8311 uint32_t *n_pattern_instr)
8315 for (i = 0; i < n_instr; i++) {
8316 if (data[i].invalid)
8319 if (instr[i].type != INSTR_DMA_HT)
8322 if (i == RTE_DIM(instr->dma.dst.header_id))
8325 if (i && data[i].n_users)
8332 *n_pattern_instr = i;
8337 instr_pattern_dma_many_replace(struct instruction *instr,
8338 struct instruction_data *data,
8343 for (i = 1; i < n_instr; i++) {
8345 instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
8346 instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
8347 instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
8348 instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
8350 data[i].invalid = 1;
8355 instr_pattern_dma_many_optimize(struct instruction *instructions,
8356 struct instruction_data *instruction_data,
8357 uint32_t n_instructions)
8361 for (i = 0; i < n_instructions; ) {
8362 struct instruction *instr = &instructions[i];
8363 struct instruction_data *data = &instruction_data[i];
8364 uint32_t n_instr = 0;
8368 detected = instr_pattern_dma_many_search(instr,
8373 instr_pattern_dma_many_replace(instr, data, n_instr);
8378 /* No pattern starting at the current instruction. */
8382 /* Eliminate the invalid instructions that have been optimized out. */
8383 n_instructions = instr_compact(instructions,
8387 return n_instructions;
8391 instr_optimize(struct instruction *instructions,
8392 struct instruction_data *instruction_data,
8393 uint32_t n_instructions)
8396 n_instructions = instr_pattern_extract_many_optimize(instructions,
8400 /* Emit many + TX. */
8401 n_instructions = instr_pattern_emit_many_tx_optimize(instructions,
8406 n_instructions = instr_pattern_dma_many_optimize(instructions,
8410 return n_instructions;
8414 instruction_config(struct rte_swx_pipeline *p,
8416 const char **instructions,
8417 uint32_t n_instructions)
8419 struct instruction *instr = NULL;
8420 struct instruction_data *data = NULL;
8424 CHECK(n_instructions, EINVAL);
8425 CHECK(instructions, EINVAL);
8426 for (i = 0; i < n_instructions; i++)
8427 CHECK_INSTRUCTION(instructions[i], EINVAL);
8429 /* Memory allocation. */
8430 instr = calloc(n_instructions, sizeof(struct instruction));
8436 data = calloc(n_instructions, sizeof(struct instruction_data));
8442 for (i = 0; i < n_instructions; i++) {
8443 char *string = strdup(instructions[i]);
8449 err = instr_translate(p, a, string, &instr[i], &data[i]);
8458 err = instr_label_check(data, n_instructions);
8462 err = instr_verify(p, a, instr, data, n_instructions);
8466 n_instructions = instr_optimize(instr, data, n_instructions);
8468 err = instr_jmp_resolve(instr, data, n_instructions);
8473 a->instructions = instr;
8474 a->n_instructions = n_instructions;
8476 p->instructions = instr;
8477 p->n_instructions = n_instructions;
8489 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
8491 static instr_exec_t instruction_table[] = {
8492 [INSTR_RX] = instr_rx_exec,
8493 [INSTR_TX] = instr_tx_exec,
8494 [INSTR_TX_I] = instr_tx_i_exec,
8496 [INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
8497 [INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
8498 [INSTR_HDR_EXTRACT3] = instr_hdr_extract3_exec,
8499 [INSTR_HDR_EXTRACT4] = instr_hdr_extract4_exec,
8500 [INSTR_HDR_EXTRACT5] = instr_hdr_extract5_exec,
8501 [INSTR_HDR_EXTRACT6] = instr_hdr_extract6_exec,
8502 [INSTR_HDR_EXTRACT7] = instr_hdr_extract7_exec,
8503 [INSTR_HDR_EXTRACT8] = instr_hdr_extract8_exec,
8505 [INSTR_HDR_EMIT] = instr_hdr_emit_exec,
8506 [INSTR_HDR_EMIT_TX] = instr_hdr_emit_tx_exec,
8507 [INSTR_HDR_EMIT2_TX] = instr_hdr_emit2_tx_exec,
8508 [INSTR_HDR_EMIT3_TX] = instr_hdr_emit3_tx_exec,
8509 [INSTR_HDR_EMIT4_TX] = instr_hdr_emit4_tx_exec,
8510 [INSTR_HDR_EMIT5_TX] = instr_hdr_emit5_tx_exec,
8511 [INSTR_HDR_EMIT6_TX] = instr_hdr_emit6_tx_exec,
8512 [INSTR_HDR_EMIT7_TX] = instr_hdr_emit7_tx_exec,
8513 [INSTR_HDR_EMIT8_TX] = instr_hdr_emit8_tx_exec,
8515 [INSTR_HDR_VALIDATE] = instr_hdr_validate_exec,
8516 [INSTR_HDR_INVALIDATE] = instr_hdr_invalidate_exec,
8518 [INSTR_MOV] = instr_mov_exec,
8519 [INSTR_MOV_MH] = instr_mov_mh_exec,
8520 [INSTR_MOV_HM] = instr_mov_hm_exec,
8521 [INSTR_MOV_HH] = instr_mov_hh_exec,
8522 [INSTR_MOV_I] = instr_mov_i_exec,
8524 [INSTR_DMA_HT] = instr_dma_ht_exec,
8525 [INSTR_DMA_HT2] = instr_dma_ht2_exec,
8526 [INSTR_DMA_HT3] = instr_dma_ht3_exec,
8527 [INSTR_DMA_HT4] = instr_dma_ht4_exec,
8528 [INSTR_DMA_HT5] = instr_dma_ht5_exec,
8529 [INSTR_DMA_HT6] = instr_dma_ht6_exec,
8530 [INSTR_DMA_HT7] = instr_dma_ht7_exec,
8531 [INSTR_DMA_HT8] = instr_dma_ht8_exec,
8533 [INSTR_ALU_ADD] = instr_alu_add_exec,
8534 [INSTR_ALU_ADD_MH] = instr_alu_add_mh_exec,
8535 [INSTR_ALU_ADD_HM] = instr_alu_add_hm_exec,
8536 [INSTR_ALU_ADD_HH] = instr_alu_add_hh_exec,
8537 [INSTR_ALU_ADD_MI] = instr_alu_add_mi_exec,
8538 [INSTR_ALU_ADD_HI] = instr_alu_add_hi_exec,
8540 [INSTR_ALU_SUB] = instr_alu_sub_exec,
8541 [INSTR_ALU_SUB_MH] = instr_alu_sub_mh_exec,
8542 [INSTR_ALU_SUB_HM] = instr_alu_sub_hm_exec,
8543 [INSTR_ALU_SUB_HH] = instr_alu_sub_hh_exec,
8544 [INSTR_ALU_SUB_MI] = instr_alu_sub_mi_exec,
8545 [INSTR_ALU_SUB_HI] = instr_alu_sub_hi_exec,
8547 [INSTR_ALU_CKADD_FIELD] = instr_alu_ckadd_field_exec,
8548 [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec,
8549 [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec,
8550 [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec,
8552 [INSTR_ALU_AND] = instr_alu_and_exec,
8553 [INSTR_ALU_AND_MH] = instr_alu_and_mh_exec,
8554 [INSTR_ALU_AND_HM] = instr_alu_and_hm_exec,
8555 [INSTR_ALU_AND_HH] = instr_alu_and_hh_exec,
8556 [INSTR_ALU_AND_I] = instr_alu_and_i_exec,
8558 [INSTR_ALU_OR] = instr_alu_or_exec,
8559 [INSTR_ALU_OR_MH] = instr_alu_or_mh_exec,
8560 [INSTR_ALU_OR_HM] = instr_alu_or_hm_exec,
8561 [INSTR_ALU_OR_HH] = instr_alu_or_hh_exec,
8562 [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
8564 [INSTR_ALU_XOR] = instr_alu_xor_exec,
8565 [INSTR_ALU_XOR_MH] = instr_alu_xor_mh_exec,
8566 [INSTR_ALU_XOR_HM] = instr_alu_xor_hm_exec,
8567 [INSTR_ALU_XOR_HH] = instr_alu_xor_hh_exec,
8568 [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
8570 [INSTR_ALU_SHL] = instr_alu_shl_exec,
8571 [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
8572 [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
8573 [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
8574 [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
8575 [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
8577 [INSTR_ALU_SHR] = instr_alu_shr_exec,
8578 [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
8579 [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
8580 [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
8581 [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
8582 [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
8584 [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec,
8585 [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec,
8586 [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec,
8588 [INSTR_REGRD_HRH] = instr_regrd_hrh_exec,
8589 [INSTR_REGRD_HRM] = instr_regrd_hrm_exec,
8590 [INSTR_REGRD_MRH] = instr_regrd_mrh_exec,
8591 [INSTR_REGRD_MRM] = instr_regrd_mrm_exec,
8592 [INSTR_REGRD_HRI] = instr_regrd_hri_exec,
8593 [INSTR_REGRD_MRI] = instr_regrd_mri_exec,
8595 [INSTR_REGWR_RHH] = instr_regwr_rhh_exec,
8596 [INSTR_REGWR_RHM] = instr_regwr_rhm_exec,
8597 [INSTR_REGWR_RMH] = instr_regwr_rmh_exec,
8598 [INSTR_REGWR_RMM] = instr_regwr_rmm_exec,
8599 [INSTR_REGWR_RHI] = instr_regwr_rhi_exec,
8600 [INSTR_REGWR_RMI] = instr_regwr_rmi_exec,
8601 [INSTR_REGWR_RIH] = instr_regwr_rih_exec,
8602 [INSTR_REGWR_RIM] = instr_regwr_rim_exec,
8603 [INSTR_REGWR_RII] = instr_regwr_rii_exec,
8605 [INSTR_REGADD_RHH] = instr_regadd_rhh_exec,
8606 [INSTR_REGADD_RHM] = instr_regadd_rhm_exec,
8607 [INSTR_REGADD_RMH] = instr_regadd_rmh_exec,
8608 [INSTR_REGADD_RMM] = instr_regadd_rmm_exec,
8609 [INSTR_REGADD_RHI] = instr_regadd_rhi_exec,
8610 [INSTR_REGADD_RMI] = instr_regadd_rmi_exec,
8611 [INSTR_REGADD_RIH] = instr_regadd_rih_exec,
8612 [INSTR_REGADD_RIM] = instr_regadd_rim_exec,
8613 [INSTR_REGADD_RII] = instr_regadd_rii_exec,
8615 [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
8616 [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
8617 [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
8619 [INSTR_METER_HHM] = instr_meter_hhm_exec,
8620 [INSTR_METER_HHI] = instr_meter_hhi_exec,
8621 [INSTR_METER_HMM] = instr_meter_hmm_exec,
8622 [INSTR_METER_HMI] = instr_meter_hmi_exec,
8623 [INSTR_METER_MHM] = instr_meter_mhm_exec,
8624 [INSTR_METER_MHI] = instr_meter_mhi_exec,
8625 [INSTR_METER_MMM] = instr_meter_mmm_exec,
8626 [INSTR_METER_MMI] = instr_meter_mmi_exec,
8627 [INSTR_METER_IHM] = instr_meter_ihm_exec,
8628 [INSTR_METER_IHI] = instr_meter_ihi_exec,
8629 [INSTR_METER_IMM] = instr_meter_imm_exec,
8630 [INSTR_METER_IMI] = instr_meter_imi_exec,
8632 [INSTR_TABLE] = instr_table_exec,
8633 [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
8634 [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
8636 [INSTR_JMP] = instr_jmp_exec,
8637 [INSTR_JMP_VALID] = instr_jmp_valid_exec,
8638 [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
8639 [INSTR_JMP_HIT] = instr_jmp_hit_exec,
8640 [INSTR_JMP_MISS] = instr_jmp_miss_exec,
8641 [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
8642 [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
8644 [INSTR_JMP_EQ] = instr_jmp_eq_exec,
8645 [INSTR_JMP_EQ_MH] = instr_jmp_eq_mh_exec,
8646 [INSTR_JMP_EQ_HM] = instr_jmp_eq_hm_exec,
8647 [INSTR_JMP_EQ_HH] = instr_jmp_eq_hh_exec,
8648 [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
8650 [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
8651 [INSTR_JMP_NEQ_MH] = instr_jmp_neq_mh_exec,
8652 [INSTR_JMP_NEQ_HM] = instr_jmp_neq_hm_exec,
8653 [INSTR_JMP_NEQ_HH] = instr_jmp_neq_hh_exec,
8654 [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
8656 [INSTR_JMP_LT] = instr_jmp_lt_exec,
8657 [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
8658 [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
8659 [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
8660 [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
8661 [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
8663 [INSTR_JMP_GT] = instr_jmp_gt_exec,
8664 [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
8665 [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
8666 [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
8667 [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
8668 [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
8670 [INSTR_RETURN] = instr_return_exec,
8674 instr_exec(struct rte_swx_pipeline *p)
8676 struct thread *t = &p->threads[p->thread_id];
8677 struct instruction *ip = t->ip;
8678 instr_exec_t instr = instruction_table[ip->type];
8686 static struct action *
8687 action_find(struct rte_swx_pipeline *p, const char *name)
8689 struct action *elem;
8694 TAILQ_FOREACH(elem, &p->actions, node)
8695 if (strcmp(elem->name, name) == 0)
8701 static struct action *
8702 action_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8704 struct action *action = NULL;
8706 TAILQ_FOREACH(action, &p->actions, node)
8707 if (action->id == id)
8713 static struct field *
8714 action_field_find(struct action *a, const char *name)
8716 return a->st ? struct_type_field_find(a->st, name) : NULL;
8719 static struct field *
8720 action_field_parse(struct action *action, const char *name)
8722 if (name[0] != 't' || name[1] != '.')
8725 return action_field_find(action, &name[2]);
8729 rte_swx_pipeline_action_config(struct rte_swx_pipeline *p,
8731 const char *args_struct_type_name,
8732 const char **instructions,
8733 uint32_t n_instructions)
8735 struct struct_type *args_struct_type;
8741 CHECK_NAME(name, EINVAL);
8742 CHECK(!action_find(p, name), EEXIST);
8744 if (args_struct_type_name) {
8745 CHECK_NAME(args_struct_type_name, EINVAL);
8746 args_struct_type = struct_type_find(p, args_struct_type_name);
8747 CHECK(args_struct_type, EINVAL);
8749 args_struct_type = NULL;
8752 /* Node allocation. */
8753 a = calloc(1, sizeof(struct action));
8756 /* Node initialization. */
8757 strcpy(a->name, name);
8758 a->st = args_struct_type;
8759 a->id = p->n_actions;
8761 /* Instruction translation. */
8762 err = instruction_config(p, a, instructions, n_instructions);
8768 /* Node add to tailq. */
8769 TAILQ_INSERT_TAIL(&p->actions, a, node);
8776 action_build(struct rte_swx_pipeline *p)
8778 struct action *action;
8780 p->action_instructions = calloc(p->n_actions,
8781 sizeof(struct instruction *));
8782 CHECK(p->action_instructions, ENOMEM);
8784 TAILQ_FOREACH(action, &p->actions, node)
8785 p->action_instructions[action->id] = action->instructions;
8791 action_build_free(struct rte_swx_pipeline *p)
8793 free(p->action_instructions);
8794 p->action_instructions = NULL;
8798 action_free(struct rte_swx_pipeline *p)
8800 action_build_free(p);
8803 struct action *action;
8805 action = TAILQ_FIRST(&p->actions);
8809 TAILQ_REMOVE(&p->actions, action, node);
8810 free(action->instructions);
8818 static struct table_type *
8819 table_type_find(struct rte_swx_pipeline *p, const char *name)
8821 struct table_type *elem;
8823 TAILQ_FOREACH(elem, &p->table_types, node)
8824 if (strcmp(elem->name, name) == 0)
8830 static struct table_type *
8831 table_type_resolve(struct rte_swx_pipeline *p,
8832 const char *recommended_type_name,
8833 enum rte_swx_table_match_type match_type)
8835 struct table_type *elem;
8837 /* Only consider the recommended type if the match type is correct. */
8838 if (recommended_type_name)
8839 TAILQ_FOREACH(elem, &p->table_types, node)
8840 if (!strcmp(elem->name, recommended_type_name) &&
8841 (elem->match_type == match_type))
8844 /* Ignore the recommended type and get the first element with this match
8847 TAILQ_FOREACH(elem, &p->table_types, node)
8848 if (elem->match_type == match_type)
8854 static struct table *
8855 table_find(struct rte_swx_pipeline *p, const char *name)
8859 TAILQ_FOREACH(elem, &p->tables, node)
8860 if (strcmp(elem->name, name) == 0)
8866 static struct table *
8867 table_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
8869 struct table *table = NULL;
8871 TAILQ_FOREACH(table, &p->tables, node)
8872 if (table->id == id)
8879 rte_swx_pipeline_table_type_register(struct rte_swx_pipeline *p,
8881 enum rte_swx_table_match_type match_type,
8882 struct rte_swx_table_ops *ops)
8884 struct table_type *elem;
8888 CHECK_NAME(name, EINVAL);
8889 CHECK(!table_type_find(p, name), EEXIST);
8892 CHECK(ops->create, EINVAL);
8893 CHECK(ops->lkp, EINVAL);
8894 CHECK(ops->free, EINVAL);
8896 /* Node allocation. */
8897 elem = calloc(1, sizeof(struct table_type));
8898 CHECK(elem, ENOMEM);
8900 /* Node initialization. */
8901 strcpy(elem->name, name);
8902 elem->match_type = match_type;
8903 memcpy(&elem->ops, ops, sizeof(*ops));
8905 /* Node add to tailq. */
8906 TAILQ_INSERT_TAIL(&p->table_types, elem, node);
8911 static enum rte_swx_table_match_type
8912 table_match_type_resolve(struct rte_swx_match_field_params *fields,
8914 uint32_t max_offset_field_id)
8916 uint32_t n_fields_em = 0, i;
8918 for (i = 0; i < n_fields; i++)
8919 if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
8922 if (n_fields_em == n_fields)
8923 return RTE_SWX_TABLE_MATCH_EXACT;
8925 if ((n_fields_em == n_fields - 1) &&
8926 (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
8927 return RTE_SWX_TABLE_MATCH_LPM;
8929 return RTE_SWX_TABLE_MATCH_WILDCARD;
8933 table_match_fields_check(struct rte_swx_pipeline *p,
8934 struct rte_swx_pipeline_table_params *params,
8935 struct header **header,
8936 uint32_t *min_offset_field_id,
8937 uint32_t *max_offset_field_id)
8939 struct header *h0 = NULL;
8940 struct field *hf, *mf;
8941 uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
8944 /* Return if no match fields. */
8945 if (!params->n_fields) {
8946 if (params->fields) {
8954 /* Memory allocation. */
8955 offset = calloc(params->n_fields, sizeof(uint32_t));
8961 /* Check that all the match fields belong to either the same header or
8964 hf = header_field_parse(p, params->fields[0].name, &h0);
8965 mf = metadata_field_parse(p, params->fields[0].name);
8971 offset[0] = h0 ? hf->offset : mf->offset;
8973 for (i = 1; i < params->n_fields; i++)
8977 hf = header_field_parse(p, params->fields[i].name, &h);
8978 if (!hf || (h->id != h0->id)) {
8983 offset[i] = hf->offset;
8985 mf = metadata_field_parse(p, params->fields[i].name);
8991 offset[i] = mf->offset;
8994 /* Check that there are no duplicated match fields. */
8995 for (i = 0; i < params->n_fields; i++) {
8998 for (j = 0; j < i; j++)
8999 if (offset[j] == offset[i]) {
9005 /* Find the min and max offset fields. */
9006 min_offset = offset[0];
9007 max_offset = offset[0];
9011 for (i = 1; i < params->n_fields; i++) {
9012 if (offset[i] < min_offset) {
9013 min_offset = offset[i];
9017 if (offset[i] > max_offset) {
9018 max_offset = offset[i];
9027 if (min_offset_field_id)
9028 *min_offset_field_id = min_offset_pos;
9030 if (max_offset_field_id)
9031 *max_offset_field_id = max_offset_pos;
9039 rte_swx_pipeline_table_config(struct rte_swx_pipeline *p,
9041 struct rte_swx_pipeline_table_params *params,
9042 const char *recommended_table_type_name,
9046 struct table_type *type;
9048 struct action *default_action;
9049 struct header *header = NULL;
9050 uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
9055 CHECK_NAME(name, EINVAL);
9056 CHECK(!table_find(p, name), EEXIST);
9058 CHECK(params, EINVAL);
9061 status = table_match_fields_check(p,
9064 &min_offset_field_id,
9065 &max_offset_field_id);
9069 /* Action checks. */
9070 CHECK(params->n_actions, EINVAL);
9071 CHECK(params->action_names, EINVAL);
9072 for (i = 0; i < params->n_actions; i++) {
9073 const char *action_name = params->action_names[i];
9075 uint32_t action_data_size;
9077 CHECK_NAME(action_name, EINVAL);
9079 a = action_find(p, action_name);
9082 action_data_size = a->st ? a->st->n_bits / 8 : 0;
9083 if (action_data_size > action_data_size_max)
9084 action_data_size_max = action_data_size;
9087 CHECK_NAME(params->default_action_name, EINVAL);
9088 for (i = 0; i < p->n_actions; i++)
9089 if (!strcmp(params->action_names[i],
9090 params->default_action_name))
9092 CHECK(i < params->n_actions, EINVAL);
9093 default_action = action_find(p, params->default_action_name);
9094 CHECK((default_action->st && params->default_action_data) ||
9095 !params->default_action_data, EINVAL);
9097 /* Table type checks. */
9098 if (recommended_table_type_name)
9099 CHECK_NAME(recommended_table_type_name, EINVAL);
9101 if (params->n_fields) {
9102 enum rte_swx_table_match_type match_type;
9104 match_type = table_match_type_resolve(params->fields,
9106 max_offset_field_id);
9107 type = table_type_resolve(p,
9108 recommended_table_type_name,
9110 CHECK(type, EINVAL);
9115 /* Memory allocation. */
9116 t = calloc(1, sizeof(struct table));
9119 t->fields = calloc(params->n_fields, sizeof(struct match_field));
9125 t->actions = calloc(params->n_actions, sizeof(struct action *));
9132 if (action_data_size_max) {
9133 t->default_action_data = calloc(1, action_data_size_max);
9134 if (!t->default_action_data) {
9142 /* Node initialization. */
9143 strcpy(t->name, name);
9144 if (args && args[0])
9145 strcpy(t->args, args);
9148 for (i = 0; i < params->n_fields; i++) {
9149 struct rte_swx_match_field_params *field = ¶ms->fields[i];
9150 struct match_field *f = &t->fields[i];
9152 f->match_type = field->match_type;
9154 header_field_parse(p, field->name, NULL) :
9155 metadata_field_parse(p, field->name);
9157 t->n_fields = params->n_fields;
9160 for (i = 0; i < params->n_actions; i++)
9161 t->actions[i] = action_find(p, params->action_names[i]);
9162 t->default_action = default_action;
9163 if (default_action->st)
9164 memcpy(t->default_action_data,
9165 params->default_action_data,
9166 default_action->st->n_bits / 8);
9167 t->n_actions = params->n_actions;
9168 t->default_action_is_const = params->default_action_is_const;
9169 t->action_data_size_max = action_data_size_max;
9172 t->id = p->n_tables;
9174 /* Node add to tailq. */
9175 TAILQ_INSERT_TAIL(&p->tables, t, node);
9181 static struct rte_swx_table_params *
9182 table_params_get(struct table *table)
9184 struct rte_swx_table_params *params;
9185 struct field *first, *last;
9187 uint32_t key_size, key_offset, action_data_size, i;
9189 /* Memory allocation. */
9190 params = calloc(1, sizeof(struct rte_swx_table_params));
9194 /* Find first (smallest offset) and last (biggest offset) match fields. */
9195 first = table->fields[0].field;
9196 last = table->fields[0].field;
9198 for (i = 0; i < table->n_fields; i++) {
9199 struct field *f = table->fields[i].field;
9201 if (f->offset < first->offset)
9204 if (f->offset > last->offset)
9208 /* Key offset and size. */
9209 key_offset = first->offset / 8;
9210 key_size = (last->offset + last->n_bits - first->offset) / 8;
9212 /* Memory allocation. */
9213 key_mask = calloc(1, key_size);
9220 for (i = 0; i < table->n_fields; i++) {
9221 struct field *f = table->fields[i].field;
9222 uint32_t start = (f->offset - first->offset) / 8;
9223 size_t size = f->n_bits / 8;
9225 memset(&key_mask[start], 0xFF, size);
9228 /* Action data size. */
9229 action_data_size = 0;
9230 for (i = 0; i < table->n_actions; i++) {
9231 struct action *action = table->actions[i];
9232 uint32_t ads = action->st ? action->st->n_bits / 8 : 0;
9234 if (ads > action_data_size)
9235 action_data_size = ads;
9239 params->match_type = table->type->match_type;
9240 params->key_size = key_size;
9241 params->key_offset = key_offset;
9242 params->key_mask0 = key_mask;
9243 params->action_data_size = action_data_size;
9244 params->n_keys_max = table->size;
9250 table_params_free(struct rte_swx_table_params *params)
9255 free(params->key_mask0);
9260 table_state_build(struct rte_swx_pipeline *p)
9262 struct table *table;
9264 p->table_state = calloc(p->n_tables,
9265 sizeof(struct rte_swx_table_state));
9266 CHECK(p->table_state, ENOMEM);
9268 TAILQ_FOREACH(table, &p->tables, node) {
9269 struct rte_swx_table_state *ts = &p->table_state[table->id];
9272 struct rte_swx_table_params *params;
9275 params = table_params_get(table);
9276 CHECK(params, ENOMEM);
9278 ts->obj = table->type->ops.create(params,
9283 table_params_free(params);
9284 CHECK(ts->obj, ENODEV);
9287 /* ts->default_action_data. */
9288 if (table->action_data_size_max) {
9289 ts->default_action_data =
9290 malloc(table->action_data_size_max);
9291 CHECK(ts->default_action_data, ENOMEM);
9293 memcpy(ts->default_action_data,
9294 table->default_action_data,
9295 table->action_data_size_max);
9298 /* ts->default_action_id. */
9299 ts->default_action_id = table->default_action->id;
9306 table_state_build_free(struct rte_swx_pipeline *p)
9310 if (!p->table_state)
9313 for (i = 0; i < p->n_tables; i++) {
9314 struct rte_swx_table_state *ts = &p->table_state[i];
9315 struct table *table = table_find_by_id(p, i);
9318 if (table->type && ts->obj)
9319 table->type->ops.free(ts->obj);
9321 /* ts->default_action_data. */
9322 free(ts->default_action_data);
9325 free(p->table_state);
9326 p->table_state = NULL;
9330 table_state_free(struct rte_swx_pipeline *p)
9332 table_state_build_free(p);
9336 table_stub_lkp(void *table __rte_unused,
9337 void *mailbox __rte_unused,
9338 uint8_t **key __rte_unused,
9339 uint64_t *action_id __rte_unused,
9340 uint8_t **action_data __rte_unused,
9344 return 1; /* DONE. */
9348 table_build(struct rte_swx_pipeline *p)
9352 /* Per pipeline: table statistics. */
9353 p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
9354 CHECK(p->table_stats, ENOMEM);
9356 for (i = 0; i < p->n_tables; i++) {
9357 p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
9358 CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
9361 /* Per thread: table runt-time. */
9362 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9363 struct thread *t = &p->threads[i];
9364 struct table *table;
9366 t->tables = calloc(p->n_tables, sizeof(struct table_runtime));
9367 CHECK(t->tables, ENOMEM);
9369 TAILQ_FOREACH(table, &p->tables, node) {
9370 struct table_runtime *r = &t->tables[table->id];
9375 size = table->type->ops.mailbox_size_get();
9378 r->func = table->type->ops.lkp;
9382 r->mailbox = calloc(1, size);
9383 CHECK(r->mailbox, ENOMEM);
9387 r->key = table->header ?
9388 &t->structs[table->header->struct_id] :
9389 &t->structs[p->metadata_struct_id];
9391 r->func = table_stub_lkp;
9400 table_build_free(struct rte_swx_pipeline *p)
9404 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9405 struct thread *t = &p->threads[i];
9411 for (j = 0; j < p->n_tables; j++) {
9412 struct table_runtime *r = &t->tables[j];
9421 if (p->table_stats) {
9422 for (i = 0; i < p->n_tables; i++)
9423 free(p->table_stats[i].n_pkts_action);
9425 free(p->table_stats);
9430 table_free(struct rte_swx_pipeline *p)
9432 table_build_free(p);
9438 elem = TAILQ_FIRST(&p->tables);
9442 TAILQ_REMOVE(&p->tables, elem, node);
9444 free(elem->actions);
9445 free(elem->default_action_data);
9451 struct table_type *elem;
9453 elem = TAILQ_FIRST(&p->table_types);
9457 TAILQ_REMOVE(&p->table_types, elem, node);
9465 static struct regarray *
9466 regarray_find(struct rte_swx_pipeline *p, const char *name)
9468 struct regarray *elem;
9470 TAILQ_FOREACH(elem, &p->regarrays, node)
9471 if (!strcmp(elem->name, name))
9477 static struct regarray *
9478 regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9480 struct regarray *elem = NULL;
9482 TAILQ_FOREACH(elem, &p->regarrays, node)
9490 rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p,
9499 CHECK_NAME(name, EINVAL);
9500 CHECK(!regarray_find(p, name), EEXIST);
9502 CHECK(size, EINVAL);
9503 size = rte_align32pow2(size);
9505 /* Memory allocation. */
9506 r = calloc(1, sizeof(struct regarray));
9509 /* Node initialization. */
9510 strcpy(r->name, name);
9511 r->init_val = init_val;
9513 r->id = p->n_regarrays;
9515 /* Node add to tailq. */
9516 TAILQ_INSERT_TAIL(&p->regarrays, r, node);
9523 regarray_build(struct rte_swx_pipeline *p)
9525 struct regarray *regarray;
9527 if (!p->n_regarrays)
9530 p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime));
9531 CHECK(p->regarray_runtime, ENOMEM);
9533 TAILQ_FOREACH(regarray, &p->regarrays, node) {
9534 struct regarray_runtime *r = &p->regarray_runtime[regarray->id];
9537 r->regarray = env_malloc(regarray->size * sizeof(uint64_t),
9538 RTE_CACHE_LINE_SIZE,
9540 CHECK(r->regarray, ENOMEM);
9542 if (regarray->init_val)
9543 for (i = 0; i < regarray->size; i++)
9544 r->regarray[i] = regarray->init_val;
9546 r->size_mask = regarray->size - 1;
9553 regarray_build_free(struct rte_swx_pipeline *p)
9557 if (!p->regarray_runtime)
9560 for (i = 0; i < p->n_regarrays; i++) {
9561 struct regarray *regarray = regarray_find_by_id(p, i);
9562 struct regarray_runtime *r = &p->regarray_runtime[i];
9564 env_free(r->regarray, regarray->size * sizeof(uint64_t));
9567 free(p->regarray_runtime);
9568 p->regarray_runtime = NULL;
9572 regarray_free(struct rte_swx_pipeline *p)
9574 regarray_build_free(p);
9577 struct regarray *elem;
9579 elem = TAILQ_FIRST(&p->regarrays);
9583 TAILQ_REMOVE(&p->regarrays, elem, node);
9591 static struct meter_profile *
9592 meter_profile_find(struct rte_swx_pipeline *p, const char *name)
9594 struct meter_profile *elem;
9596 TAILQ_FOREACH(elem, &p->meter_profiles, node)
9597 if (!strcmp(elem->name, name))
9603 static struct metarray *
9604 metarray_find(struct rte_swx_pipeline *p, const char *name)
9606 struct metarray *elem;
9608 TAILQ_FOREACH(elem, &p->metarrays, node)
9609 if (!strcmp(elem->name, name))
9615 static struct metarray *
9616 metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
9618 struct metarray *elem = NULL;
9620 TAILQ_FOREACH(elem, &p->metarrays, node)
9628 rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
9636 CHECK_NAME(name, EINVAL);
9637 CHECK(!metarray_find(p, name), EEXIST);
9639 CHECK(size, EINVAL);
9640 size = rte_align32pow2(size);
9642 /* Memory allocation. */
9643 m = calloc(1, sizeof(struct metarray));
9646 /* Node initialization. */
9647 strcpy(m->name, name);
9649 m->id = p->n_metarrays;
9651 /* Node add to tailq. */
9652 TAILQ_INSERT_TAIL(&p->metarrays, m, node);
9658 struct meter_profile meter_profile_default = {
9667 .cir_bytes_per_period = 1,
9669 .pir_bytes_per_period = 1,
9676 meter_init(struct meter *m)
9678 memset(m, 0, sizeof(struct meter));
9679 rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
9680 m->profile = &meter_profile_default;
9681 m->color_mask = RTE_COLOR_GREEN;
9683 meter_profile_default.n_users++;
9687 metarray_build(struct rte_swx_pipeline *p)
9691 if (!p->n_metarrays)
9694 p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
9695 CHECK(p->metarray_runtime, ENOMEM);
9697 TAILQ_FOREACH(m, &p->metarrays, node) {
9698 struct metarray_runtime *r = &p->metarray_runtime[m->id];
9701 r->metarray = env_malloc(m->size * sizeof(struct meter),
9702 RTE_CACHE_LINE_SIZE,
9704 CHECK(r->metarray, ENOMEM);
9706 for (i = 0; i < m->size; i++)
9707 meter_init(&r->metarray[i]);
9709 r->size_mask = m->size - 1;
9716 metarray_build_free(struct rte_swx_pipeline *p)
9720 if (!p->metarray_runtime)
9723 for (i = 0; i < p->n_metarrays; i++) {
9724 struct metarray *m = metarray_find_by_id(p, i);
9725 struct metarray_runtime *r = &p->metarray_runtime[i];
9727 env_free(r->metarray, m->size * sizeof(struct meter));
9730 free(p->metarray_runtime);
9731 p->metarray_runtime = NULL;
9735 metarray_free(struct rte_swx_pipeline *p)
9737 metarray_build_free(p);
9741 struct metarray *elem;
9743 elem = TAILQ_FIRST(&p->metarrays);
9747 TAILQ_REMOVE(&p->metarrays, elem, node);
9751 /* Meter profiles. */
9753 struct meter_profile *elem;
9755 elem = TAILQ_FIRST(&p->meter_profiles);
9759 TAILQ_REMOVE(&p->meter_profiles, elem, node);
9768 rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node)
9770 struct rte_swx_pipeline *pipeline;
9772 /* Check input parameters. */
9775 /* Memory allocation. */
9776 pipeline = calloc(1, sizeof(struct rte_swx_pipeline));
9777 CHECK(pipeline, ENOMEM);
9779 /* Initialization. */
9780 TAILQ_INIT(&pipeline->struct_types);
9781 TAILQ_INIT(&pipeline->port_in_types);
9782 TAILQ_INIT(&pipeline->ports_in);
9783 TAILQ_INIT(&pipeline->port_out_types);
9784 TAILQ_INIT(&pipeline->ports_out);
9785 TAILQ_INIT(&pipeline->extern_types);
9786 TAILQ_INIT(&pipeline->extern_objs);
9787 TAILQ_INIT(&pipeline->extern_funcs);
9788 TAILQ_INIT(&pipeline->headers);
9789 TAILQ_INIT(&pipeline->actions);
9790 TAILQ_INIT(&pipeline->table_types);
9791 TAILQ_INIT(&pipeline->tables);
9792 TAILQ_INIT(&pipeline->regarrays);
9793 TAILQ_INIT(&pipeline->meter_profiles);
9794 TAILQ_INIT(&pipeline->metarrays);
9796 pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
9797 pipeline->numa_node = numa_node;
9804 rte_swx_pipeline_free(struct rte_swx_pipeline *p)
9809 free(p->instructions);
9813 table_state_free(p);
9818 extern_func_free(p);
9828 rte_swx_pipeline_instructions_config(struct rte_swx_pipeline *p,
9829 const char **instructions,
9830 uint32_t n_instructions)
9835 err = instruction_config(p, NULL, instructions, n_instructions);
9839 /* Thread instruction pointer reset. */
9840 for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
9841 struct thread *t = &p->threads[i];
9843 thread_ip_reset(p, t);
9850 rte_swx_pipeline_build(struct rte_swx_pipeline *p)
9855 CHECK(p->build_done == 0, EEXIST);
9857 status = port_in_build(p);
9861 status = port_out_build(p);
9865 status = struct_build(p);
9869 status = extern_obj_build(p);
9873 status = extern_func_build(p);
9877 status = header_build(p);
9881 status = metadata_build(p);
9885 status = action_build(p);
9889 status = table_build(p);
9893 status = table_state_build(p);
9897 status = regarray_build(p);
9901 status = metarray_build(p);
9909 metarray_build_free(p);
9910 regarray_build_free(p);
9911 table_state_build_free(p);
9912 table_build_free(p);
9913 action_build_free(p);
9914 metadata_build_free(p);
9915 header_build_free(p);
9916 extern_func_build_free(p);
9917 extern_obj_build_free(p);
9918 port_out_build_free(p);
9919 port_in_build_free(p);
9920 struct_build_free(p);
9926 rte_swx_pipeline_run(struct rte_swx_pipeline *p, uint32_t n_instructions)
9930 for (i = 0; i < n_instructions; i++)
9935 rte_swx_pipeline_flush(struct rte_swx_pipeline *p)
9939 for (i = 0; i < p->n_ports_out; i++) {
9940 struct port_out_runtime *port = &p->out[i];
9943 port->flush(port->obj);
9951 rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
9952 struct rte_swx_ctl_pipeline_info *pipeline)
9954 struct action *action;
9955 struct table *table;
9956 uint32_t n_actions = 0, n_tables = 0;
9958 if (!p || !pipeline)
9961 TAILQ_FOREACH(action, &p->actions, node)
9964 TAILQ_FOREACH(table, &p->tables, node)
9967 pipeline->n_ports_in = p->n_ports_in;
9968 pipeline->n_ports_out = p->n_ports_out;
9969 pipeline->n_actions = n_actions;
9970 pipeline->n_tables = n_tables;
9971 pipeline->n_regarrays = p->n_regarrays;
9972 pipeline->n_metarrays = p->n_metarrays;
9978 rte_swx_ctl_pipeline_numa_node_get(struct rte_swx_pipeline *p, int *numa_node)
9980 if (!p || !numa_node)
9983 *numa_node = p->numa_node;
9988 rte_swx_ctl_action_info_get(struct rte_swx_pipeline *p,
9990 struct rte_swx_ctl_action_info *action)
9992 struct action *a = NULL;
9994 if (!p || (action_id >= p->n_actions) || !action)
9997 a = action_find_by_id(p, action_id);
10001 strcpy(action->name, a->name);
10002 action->n_args = a->st ? a->st->n_fields : 0;
10007 rte_swx_ctl_action_arg_info_get(struct rte_swx_pipeline *p,
10008 uint32_t action_id,
10009 uint32_t action_arg_id,
10010 struct rte_swx_ctl_action_arg_info *action_arg)
10012 struct action *a = NULL;
10013 struct field *arg = NULL;
10015 if (!p || (action_id >= p->n_actions) || !action_arg)
10018 a = action_find_by_id(p, action_id);
10019 if (!a || !a->st || (action_arg_id >= a->st->n_fields))
10022 arg = &a->st->fields[action_arg_id];
10023 strcpy(action_arg->name, arg->name);
10024 action_arg->n_bits = arg->n_bits;
10030 rte_swx_ctl_table_info_get(struct rte_swx_pipeline *p,
10032 struct rte_swx_ctl_table_info *table)
10034 struct table *t = NULL;
10039 t = table_find_by_id(p, table_id);
10043 strcpy(table->name, t->name);
10044 strcpy(table->args, t->args);
10045 table->n_match_fields = t->n_fields;
10046 table->n_actions = t->n_actions;
10047 table->default_action_is_const = t->default_action_is_const;
10048 table->size = t->size;
10053 rte_swx_ctl_table_match_field_info_get(struct rte_swx_pipeline *p,
10055 uint32_t match_field_id,
10056 struct rte_swx_ctl_table_match_field_info *match_field)
10059 struct match_field *f;
10061 if (!p || (table_id >= p->n_tables) || !match_field)
10064 t = table_find_by_id(p, table_id);
10065 if (!t || (match_field_id >= t->n_fields))
10068 f = &t->fields[match_field_id];
10069 match_field->match_type = f->match_type;
10070 match_field->is_header = t->header ? 1 : 0;
10071 match_field->n_bits = f->field->n_bits;
10072 match_field->offset = f->field->offset;
10078 rte_swx_ctl_table_action_info_get(struct rte_swx_pipeline *p,
10080 uint32_t table_action_id,
10081 struct rte_swx_ctl_table_action_info *table_action)
10085 if (!p || (table_id >= p->n_tables) || !table_action)
10088 t = table_find_by_id(p, table_id);
10089 if (!t || (table_action_id >= t->n_actions))
10092 table_action->action_id = t->actions[table_action_id]->id;
10098 rte_swx_ctl_table_ops_get(struct rte_swx_pipeline *p,
10100 struct rte_swx_table_ops *table_ops,
10105 if (!p || (table_id >= p->n_tables))
10108 t = table_find_by_id(p, table_id);
10114 memcpy(table_ops, &t->type->ops, sizeof(*table_ops));
10124 rte_swx_pipeline_table_state_get(struct rte_swx_pipeline *p,
10125 struct rte_swx_table_state **table_state)
10127 if (!p || !table_state || !p->build_done)
10130 *table_state = p->table_state;
10135 rte_swx_pipeline_table_state_set(struct rte_swx_pipeline *p,
10136 struct rte_swx_table_state *table_state)
10138 if (!p || !table_state || !p->build_done)
10141 p->table_state = table_state;
10146 rte_swx_ctl_pipeline_port_in_stats_read(struct rte_swx_pipeline *p,
10148 struct rte_swx_port_in_stats *stats)
10150 struct port_in *port;
10155 port = port_in_find(p, port_id);
10159 port->type->ops.stats_read(port->obj, stats);
10164 rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p,
10166 struct rte_swx_port_out_stats *stats)
10168 struct port_out *port;
10173 port = port_out_find(p, port_id);
10177 port->type->ops.stats_read(port->obj, stats);
10182 rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
10183 const char *table_name,
10184 struct rte_swx_table_stats *stats)
10186 struct table *table;
10187 struct table_statistics *table_stats;
10189 if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
10192 table = table_find(p, table_name);
10196 table_stats = &p->table_stats[table->id];
10198 memcpy(&stats->n_pkts_action,
10199 &table_stats->n_pkts_action,
10200 p->n_actions * sizeof(uint64_t));
10202 stats->n_pkts_hit = table_stats->n_pkts_hit[1];
10203 stats->n_pkts_miss = table_stats->n_pkts_hit[0];
10209 rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
10210 uint32_t regarray_id,
10211 struct rte_swx_ctl_regarray_info *regarray)
10213 struct regarray *r;
10215 if (!p || !regarray)
10218 r = regarray_find_by_id(p, regarray_id);
10222 strcpy(regarray->name, r->name);
10223 regarray->size = r->size;
10228 rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p,
10229 const char *regarray_name,
10230 uint32_t regarray_index,
10233 struct regarray *regarray;
10234 struct regarray_runtime *r;
10236 if (!p || !regarray_name || !value)
10239 regarray = regarray_find(p, regarray_name);
10240 if (!regarray || (regarray_index >= regarray->size))
10243 r = &p->regarray_runtime[regarray->id];
10244 *value = r->regarray[regarray_index];
10249 rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p,
10250 const char *regarray_name,
10251 uint32_t regarray_index,
10254 struct regarray *regarray;
10255 struct regarray_runtime *r;
10257 if (!p || !regarray_name)
10260 regarray = regarray_find(p, regarray_name);
10261 if (!regarray || (regarray_index >= regarray->size))
10264 r = &p->regarray_runtime[regarray->id];
10265 r->regarray[regarray_index] = value;
10270 rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
10271 uint32_t metarray_id,
10272 struct rte_swx_ctl_metarray_info *metarray)
10274 struct metarray *m;
10276 if (!p || !metarray)
10279 m = metarray_find_by_id(p, metarray_id);
10283 strcpy(metarray->name, m->name);
10284 metarray->size = m->size;
10289 rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
10291 struct rte_meter_trtcm_params *params)
10293 struct meter_profile *mp;
10297 CHECK_NAME(name, EINVAL);
10298 CHECK(params, EINVAL);
10299 CHECK(!meter_profile_find(p, name), EEXIST);
10301 /* Node allocation. */
10302 mp = calloc(1, sizeof(struct meter_profile));
10305 /* Node initialization. */
10306 strcpy(mp->name, name);
10307 memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
10308 status = rte_meter_trtcm_profile_config(&mp->profile, params);
10314 /* Node add to tailq. */
10315 TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
10321 rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
10324 struct meter_profile *mp;
10327 CHECK_NAME(name, EINVAL);
10329 mp = meter_profile_find(p, name);
10331 CHECK(!mp->n_users, EBUSY);
10333 /* Remove node from tailq. */
10334 TAILQ_REMOVE(&p->meter_profiles, mp, node);
10341 rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
10342 const char *metarray_name,
10343 uint32_t metarray_index)
10345 struct meter_profile *mp_old;
10346 struct metarray *metarray;
10347 struct metarray_runtime *metarray_runtime;
10351 CHECK_NAME(metarray_name, EINVAL);
10353 metarray = metarray_find(p, metarray_name);
10354 CHECK(metarray, EINVAL);
10355 CHECK(metarray_index < metarray->size, EINVAL);
10357 metarray_runtime = &p->metarray_runtime[metarray->id];
10358 m = &metarray_runtime->metarray[metarray_index];
10359 mp_old = m->profile;
10369 rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
10370 const char *metarray_name,
10371 uint32_t metarray_index,
10372 const char *profile_name)
10374 struct meter_profile *mp, *mp_old;
10375 struct metarray *metarray;
10376 struct metarray_runtime *metarray_runtime;
10380 CHECK_NAME(metarray_name, EINVAL);
10382 metarray = metarray_find(p, metarray_name);
10383 CHECK(metarray, EINVAL);
10384 CHECK(metarray_index < metarray->size, EINVAL);
10386 mp = meter_profile_find(p, profile_name);
10389 metarray_runtime = &p->metarray_runtime[metarray->id];
10390 m = &metarray_runtime->metarray[metarray_index];
10391 mp_old = m->profile;
10393 memset(m, 0, sizeof(struct meter));
10394 rte_meter_trtcm_config(&m->m, &mp->profile);
10396 m->color_mask = RTE_COLORS;
10405 rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
10406 const char *metarray_name,
10407 uint32_t metarray_index,
10408 struct rte_swx_ctl_meter_stats *stats)
10410 struct metarray *metarray;
10411 struct metarray_runtime *metarray_runtime;
10415 CHECK_NAME(metarray_name, EINVAL);
10417 metarray = metarray_find(p, metarray_name);
10418 CHECK(metarray, EINVAL);
10419 CHECK(metarray_index < metarray->size, EINVAL);
10421 CHECK(stats, EINVAL);
10423 metarray_runtime = &p->metarray_runtime[metarray->id];
10424 m = &metarray_runtime->metarray[metarray_index];
10426 memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
10427 memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));