1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
11 #include <rte_byteorder.h>
12 #include <rte_common.h>
13 #include <rte_cycles.h>
14 #include <rte_prefetch.h>
15 #include <rte_meter.h>
17 #include <rte_swx_table_selector.h>
18 #include <rte_swx_table_learner.h>
19 #include <rte_swx_pipeline.h>
20 #include <rte_swx_ctl.h>
27 #define TRACE(...) printf(__VA_ARGS__)
35 #define ntoh64(x) rte_be_to_cpu_64(x)
36 #define hton64(x) rte_cpu_to_be_64(x)
42 char name[RTE_SWX_NAME_SIZE];
49 TAILQ_ENTRY(struct_type) node;
50 char name[RTE_SWX_NAME_SIZE];
58 TAILQ_HEAD(struct_type_tailq, struct_type);
64 TAILQ_ENTRY(port_in_type) node;
65 char name[RTE_SWX_NAME_SIZE];
66 struct rte_swx_port_in_ops ops;
69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
72 TAILQ_ENTRY(port_in) node;
73 struct port_in_type *type;
78 TAILQ_HEAD(port_in_tailq, port_in);
80 struct port_in_runtime {
81 rte_swx_port_in_pkt_rx_t pkt_rx;
88 struct port_out_type {
89 TAILQ_ENTRY(port_out_type) node;
90 char name[RTE_SWX_NAME_SIZE];
91 struct rte_swx_port_out_ops ops;
94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
97 TAILQ_ENTRY(port_out) node;
98 struct port_out_type *type;
103 TAILQ_HEAD(port_out_tailq, port_out);
105 struct port_out_runtime {
106 rte_swx_port_out_pkt_tx_t pkt_tx;
107 rte_swx_port_out_flush_t flush;
114 struct extern_type_member_func {
115 TAILQ_ENTRY(extern_type_member_func) node;
116 char name[RTE_SWX_NAME_SIZE];
117 rte_swx_extern_type_member_func_t func;
121 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
124 TAILQ_ENTRY(extern_type) node;
125 char name[RTE_SWX_NAME_SIZE];
126 struct struct_type *mailbox_struct_type;
127 rte_swx_extern_type_constructor_t constructor;
128 rte_swx_extern_type_destructor_t destructor;
129 struct extern_type_member_func_tailq funcs;
133 TAILQ_HEAD(extern_type_tailq, extern_type);
136 TAILQ_ENTRY(extern_obj) node;
137 char name[RTE_SWX_NAME_SIZE];
138 struct extern_type *type;
144 TAILQ_HEAD(extern_obj_tailq, extern_obj);
146 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
147 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
150 struct extern_obj_runtime {
153 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
160 TAILQ_ENTRY(extern_func) node;
161 char name[RTE_SWX_NAME_SIZE];
162 struct struct_type *mailbox_struct_type;
163 rte_swx_extern_func_t func;
168 TAILQ_HEAD(extern_func_tailq, extern_func);
170 struct extern_func_runtime {
172 rte_swx_extern_func_t func;
179 TAILQ_ENTRY(header) node;
180 char name[RTE_SWX_NAME_SIZE];
181 struct struct_type *st;
186 TAILQ_HEAD(header_tailq, header);
188 struct header_runtime {
193 struct header_out_runtime {
203 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
204 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
205 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
206 * when transferred to packet meta-data and in NBO when transferred to packet
210 /* Notation conventions:
211 * -Header field: H = h.header.field (dst/src)
212 * -Meta-data field: M = m.field (dst/src)
213 * -Extern object mailbox field: E = e.field (dst/src)
214 * -Extern function mailbox field: F = f.field (dst/src)
215 * -Table action data field: T = t.field (src only)
216 * -Immediate value: I = 32-bit unsigned value (src only)
219 enum instruction_type {
226 INSTR_TX, /* port_out = M */
227 INSTR_TX_I, /* port_out = I */
229 /* extract h.header */
239 /* extract h.header m.last_field_size */
242 /* lookahead h.header */
256 /* validate h.header */
259 /* invalidate h.header */
260 INSTR_HDR_INVALIDATE,
264 * dst = HMEF, src = HMEFTI
266 INSTR_MOV, /* dst = MEF, src = MEFT */
267 INSTR_MOV_MH, /* dst = MEF, src = H */
268 INSTR_MOV_HM, /* dst = H, src = MEFT */
269 INSTR_MOV_HH, /* dst = H, src = H */
270 INSTR_MOV_I, /* dst = HMEF, src = I */
272 /* dma h.header t.field
273 * memcpy(h.header, t.field, sizeof(h.header))
286 * dst = HMEF, src = HMEFTI
288 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
289 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
290 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
291 INSTR_ALU_ADD_HH, /* dst = H, src = H */
292 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
293 INSTR_ALU_ADD_HI, /* dst = H, src = I */
297 * dst = HMEF, src = HMEFTI
299 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
300 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
301 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
302 INSTR_ALU_SUB_HH, /* dst = H, src = H */
303 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
304 INSTR_ALU_SUB_HI, /* dst = H, src = I */
307 * dst = dst '+ src[0:1] '+ src[2:3] + ...
308 * dst = H, src = {H, h.header}
310 INSTR_ALU_CKADD_FIELD, /* src = H */
311 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
312 INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
318 INSTR_ALU_CKSUB_FIELD,
322 * dst = HMEF, src = HMEFTI
324 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
325 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
326 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
327 INSTR_ALU_AND_HH, /* dst = H, src = H */
328 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
332 * dst = HMEF, src = HMEFTI
334 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
335 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
336 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
337 INSTR_ALU_OR_HH, /* dst = H, src = H */
338 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
342 * dst = HMEF, src = HMEFTI
344 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
345 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
346 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
347 INSTR_ALU_XOR_HH, /* dst = H, src = H */
348 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
352 * dst = HMEF, src = HMEFTI
354 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
355 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
356 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
357 INSTR_ALU_SHL_HH, /* dst = H, src = H */
358 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
359 INSTR_ALU_SHL_HI, /* dst = H, src = I */
363 * dst = HMEF, src = HMEFTI
365 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
366 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
367 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
368 INSTR_ALU_SHR_HH, /* dst = H, src = H */
369 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
370 INSTR_ALU_SHR_HI, /* dst = H, src = I */
372 /* regprefetch REGARRAY index
373 * prefetch REGARRAY[index]
376 INSTR_REGPREFETCH_RH, /* index = H */
377 INSTR_REGPREFETCH_RM, /* index = MEFT */
378 INSTR_REGPREFETCH_RI, /* index = I */
380 /* regrd dst REGARRAY index
381 * dst = REGARRAY[index]
382 * dst = HMEF, index = HMEFTI
384 INSTR_REGRD_HRH, /* dst = H, index = H */
385 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
386 INSTR_REGRD_HRI, /* dst = H, index = I */
387 INSTR_REGRD_MRH, /* dst = MEF, index = H */
388 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
389 INSTR_REGRD_MRI, /* dst = MEF, index = I */
391 /* regwr REGARRAY index src
392 * REGARRAY[index] = src
393 * index = HMEFTI, src = HMEFTI
395 INSTR_REGWR_RHH, /* index = H, src = H */
396 INSTR_REGWR_RHM, /* index = H, src = MEFT */
397 INSTR_REGWR_RHI, /* index = H, src = I */
398 INSTR_REGWR_RMH, /* index = MEFT, src = H */
399 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
400 INSTR_REGWR_RMI, /* index = MEFT, src = I */
401 INSTR_REGWR_RIH, /* index = I, src = H */
402 INSTR_REGWR_RIM, /* index = I, src = MEFT */
403 INSTR_REGWR_RII, /* index = I, src = I */
405 /* regadd REGARRAY index src
406 * REGARRAY[index] += src
407 * index = HMEFTI, src = HMEFTI
409 INSTR_REGADD_RHH, /* index = H, src = H */
410 INSTR_REGADD_RHM, /* index = H, src = MEFT */
411 INSTR_REGADD_RHI, /* index = H, src = I */
412 INSTR_REGADD_RMH, /* index = MEFT, src = H */
413 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
414 INSTR_REGADD_RMI, /* index = MEFT, src = I */
415 INSTR_REGADD_RIH, /* index = I, src = H */
416 INSTR_REGADD_RIM, /* index = I, src = MEFT */
417 INSTR_REGADD_RII, /* index = I, src = I */
419 /* metprefetch METARRAY index
420 * prefetch METARRAY[index]
423 INSTR_METPREFETCH_H, /* index = H */
424 INSTR_METPREFETCH_M, /* index = MEFT */
425 INSTR_METPREFETCH_I, /* index = I */
427 /* meter METARRAY index length color_in color_out
428 * color_out = meter(METARRAY[index], length, color_in)
429 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
431 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
432 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
433 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
434 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
435 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
436 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
437 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
438 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
439 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
440 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
441 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
442 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
451 /* learn LEARNER ACTION_NAME */
455 INSTR_LEARNER_FORGET,
457 /* extern e.obj.func */
468 /* jmpv LABEL h.header
469 * Jump if header is valid
473 /* jmpnv LABEL h.header
474 * Jump if header is invalid
479 * Jump if table lookup hit
484 * Jump if table lookup miss
491 INSTR_JMP_ACTION_HIT,
493 /* jmpna LABEL ACTION
494 * Jump if action not run
496 INSTR_JMP_ACTION_MISS,
499 * Jump if a is equal to b
500 * a = HMEFT, b = HMEFTI
502 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
503 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
504 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
505 INSTR_JMP_EQ_HH, /* a = H, b = H */
506 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
509 * Jump if a is not equal to b
510 * a = HMEFT, b = HMEFTI
512 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
513 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
514 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
515 INSTR_JMP_NEQ_HH, /* a = H, b = H */
516 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
519 * Jump if a is less than b
520 * a = HMEFT, b = HMEFTI
522 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
523 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
524 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
525 INSTR_JMP_LT_HH, /* a = H, b = H */
526 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
527 INSTR_JMP_LT_HI, /* a = H, b = I */
530 * Jump if a is greater than b
531 * a = HMEFT, b = HMEFTI
533 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
534 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
535 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
536 INSTR_JMP_GT_HH, /* a = H, b = H */
537 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
538 INSTR_JMP_GT_HI, /* a = H, b = I */
546 struct instr_operand {
567 uint8_t header_id[8];
568 uint8_t struct_id[8];
573 struct instr_hdr_validity {
585 struct instr_extern_obj {
590 struct instr_extern_func {
594 struct instr_dst_src {
595 struct instr_operand dst;
597 struct instr_operand src;
602 struct instr_regarray {
607 struct instr_operand idx;
612 struct instr_operand dstsrc;
622 struct instr_operand idx;
626 struct instr_operand length;
629 struct instr_operand color_in;
630 uint32_t color_in_val;
633 struct instr_operand color_out;
638 uint8_t header_id[8];
639 uint8_t struct_id[8];
650 struct instruction *ip;
653 struct instr_operand a;
659 struct instr_operand b;
665 enum instruction_type type;
668 struct instr_hdr_validity valid;
669 struct instr_dst_src mov;
670 struct instr_regarray regarray;
671 struct instr_meter meter;
672 struct instr_dma dma;
673 struct instr_dst_src alu;
674 struct instr_table table;
675 struct instr_learn learn;
676 struct instr_extern_obj ext_obj;
677 struct instr_extern_func ext_func;
678 struct instr_jmp jmp;
682 struct instruction_data {
683 char label[RTE_SWX_NAME_SIZE];
684 char jmp_label[RTE_SWX_NAME_SIZE];
685 uint32_t n_users; /* user = jmp instruction to this instruction. */
693 (*action_func_t)(struct rte_swx_pipeline *p);
696 TAILQ_ENTRY(action) node;
697 char name[RTE_SWX_NAME_SIZE];
698 struct struct_type *st;
699 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
700 struct instruction *instructions;
701 struct instruction_data *instruction_data;
702 uint32_t n_instructions;
706 TAILQ_HEAD(action_tailq, action);
712 TAILQ_ENTRY(table_type) node;
713 char name[RTE_SWX_NAME_SIZE];
714 enum rte_swx_table_match_type match_type;
715 struct rte_swx_table_ops ops;
718 TAILQ_HEAD(table_type_tailq, table_type);
721 enum rte_swx_table_match_type match_type;
726 TAILQ_ENTRY(table) node;
727 char name[RTE_SWX_NAME_SIZE];
728 char args[RTE_SWX_NAME_SIZE];
729 struct table_type *type; /* NULL when n_fields == 0. */
732 struct match_field *fields;
734 struct header *header; /* Only valid when n_fields > 0. */
737 struct action **actions;
738 struct action *default_action;
739 uint8_t *default_action_data;
741 int default_action_is_const;
742 uint32_t action_data_size_max;
748 TAILQ_HEAD(table_tailq, table);
750 struct table_runtime {
751 rte_swx_table_lookup_t func;
756 struct table_statistics {
757 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
758 uint64_t *n_pkts_action;
765 TAILQ_ENTRY(selector) node;
766 char name[RTE_SWX_NAME_SIZE];
768 struct field *group_id_field;
769 struct field **selector_fields;
770 uint32_t n_selector_fields;
771 struct header *selector_header;
772 struct field *member_id_field;
774 uint32_t n_groups_max;
775 uint32_t n_members_per_group_max;
780 TAILQ_HEAD(selector_tailq, selector);
782 struct selector_runtime {
784 uint8_t **group_id_buffer;
785 uint8_t **selector_buffer;
786 uint8_t **member_id_buffer;
789 struct selector_statistics {
797 TAILQ_ENTRY(learner) node;
798 char name[RTE_SWX_NAME_SIZE];
801 struct field **fields;
803 struct header *header;
806 struct action **actions;
807 struct field **action_arg;
808 struct action *default_action;
809 uint8_t *default_action_data;
811 int default_action_is_const;
812 uint32_t action_data_size_max;
819 TAILQ_HEAD(learner_tailq, learner);
821 struct learner_runtime {
824 uint8_t **action_data;
827 struct learner_statistics {
828 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
829 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
830 uint64_t n_pkts_forget;
831 uint64_t *n_pkts_action;
838 TAILQ_ENTRY(regarray) node;
839 char name[RTE_SWX_NAME_SIZE];
845 TAILQ_HEAD(regarray_tailq, regarray);
847 struct regarray_runtime {
855 struct meter_profile {
856 TAILQ_ENTRY(meter_profile) node;
857 char name[RTE_SWX_NAME_SIZE];
858 struct rte_meter_trtcm_params params;
859 struct rte_meter_trtcm_profile profile;
863 TAILQ_HEAD(meter_profile_tailq, meter_profile);
866 TAILQ_ENTRY(metarray) node;
867 char name[RTE_SWX_NAME_SIZE];
872 TAILQ_HEAD(metarray_tailq, metarray);
875 struct rte_meter_trtcm m;
876 struct meter_profile *profile;
877 enum rte_color color_mask;
880 uint64_t n_pkts[RTE_COLORS];
881 uint64_t n_bytes[RTE_COLORS];
884 struct metarray_runtime {
885 struct meter *metarray;
894 struct rte_swx_pkt pkt;
900 /* Packet headers. */
901 struct header_runtime *headers; /* Extracted or generated headers. */
902 struct header_out_runtime *headers_out; /* Emitted headers. */
903 uint8_t *header_storage;
904 uint8_t *header_out_storage;
905 uint64_t valid_headers;
906 uint32_t n_headers_out;
908 /* Packet meta-data. */
912 struct table_runtime *tables;
913 struct selector_runtime *selectors;
914 struct learner_runtime *learners;
915 struct rte_swx_table_state *table_state;
917 int hit; /* 0 = Miss, 1 = Hit. */
921 /* Extern objects and functions. */
922 struct extern_obj_runtime *extern_objs;
923 struct extern_func_runtime *extern_funcs;
926 struct instruction *ip;
927 struct instruction *ret;
930 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
931 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
932 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
934 #define HEADER_VALID(thread, header_id) \
935 MASK64_BIT_GET((thread)->valid_headers, header_id)
937 static inline uint64_t
938 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
940 uint8_t *x_struct = t->structs[x->struct_id];
941 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
942 uint64_t x64 = *x64_ptr;
943 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
945 return x64 & x64_mask;
948 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
950 static inline uint64_t
951 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
953 uint8_t *x_struct = t->structs[x->struct_id];
954 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
955 uint64_t x64 = *x64_ptr;
957 return ntoh64(x64) >> (64 - x->n_bits);
962 #define instr_operand_nbo instr_operand_hbo
966 #define ALU(thread, ip, operator) \
968 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
969 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
970 uint64_t dst64 = *dst64_ptr; \
971 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
972 uint64_t dst = dst64 & dst64_mask; \
974 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
975 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
976 uint64_t src64 = *src64_ptr; \
977 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
978 uint64_t src = src64 & src64_mask; \
980 uint64_t result = dst operator src; \
982 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
985 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
987 #define ALU_MH(thread, ip, operator) \
989 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
990 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
991 uint64_t dst64 = *dst64_ptr; \
992 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
993 uint64_t dst = dst64 & dst64_mask; \
995 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
996 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
997 uint64_t src64 = *src64_ptr; \
998 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1000 uint64_t result = dst operator src; \
1002 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1005 #define ALU_HM(thread, ip, operator) \
1007 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1008 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1009 uint64_t dst64 = *dst64_ptr; \
1010 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1011 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1013 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1014 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1015 uint64_t src64 = *src64_ptr; \
1016 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1017 uint64_t src = src64 & src64_mask; \
1019 uint64_t result = dst operator src; \
1020 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1022 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1025 #define ALU_HM_FAST(thread, ip, operator) \
1027 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1028 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1029 uint64_t dst64 = *dst64_ptr; \
1030 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1031 uint64_t dst = dst64 & dst64_mask; \
1033 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1034 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1035 uint64_t src64 = *src64_ptr; \
1036 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1037 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1039 uint64_t result = dst operator src; \
1041 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1044 #define ALU_HH(thread, ip, operator) \
1046 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1047 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1048 uint64_t dst64 = *dst64_ptr; \
1049 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1050 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1052 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1053 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1054 uint64_t src64 = *src64_ptr; \
1055 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1057 uint64_t result = dst operator src; \
1058 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1060 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1063 #define ALU_HH_FAST(thread, ip, operator) \
1065 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1066 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1067 uint64_t dst64 = *dst64_ptr; \
1068 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1069 uint64_t dst = dst64 & dst64_mask; \
1071 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1072 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1073 uint64_t src64 = *src64_ptr; \
1074 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1076 uint64_t result = dst operator src; \
1078 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1085 #define ALU_HM_FAST ALU
1087 #define ALU_HH_FAST ALU
1091 #define ALU_I(thread, ip, operator) \
1093 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1094 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1095 uint64_t dst64 = *dst64_ptr; \
1096 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1097 uint64_t dst = dst64 & dst64_mask; \
1099 uint64_t src = (ip)->alu.src_val; \
1101 uint64_t result = dst operator src; \
1103 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1106 #define ALU_MI ALU_I
1108 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1110 #define ALU_HI(thread, ip, operator) \
1112 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1113 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1114 uint64_t dst64 = *dst64_ptr; \
1115 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1116 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1118 uint64_t src = (ip)->alu.src_val; \
1120 uint64_t result = dst operator src; \
1121 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1123 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1128 #define ALU_HI ALU_I
1132 #define MOV(thread, ip) \
1134 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1135 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1136 uint64_t dst64 = *dst64_ptr; \
1137 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1139 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1140 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1141 uint64_t src64 = *src64_ptr; \
1142 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1143 uint64_t src = src64 & src64_mask; \
1145 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1148 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1150 #define MOV_MH(thread, ip) \
1152 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1153 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1154 uint64_t dst64 = *dst64_ptr; \
1155 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1157 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1158 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1159 uint64_t src64 = *src64_ptr; \
1160 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1162 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1165 #define MOV_HM(thread, ip) \
1167 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1168 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1169 uint64_t dst64 = *dst64_ptr; \
1170 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1172 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1173 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1174 uint64_t src64 = *src64_ptr; \
1175 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1176 uint64_t src = src64 & src64_mask; \
1178 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1179 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1182 #define MOV_HH(thread, ip) \
1184 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1185 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1186 uint64_t dst64 = *dst64_ptr; \
1187 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1189 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1190 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1191 uint64_t src64 = *src64_ptr; \
1193 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1194 src = src >> (64 - (ip)->mov.dst.n_bits); \
1195 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1206 #define MOV_I(thread, ip) \
1208 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1209 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1210 uint64_t dst64 = *dst64_ptr; \
1211 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1213 uint64_t src = (ip)->mov.src_val; \
1215 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1218 #define JMP_CMP(thread, ip, operator) \
1220 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1221 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1222 uint64_t a64 = *a64_ptr; \
1223 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1224 uint64_t a = a64 & a64_mask; \
1226 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1227 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1228 uint64_t b64 = *b64_ptr; \
1229 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1230 uint64_t b = b64 & b64_mask; \
1232 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1235 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1237 #define JMP_CMP_MH(thread, ip, operator) \
1239 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1240 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1241 uint64_t a64 = *a64_ptr; \
1242 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1243 uint64_t a = a64 & a64_mask; \
1245 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1246 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1247 uint64_t b64 = *b64_ptr; \
1248 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1250 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1253 #define JMP_CMP_HM(thread, ip, operator) \
1255 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1256 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1257 uint64_t a64 = *a64_ptr; \
1258 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1260 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1261 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1262 uint64_t b64 = *b64_ptr; \
1263 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1264 uint64_t b = b64 & b64_mask; \
1266 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1269 #define JMP_CMP_HH(thread, ip, operator) \
1271 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1272 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1273 uint64_t a64 = *a64_ptr; \
1274 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1276 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1277 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1278 uint64_t b64 = *b64_ptr; \
1279 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1281 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1284 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1286 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1287 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1288 uint64_t a64 = *a64_ptr; \
1289 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1291 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1292 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1293 uint64_t b64 = *b64_ptr; \
1294 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1296 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1301 #define JMP_CMP_MH JMP_CMP
1302 #define JMP_CMP_HM JMP_CMP
1303 #define JMP_CMP_HH JMP_CMP
1304 #define JMP_CMP_HH_FAST JMP_CMP
1308 #define JMP_CMP_I(thread, ip, operator) \
1310 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1311 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1312 uint64_t a64 = *a64_ptr; \
1313 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1314 uint64_t a = a64 & a64_mask; \
1316 uint64_t b = (ip)->jmp.b_val; \
1318 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1321 #define JMP_CMP_MI JMP_CMP_I
1323 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1325 #define JMP_CMP_HI(thread, ip, operator) \
1327 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1328 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1329 uint64_t a64 = *a64_ptr; \
1330 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1332 uint64_t b = (ip)->jmp.b_val; \
1334 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1339 #define JMP_CMP_HI JMP_CMP_I
1343 #define METADATA_READ(thread, offset, n_bits) \
1345 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1346 uint64_t m64 = *m64_ptr; \
1347 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1351 #define METADATA_WRITE(thread, offset, n_bits, value) \
1353 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1354 uint64_t m64 = *m64_ptr; \
1355 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1357 uint64_t m_new = value; \
1359 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1362 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1363 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1366 struct rte_swx_pipeline {
1367 struct struct_type_tailq struct_types;
1368 struct port_in_type_tailq port_in_types;
1369 struct port_in_tailq ports_in;
1370 struct port_out_type_tailq port_out_types;
1371 struct port_out_tailq ports_out;
1372 struct extern_type_tailq extern_types;
1373 struct extern_obj_tailq extern_objs;
1374 struct extern_func_tailq extern_funcs;
1375 struct header_tailq headers;
1376 struct struct_type *metadata_st;
1377 uint32_t metadata_struct_id;
1378 struct action_tailq actions;
1379 struct table_type_tailq table_types;
1380 struct table_tailq tables;
1381 struct selector_tailq selectors;
1382 struct learner_tailq learners;
1383 struct regarray_tailq regarrays;
1384 struct meter_profile_tailq meter_profiles;
1385 struct metarray_tailq metarrays;
1387 struct port_in_runtime *in;
1388 struct port_out_runtime *out;
1389 struct instruction **action_instructions;
1390 action_func_t *action_funcs;
1391 struct rte_swx_table_state *table_state;
1392 struct table_statistics *table_stats;
1393 struct selector_statistics *selector_stats;
1394 struct learner_statistics *learner_stats;
1395 struct regarray_runtime *regarray_runtime;
1396 struct metarray_runtime *metarray_runtime;
1397 struct instruction *instructions;
1398 struct instruction_data *instruction_data;
1399 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1402 uint32_t n_ports_in;
1403 uint32_t n_ports_out;
1404 uint32_t n_extern_objs;
1405 uint32_t n_extern_funcs;
1408 uint32_t n_selectors;
1409 uint32_t n_learners;
1410 uint32_t n_regarrays;
1411 uint32_t n_metarrays;
1415 uint32_t n_instructions;
1424 pipeline_port_inc(struct rte_swx_pipeline *p)
1426 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1430 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1432 t->ip = p->instructions;
1436 thread_ip_set(struct thread *t, struct instruction *ip)
1442 thread_ip_action_call(struct rte_swx_pipeline *p,
1447 t->ip = p->action_instructions[action_id];
1451 thread_ip_inc(struct rte_swx_pipeline *p);
1454 thread_ip_inc(struct rte_swx_pipeline *p)
1456 struct thread *t = &p->threads[p->thread_id];
1462 thread_ip_inc_cond(struct thread *t, int cond)
1468 thread_yield(struct rte_swx_pipeline *p)
1470 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1474 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1476 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1483 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1485 struct port_in_runtime *port = &p->in[p->port_id];
1486 struct rte_swx_pkt *pkt = &t->pkt;
1490 pkt_received = port->pkt_rx(port->obj, pkt);
1491 t->ptr = &pkt->pkt[pkt->offset];
1492 rte_prefetch0(t->ptr);
1494 TRACE("[Thread %2u] rx %s from port %u\n",
1496 pkt_received ? "1 pkt" : "0 pkts",
1500 t->valid_headers = 0;
1501 t->n_headers_out = 0;
1504 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1507 t->table_state = p->table_state;
1510 pipeline_port_inc(p);
1512 return pkt_received;
1516 instr_rx_exec(struct rte_swx_pipeline *p)
1518 struct thread *t = &p->threads[p->thread_id];
1519 struct instruction *ip = t->ip;
1523 pkt_received = __instr_rx_exec(p, t, ip);
1526 thread_ip_inc_cond(t, pkt_received);
1534 emit_handler(struct thread *t)
1536 struct header_out_runtime *h0 = &t->headers_out[0];
1537 struct header_out_runtime *h1 = &t->headers_out[1];
1538 uint32_t offset = 0, i;
1540 /* No header change or header decapsulation. */
1541 if ((t->n_headers_out == 1) &&
1542 (h0->ptr + h0->n_bytes == t->ptr)) {
1543 TRACE("Emit handler: no header change or header decap.\n");
1545 t->pkt.offset -= h0->n_bytes;
1546 t->pkt.length += h0->n_bytes;
1551 /* Header encapsulation (optionally, with prior header decasulation). */
1552 if ((t->n_headers_out == 2) &&
1553 (h1->ptr + h1->n_bytes == t->ptr) &&
1554 (h0->ptr == h0->ptr0)) {
1557 TRACE("Emit handler: header encapsulation.\n");
1559 offset = h0->n_bytes + h1->n_bytes;
1560 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1561 t->pkt.offset -= offset;
1562 t->pkt.length += offset;
1567 /* For any other case. */
1568 TRACE("Emit handler: complex case.\n");
1570 for (i = 0; i < t->n_headers_out; i++) {
1571 struct header_out_runtime *h = &t->headers_out[i];
1573 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1574 offset += h->n_bytes;
1578 memcpy(t->ptr - offset, t->header_out_storage, offset);
1579 t->pkt.offset -= offset;
1580 t->pkt.length += offset;
1585 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1587 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1588 struct port_out_runtime *port = &p->out[port_id];
1589 struct rte_swx_pkt *pkt = &t->pkt;
1591 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1599 port->pkt_tx(port->obj, pkt);
1603 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1605 uint64_t port_id = ip->io.io.val;
1606 struct port_out_runtime *port = &p->out[port_id];
1607 struct rte_swx_pkt *pkt = &t->pkt;
1609 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1617 port->pkt_tx(port->obj, pkt);
1624 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1626 const struct instruction *ip,
1629 uint64_t valid_headers = t->valid_headers;
1630 uint8_t *ptr = t->ptr;
1631 uint32_t offset = t->pkt.offset;
1632 uint32_t length = t->pkt.length;
1635 for (i = 0; i < n_extract; i++) {
1636 uint32_t header_id = ip->io.hdr.header_id[i];
1637 uint32_t struct_id = ip->io.hdr.struct_id[i];
1638 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1640 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1646 t->structs[struct_id] = ptr;
1647 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1656 t->valid_headers = valid_headers;
1659 t->pkt.offset = offset;
1660 t->pkt.length = length;
1665 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1667 const struct instruction *ip)
1669 __instr_hdr_extract_many_exec(p, t, ip, 1);
1673 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1675 const struct instruction *ip)
1677 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1679 __instr_hdr_extract_many_exec(p, t, ip, 2);
1683 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1685 const struct instruction *ip)
1687 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1689 __instr_hdr_extract_many_exec(p, t, ip, 3);
1693 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1695 const struct instruction *ip)
1697 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1699 __instr_hdr_extract_many_exec(p, t, ip, 4);
1703 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1705 const struct instruction *ip)
1707 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1709 __instr_hdr_extract_many_exec(p, t, ip, 5);
1713 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1715 const struct instruction *ip)
1717 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1719 __instr_hdr_extract_many_exec(p, t, ip, 6);
1723 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1725 const struct instruction *ip)
1727 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1729 __instr_hdr_extract_many_exec(p, t, ip, 7);
1733 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1735 const struct instruction *ip)
1737 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1739 __instr_hdr_extract_many_exec(p, t, ip, 8);
1743 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
1745 const struct instruction *ip)
1747 uint64_t valid_headers = t->valid_headers;
1748 uint8_t *ptr = t->ptr;
1749 uint32_t offset = t->pkt.offset;
1750 uint32_t length = t->pkt.length;
1752 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1753 uint32_t header_id = ip->io.hdr.header_id[0];
1754 uint32_t struct_id = ip->io.hdr.struct_id[0];
1755 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
1757 struct header_runtime *h = &t->headers[header_id];
1759 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
1765 n_bytes += n_bytes_last;
1768 t->structs[struct_id] = ptr;
1769 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1770 h->n_bytes = n_bytes;
1773 t->pkt.offset = offset + n_bytes;
1774 t->pkt.length = length - n_bytes;
1775 t->ptr = ptr + n_bytes;
1779 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
1781 const struct instruction *ip)
1783 uint64_t valid_headers = t->valid_headers;
1784 uint8_t *ptr = t->ptr;
1786 uint32_t header_id = ip->io.hdr.header_id[0];
1787 uint32_t struct_id = ip->io.hdr.struct_id[0];
1789 TRACE("[Thread %2u]: lookahead header %u\n",
1794 t->structs[struct_id] = ptr;
1795 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1802 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
1804 const struct instruction *ip,
1807 uint64_t valid_headers = t->valid_headers;
1808 uint32_t n_headers_out = t->n_headers_out;
1809 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1810 uint8_t *ho_ptr = NULL;
1811 uint32_t ho_nbytes = 0, first = 1, i;
1813 for (i = 0; i < n_emit; i++) {
1814 uint32_t header_id = ip->io.hdr.header_id[i];
1815 uint32_t struct_id = ip->io.hdr.struct_id[i];
1817 struct header_runtime *hi = &t->headers[header_id];
1818 uint8_t *hi_ptr0 = hi->ptr0;
1819 uint32_t n_bytes = hi->n_bytes;
1821 uint8_t *hi_ptr = t->structs[struct_id];
1823 if (!MASK64_BIT_GET(valid_headers, header_id))
1826 TRACE("[Thread %2u]: emit header %u\n",
1834 if (!t->n_headers_out) {
1835 ho = &t->headers_out[0];
1841 ho_nbytes = n_bytes;
1848 ho_nbytes = ho->n_bytes;
1852 if (ho_ptr + ho_nbytes == hi_ptr) {
1853 ho_nbytes += n_bytes;
1855 ho->n_bytes = ho_nbytes;
1862 ho_nbytes = n_bytes;
1868 ho->n_bytes = ho_nbytes;
1869 t->n_headers_out = n_headers_out;
1873 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
1875 const struct instruction *ip)
1877 __instr_hdr_emit_many_exec(p, t, ip, 1);
1881 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
1883 const struct instruction *ip)
1885 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1887 __instr_hdr_emit_many_exec(p, t, ip, 1);
1888 __instr_tx_exec(p, t, ip);
1892 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
1894 const struct instruction *ip)
1896 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1898 __instr_hdr_emit_many_exec(p, t, ip, 2);
1899 __instr_tx_exec(p, t, ip);
1903 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
1905 const struct instruction *ip)
1907 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1909 __instr_hdr_emit_many_exec(p, t, ip, 3);
1910 __instr_tx_exec(p, t, ip);
1914 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
1916 const struct instruction *ip)
1918 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1920 __instr_hdr_emit_many_exec(p, t, ip, 4);
1921 __instr_tx_exec(p, t, ip);
1925 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
1927 const struct instruction *ip)
1929 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1931 __instr_hdr_emit_many_exec(p, t, ip, 5);
1932 __instr_tx_exec(p, t, ip);
1936 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
1938 const struct instruction *ip)
1940 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1942 __instr_hdr_emit_many_exec(p, t, ip, 6);
1943 __instr_tx_exec(p, t, ip);
1947 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
1949 const struct instruction *ip)
1951 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1953 __instr_hdr_emit_many_exec(p, t, ip, 7);
1954 __instr_tx_exec(p, t, ip);
1958 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
1960 const struct instruction *ip)
1962 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
1964 __instr_hdr_emit_many_exec(p, t, ip, 8);
1965 __instr_tx_exec(p, t, ip);
1972 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
1974 const struct instruction *ip)
1976 uint32_t header_id = ip->valid.header_id;
1978 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
1981 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
1988 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
1990 const struct instruction *ip)
1992 uint32_t header_id = ip->valid.header_id;
1994 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
1997 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2004 __instr_learn_exec(struct rte_swx_pipeline *p,
2006 const struct instruction *ip)
2008 uint64_t action_id = ip->learn.action_id;
2009 uint32_t learner_id = t->learner_id;
2010 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2011 p->n_selectors + learner_id];
2012 struct learner_runtime *l = &t->learners[learner_id];
2013 struct learner_statistics *stats = &p->learner_stats[learner_id];
2017 status = rte_swx_table_learner_add(ts->obj,
2021 l->action_data[action_id]);
2023 TRACE("[Thread %2u] learner %u learn %s\n",
2026 status ? "ok" : "error");
2028 stats->n_pkts_learn[status] += 1;
2035 __instr_forget_exec(struct rte_swx_pipeline *p,
2037 const struct instruction *ip __rte_unused)
2039 uint32_t learner_id = t->learner_id;
2040 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2041 p->n_selectors + learner_id];
2042 struct learner_runtime *l = &t->learners[learner_id];
2043 struct learner_statistics *stats = &p->learner_stats[learner_id];
2046 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2048 TRACE("[Thread %2u] learner %u forget\n",
2052 stats->n_pkts_forget += 1;
2058 static inline uint32_t
2059 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2061 const struct instruction *ip)
2063 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2064 uint32_t func_id = ip->ext_obj.func_id;
2065 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2066 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2069 TRACE("[Thread %2u] extern obj %u member func %u\n",
2074 done = func(obj->obj, obj->mailbox);
2079 static inline uint32_t
2080 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2082 const struct instruction *ip)
2084 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2085 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2086 rte_swx_extern_func_t func = ext_func->func;
2089 TRACE("[Thread %2u] extern func %u\n",
2093 done = func(ext_func->mailbox);
2102 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2104 const struct instruction *ip)
2106 TRACE("[Thread %2u] mov\n", p->thread_id);
2112 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2114 const struct instruction *ip)
2116 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2122 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2124 const struct instruction *ip)
2126 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2132 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2134 const struct instruction *ip)
2136 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2142 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2144 const struct instruction *ip)
2146 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2155 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2157 const struct instruction *ip,
2160 uint8_t *action_data = t->structs[0];
2161 uint64_t valid_headers = t->valid_headers;
2164 for (i = 0; i < n_dma; i++) {
2165 uint32_t header_id = ip->dma.dst.header_id[i];
2166 uint32_t struct_id = ip->dma.dst.struct_id[i];
2167 uint32_t offset = ip->dma.src.offset[i];
2168 uint32_t n_bytes = ip->dma.n_bytes[i];
2170 struct header_runtime *h = &t->headers[header_id];
2171 uint8_t *h_ptr0 = h->ptr0;
2172 uint8_t *h_ptr = t->structs[struct_id];
2174 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2176 void *src = &action_data[offset];
2178 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2181 memcpy(dst, src, n_bytes);
2182 t->structs[struct_id] = dst;
2183 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2186 t->valid_headers = valid_headers;
2190 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2192 __instr_dma_ht_many_exec(p, t, ip, 1);
2196 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2198 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2200 __instr_dma_ht_many_exec(p, t, ip, 2);
2204 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2206 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2208 __instr_dma_ht_many_exec(p, t, ip, 3);
2212 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2214 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2216 __instr_dma_ht_many_exec(p, t, ip, 4);
2220 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2222 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2224 __instr_dma_ht_many_exec(p, t, ip, 5);
2228 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2230 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2232 __instr_dma_ht_many_exec(p, t, ip, 6);
2236 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2238 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2240 __instr_dma_ht_many_exec(p, t, ip, 7);
2244 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2246 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2248 __instr_dma_ht_many_exec(p, t, ip, 8);
2255 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2257 const struct instruction *ip)
2259 TRACE("[Thread %2u] add\n", p->thread_id);
2265 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2267 const struct instruction *ip)
2269 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2275 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2277 const struct instruction *ip)
2279 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2285 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2287 const struct instruction *ip)
2289 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2295 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2297 const struct instruction *ip)
2299 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2305 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2307 const struct instruction *ip)
2309 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2315 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2317 const struct instruction *ip)
2319 TRACE("[Thread %2u] sub\n", p->thread_id);
2325 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2327 const struct instruction *ip)
2329 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2335 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2337 const struct instruction *ip)
2339 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2345 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2347 const struct instruction *ip)
2349 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2355 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2357 const struct instruction *ip)
2359 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2365 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2367 const struct instruction *ip)
2369 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2375 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2377 const struct instruction *ip)
2379 TRACE("[Thread %2u] shl\n", p->thread_id);
2385 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2387 const struct instruction *ip)
2389 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2395 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2397 const struct instruction *ip)
2399 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2405 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2407 const struct instruction *ip)
2409 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2415 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2417 const struct instruction *ip)
2419 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2425 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2427 const struct instruction *ip)
2429 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2435 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2437 const struct instruction *ip)
2439 TRACE("[Thread %2u] shr\n", p->thread_id);
2445 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2447 const struct instruction *ip)
2449 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2455 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2457 const struct instruction *ip)
2459 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2465 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2467 const struct instruction *ip)
2469 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2475 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2477 const struct instruction *ip)
2479 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2486 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2488 const struct instruction *ip)
2490 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2496 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2498 const struct instruction *ip)
2500 TRACE("[Thread %2u] and\n", p->thread_id);
2506 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2508 const struct instruction *ip)
2510 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2516 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2518 const struct instruction *ip)
2520 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2522 ALU_HM_FAST(t, ip, &);
2526 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2528 const struct instruction *ip)
2530 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2532 ALU_HH_FAST(t, ip, &);
2536 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2538 const struct instruction *ip)
2540 TRACE("[Thread %2u] and (i)\n", p->thread_id);
2546 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2548 const struct instruction *ip)
2550 TRACE("[Thread %2u] or\n", p->thread_id);
2556 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2558 const struct instruction *ip)
2560 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2566 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2568 const struct instruction *ip)
2570 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
2572 ALU_HM_FAST(t, ip, |);
2576 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2578 const struct instruction *ip)
2580 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
2582 ALU_HH_FAST(t, ip, |);
2586 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
2588 const struct instruction *ip)
2590 TRACE("[Thread %2u] or (i)\n", p->thread_id);
2596 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
2598 const struct instruction *ip)
2600 TRACE("[Thread %2u] xor\n", p->thread_id);
2606 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2608 const struct instruction *ip)
2610 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
2616 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2618 const struct instruction *ip)
2620 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
2622 ALU_HM_FAST(t, ip, ^);
2626 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2628 const struct instruction *ip)
2630 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
2632 ALU_HH_FAST(t, ip, ^);
2636 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
2638 const struct instruction *ip)
2640 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
2646 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
2648 const struct instruction *ip)
2650 uint8_t *dst_struct, *src_struct;
2651 uint16_t *dst16_ptr, dst;
2652 uint64_t *src64_ptr, src64, src64_mask, src;
2655 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
2658 dst_struct = t->structs[ip->alu.dst.struct_id];
2659 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2662 src_struct = t->structs[ip->alu.src.struct_id];
2663 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2665 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2666 src = src64 & src64_mask;
2671 /* The first input (r) is a 16-bit number. The second and the third
2672 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
2673 * three numbers (output r) is a 34-bit number.
2675 r += (src >> 32) + (src & 0xFFFFFFFF);
2677 /* The first input is a 16-bit number. The second input is an 18-bit
2678 * number. In the worst case scenario, the sum of the two numbers is a
2681 r = (r & 0xFFFF) + (r >> 16);
2683 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2684 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
2686 r = (r & 0xFFFF) + (r >> 16);
2688 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2689 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2690 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
2691 * therefore the output r is always a 16-bit number.
2693 r = (r & 0xFFFF) + (r >> 16);
2698 *dst16_ptr = (uint16_t)r;
2702 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
2704 const struct instruction *ip)
2706 uint8_t *dst_struct, *src_struct;
2707 uint16_t *dst16_ptr, dst;
2708 uint64_t *src64_ptr, src64, src64_mask, src;
2711 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
2714 dst_struct = t->structs[ip->alu.dst.struct_id];
2715 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2718 src_struct = t->structs[ip->alu.src.struct_id];
2719 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2721 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2722 src = src64 & src64_mask;
2727 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
2728 * the following sequence of operations in 2's complement arithmetic:
2729 * a '- b = (a - b) % 0xFFFF.
2731 * In order to prevent an underflow for the below subtraction, in which
2732 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
2733 * minuend), we first add a multiple of the 0xFFFF modulus to the
2734 * minuend. The number we add to the minuend needs to be a 34-bit number
2735 * or higher, so for readability reasons we picked the 36-bit multiple.
2736 * We are effectively turning the 16-bit minuend into a 36-bit number:
2737 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
2739 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
2741 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
2742 * result (the output r) is a 36-bit number.
2744 r -= (src >> 32) + (src & 0xFFFFFFFF);
2746 /* The first input is a 16-bit number. The second input is a 20-bit
2747 * number. Their sum is a 21-bit number.
2749 r = (r & 0xFFFF) + (r >> 16);
2751 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2752 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
2754 r = (r & 0xFFFF) + (r >> 16);
2756 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2757 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2758 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
2759 * generated, therefore the output r is always a 16-bit number.
2761 r = (r & 0xFFFF) + (r >> 16);
2766 *dst16_ptr = (uint16_t)r;
2770 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
2772 const struct instruction *ip)
2774 uint8_t *dst_struct, *src_struct;
2775 uint16_t *dst16_ptr;
2776 uint32_t *src32_ptr;
2779 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
2782 dst_struct = t->structs[ip->alu.dst.struct_id];
2783 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2785 src_struct = t->structs[ip->alu.src.struct_id];
2786 src32_ptr = (uint32_t *)&src_struct[0];
2788 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
2789 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
2790 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
2791 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
2792 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
2794 /* The first input is a 16-bit number. The second input is a 19-bit
2795 * number. Their sum is a 20-bit number.
2797 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2799 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2800 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
2802 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2804 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2805 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2806 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
2807 * generated, therefore the output r is always a 16-bit number.
2809 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2812 r0 = r0 ? r0 : 0xFFFF;
2814 *dst16_ptr = (uint16_t)r0;
2818 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
2820 const struct instruction *ip)
2822 uint8_t *dst_struct, *src_struct;
2823 uint16_t *dst16_ptr;
2824 uint32_t *src32_ptr;
2828 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
2831 dst_struct = t->structs[ip->alu.dst.struct_id];
2832 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2834 src_struct = t->structs[ip->alu.src.struct_id];
2835 src32_ptr = (uint32_t *)&src_struct[0];
2837 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
2838 * Therefore, in the worst case scenario, a 35-bit number is added to a
2839 * 16-bit number (the input r), so the output r is 36-bit number.
2841 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
2844 /* The first input is a 16-bit number. The second input is a 20-bit
2845 * number. Their sum is a 21-bit number.
2847 r = (r & 0xFFFF) + (r >> 16);
2849 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2850 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
2852 r = (r & 0xFFFF) + (r >> 16);
2854 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2855 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2856 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
2857 * generated, therefore the output r is always a 16-bit number.
2859 r = (r & 0xFFFF) + (r >> 16);
2864 *dst16_ptr = (uint16_t)r;
2870 static inline uint64_t *
2871 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
2873 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2877 static inline uint64_t
2878 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2880 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2882 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
2883 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
2884 uint64_t idx64 = *idx64_ptr;
2885 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
2886 uint64_t idx = idx64 & idx64_mask & r->size_mask;
2891 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2893 static inline uint64_t
2894 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2896 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2898 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
2899 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
2900 uint64_t idx64 = *idx64_ptr;
2901 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
2908 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
2912 static inline uint64_t
2913 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
2915 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2917 uint64_t idx = ip->regarray.idx_val & r->size_mask;
2922 static inline uint64_t
2923 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
2925 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
2926 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
2927 uint64_t src64 = *src64_ptr;
2928 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2929 uint64_t src = src64 & src64_mask;
2934 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2936 static inline uint64_t
2937 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
2939 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
2940 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
2941 uint64_t src64 = *src64_ptr;
2942 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
2949 #define instr_regarray_src_nbo instr_regarray_src_hbo
2954 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
2956 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
2957 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
2958 uint64_t dst64 = *dst64_ptr;
2959 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2961 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
2965 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2968 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
2970 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
2971 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
2972 uint64_t dst64 = *dst64_ptr;
2973 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2975 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
2976 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
2981 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
2986 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
2988 const struct instruction *ip)
2990 uint64_t *regarray, idx;
2992 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
2994 regarray = instr_regarray_regarray(p, ip);
2995 idx = instr_regarray_idx_nbo(p, t, ip);
2996 rte_prefetch0(®array[idx]);
3000 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3002 const struct instruction *ip)
3004 uint64_t *regarray, idx;
3006 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3008 regarray = instr_regarray_regarray(p, ip);
3009 idx = instr_regarray_idx_hbo(p, t, ip);
3010 rte_prefetch0(®array[idx]);
3014 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3015 struct thread *t __rte_unused,
3016 const struct instruction *ip)
3018 uint64_t *regarray, idx;
3020 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3022 regarray = instr_regarray_regarray(p, ip);
3023 idx = instr_regarray_idx_imm(p, ip);
3024 rte_prefetch0(®array[idx]);
3028 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3030 const struct instruction *ip)
3032 uint64_t *regarray, idx;
3034 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3036 regarray = instr_regarray_regarray(p, ip);
3037 idx = instr_regarray_idx_nbo(p, t, ip);
3038 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3042 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3044 const struct instruction *ip)
3046 uint64_t *regarray, idx;
3048 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3051 regarray = instr_regarray_regarray(p, ip);
3052 idx = instr_regarray_idx_hbo(p, t, ip);
3053 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3057 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3059 uint64_t *regarray, idx;
3061 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3063 regarray = instr_regarray_regarray(p, ip);
3064 idx = instr_regarray_idx_nbo(p, t, ip);
3065 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3069 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3071 uint64_t *regarray, idx;
3073 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3075 regarray = instr_regarray_regarray(p, ip);
3076 idx = instr_regarray_idx_hbo(p, t, ip);
3077 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3081 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3083 uint64_t *regarray, idx;
3085 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3087 regarray = instr_regarray_regarray(p, ip);
3088 idx = instr_regarray_idx_imm(p, ip);
3089 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3093 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3095 uint64_t *regarray, idx;
3097 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3099 regarray = instr_regarray_regarray(p, ip);
3100 idx = instr_regarray_idx_imm(p, ip);
3101 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3105 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3107 uint64_t *regarray, idx, src;
3109 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3111 regarray = instr_regarray_regarray(p, ip);
3112 idx = instr_regarray_idx_nbo(p, t, ip);
3113 src = instr_regarray_src_nbo(t, ip);
3114 regarray[idx] = src;
3118 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3120 uint64_t *regarray, idx, src;
3122 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3124 regarray = instr_regarray_regarray(p, ip);
3125 idx = instr_regarray_idx_nbo(p, t, ip);
3126 src = instr_regarray_src_hbo(t, ip);
3127 regarray[idx] = src;
3131 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3133 uint64_t *regarray, idx, src;
3135 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3137 regarray = instr_regarray_regarray(p, ip);
3138 idx = instr_regarray_idx_hbo(p, t, ip);
3139 src = instr_regarray_src_nbo(t, ip);
3140 regarray[idx] = src;
3144 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3146 uint64_t *regarray, idx, src;
3148 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3150 regarray = instr_regarray_regarray(p, ip);
3151 idx = instr_regarray_idx_hbo(p, t, ip);
3152 src = instr_regarray_src_hbo(t, ip);
3153 regarray[idx] = src;
3157 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3159 uint64_t *regarray, idx, src;
3161 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3163 regarray = instr_regarray_regarray(p, ip);
3164 idx = instr_regarray_idx_nbo(p, t, ip);
3165 src = ip->regarray.dstsrc_val;
3166 regarray[idx] = src;
3170 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3172 uint64_t *regarray, idx, src;
3174 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3176 regarray = instr_regarray_regarray(p, ip);
3177 idx = instr_regarray_idx_hbo(p, t, ip);
3178 src = ip->regarray.dstsrc_val;
3179 regarray[idx] = src;
3183 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3185 uint64_t *regarray, idx, src;
3187 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3189 regarray = instr_regarray_regarray(p, ip);
3190 idx = instr_regarray_idx_imm(p, ip);
3191 src = instr_regarray_src_nbo(t, ip);
3192 regarray[idx] = src;
3196 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3198 uint64_t *regarray, idx, src;
3200 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3202 regarray = instr_regarray_regarray(p, ip);
3203 idx = instr_regarray_idx_imm(p, ip);
3204 src = instr_regarray_src_hbo(t, ip);
3205 regarray[idx] = src;
3209 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3210 struct thread *t __rte_unused,
3211 const struct instruction *ip)
3213 uint64_t *regarray, idx, src;
3215 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3217 regarray = instr_regarray_regarray(p, ip);
3218 idx = instr_regarray_idx_imm(p, ip);
3219 src = ip->regarray.dstsrc_val;
3220 regarray[idx] = src;
3224 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3226 uint64_t *regarray, idx, src;
3228 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3230 regarray = instr_regarray_regarray(p, ip);
3231 idx = instr_regarray_idx_nbo(p, t, ip);
3232 src = instr_regarray_src_nbo(t, ip);
3233 regarray[idx] += src;
3237 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3239 uint64_t *regarray, idx, src;
3241 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3243 regarray = instr_regarray_regarray(p, ip);
3244 idx = instr_regarray_idx_nbo(p, t, ip);
3245 src = instr_regarray_src_hbo(t, ip);
3246 regarray[idx] += src;
3250 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3252 uint64_t *regarray, idx, src;
3254 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3256 regarray = instr_regarray_regarray(p, ip);
3257 idx = instr_regarray_idx_hbo(p, t, ip);
3258 src = instr_regarray_src_nbo(t, ip);
3259 regarray[idx] += src;
3263 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3265 uint64_t *regarray, idx, src;
3267 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3269 regarray = instr_regarray_regarray(p, ip);
3270 idx = instr_regarray_idx_hbo(p, t, ip);
3271 src = instr_regarray_src_hbo(t, ip);
3272 regarray[idx] += src;
3276 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3278 uint64_t *regarray, idx, src;
3280 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3282 regarray = instr_regarray_regarray(p, ip);
3283 idx = instr_regarray_idx_nbo(p, t, ip);
3284 src = ip->regarray.dstsrc_val;
3285 regarray[idx] += src;
3289 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3291 uint64_t *regarray, idx, src;
3293 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3295 regarray = instr_regarray_regarray(p, ip);
3296 idx = instr_regarray_idx_hbo(p, t, ip);
3297 src = ip->regarray.dstsrc_val;
3298 regarray[idx] += src;
3302 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3304 uint64_t *regarray, idx, src;
3306 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3308 regarray = instr_regarray_regarray(p, ip);
3309 idx = instr_regarray_idx_imm(p, ip);
3310 src = instr_regarray_src_nbo(t, ip);
3311 regarray[idx] += src;
3315 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3317 uint64_t *regarray, idx, src;
3319 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3321 regarray = instr_regarray_regarray(p, ip);
3322 idx = instr_regarray_idx_imm(p, ip);
3323 src = instr_regarray_src_hbo(t, ip);
3324 regarray[idx] += src;
3328 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3329 struct thread *t __rte_unused,
3330 const struct instruction *ip)
3332 uint64_t *regarray, idx, src;
3334 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3336 regarray = instr_regarray_regarray(p, ip);
3337 idx = instr_regarray_idx_imm(p, ip);
3338 src = ip->regarray.dstsrc_val;
3339 regarray[idx] += src;
3345 static inline struct meter *
3346 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3348 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3350 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3351 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3352 uint64_t idx64 = *idx64_ptr;
3353 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3354 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3356 return &r->metarray[idx];
3359 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3361 static inline struct meter *
3362 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3364 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3366 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3367 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3368 uint64_t idx64 = *idx64_ptr;
3369 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3371 return &r->metarray[idx];
3376 #define instr_meter_idx_nbo instr_meter_idx_hbo
3380 static inline struct meter *
3381 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3383 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3385 uint64_t idx = ip->meter.idx_val & r->size_mask;
3387 return &r->metarray[idx];
3390 static inline uint32_t
3391 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3393 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3394 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3395 uint64_t src64 = *src64_ptr;
3396 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3397 uint64_t src = src64 & src64_mask;
3399 return (uint32_t)src;
3402 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3404 static inline uint32_t
3405 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3407 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3408 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3409 uint64_t src64 = *src64_ptr;
3410 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3412 return (uint32_t)src;
3417 #define instr_meter_length_nbo instr_meter_length_hbo
3421 static inline enum rte_color
3422 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3424 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3425 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3426 uint64_t src64 = *src64_ptr;
3427 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3428 uint64_t src = src64 & src64_mask;
3430 return (enum rte_color)src;
3434 instr_meter_color_out_hbo_set(struct thread *t,
3435 const struct instruction *ip,
3436 enum rte_color color_out)
3438 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3439 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3440 uint64_t dst64 = *dst64_ptr;
3441 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3443 uint64_t src = (uint64_t)color_out;
3445 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3449 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3451 const struct instruction *ip)
3455 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3457 m = instr_meter_idx_nbo(p, t, ip);
3462 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3464 const struct instruction *ip)
3468 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3470 m = instr_meter_idx_hbo(p, t, ip);
3475 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3476 struct thread *t __rte_unused,
3477 const struct instruction *ip)
3481 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3483 m = instr_meter_idx_imm(p, ip);
3488 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3491 uint64_t time, n_pkts, n_bytes;
3493 enum rte_color color_in, color_out;
3495 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3497 m = instr_meter_idx_nbo(p, t, ip);
3498 rte_prefetch0(m->n_pkts);
3499 time = rte_get_tsc_cycles();
3500 length = instr_meter_length_nbo(t, ip);
3501 color_in = instr_meter_color_in_hbo(t, ip);
3503 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3504 &m->profile->profile,
3509 color_out &= m->color_mask;
3511 n_pkts = m->n_pkts[color_out];
3512 n_bytes = m->n_bytes[color_out];
3514 instr_meter_color_out_hbo_set(t, ip, color_out);
3516 m->n_pkts[color_out] = n_pkts + 1;
3517 m->n_bytes[color_out] = n_bytes + length;
3521 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3524 uint64_t time, n_pkts, n_bytes;
3526 enum rte_color color_in, color_out;
3528 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3530 m = instr_meter_idx_nbo(p, t, ip);
3531 rte_prefetch0(m->n_pkts);
3532 time = rte_get_tsc_cycles();
3533 length = instr_meter_length_nbo(t, ip);
3534 color_in = (enum rte_color)ip->meter.color_in_val;
3536 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3537 &m->profile->profile,
3542 color_out &= m->color_mask;
3544 n_pkts = m->n_pkts[color_out];
3545 n_bytes = m->n_bytes[color_out];
3547 instr_meter_color_out_hbo_set(t, ip, color_out);
3549 m->n_pkts[color_out] = n_pkts + 1;
3550 m->n_bytes[color_out] = n_bytes + length;
3554 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3557 uint64_t time, n_pkts, n_bytes;
3559 enum rte_color color_in, color_out;
3561 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
3563 m = instr_meter_idx_nbo(p, t, ip);
3564 rte_prefetch0(m->n_pkts);
3565 time = rte_get_tsc_cycles();
3566 length = instr_meter_length_hbo(t, ip);
3567 color_in = instr_meter_color_in_hbo(t, ip);
3569 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3570 &m->profile->profile,
3575 color_out &= m->color_mask;
3577 n_pkts = m->n_pkts[color_out];
3578 n_bytes = m->n_bytes[color_out];
3580 instr_meter_color_out_hbo_set(t, ip, color_out);
3582 m->n_pkts[color_out] = n_pkts + 1;
3583 m->n_bytes[color_out] = n_bytes + length;
3587 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3590 uint64_t time, n_pkts, n_bytes;
3592 enum rte_color color_in, color_out;
3594 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
3596 m = instr_meter_idx_nbo(p, t, ip);
3597 rte_prefetch0(m->n_pkts);
3598 time = rte_get_tsc_cycles();
3599 length = instr_meter_length_hbo(t, ip);
3600 color_in = (enum rte_color)ip->meter.color_in_val;
3602 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3603 &m->profile->profile,
3608 color_out &= m->color_mask;
3610 n_pkts = m->n_pkts[color_out];
3611 n_bytes = m->n_bytes[color_out];
3613 instr_meter_color_out_hbo_set(t, ip, color_out);
3615 m->n_pkts[color_out] = n_pkts + 1;
3616 m->n_bytes[color_out] = n_bytes + length;
3620 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3623 uint64_t time, n_pkts, n_bytes;
3625 enum rte_color color_in, color_out;
3627 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
3629 m = instr_meter_idx_hbo(p, t, ip);
3630 rte_prefetch0(m->n_pkts);
3631 time = rte_get_tsc_cycles();
3632 length = instr_meter_length_nbo(t, ip);
3633 color_in = instr_meter_color_in_hbo(t, ip);
3635 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3636 &m->profile->profile,
3641 color_out &= m->color_mask;
3643 n_pkts = m->n_pkts[color_out];
3644 n_bytes = m->n_bytes[color_out];
3646 instr_meter_color_out_hbo_set(t, ip, color_out);
3648 m->n_pkts[color_out] = n_pkts + 1;
3649 m->n_bytes[color_out] = n_bytes + length;
3653 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3656 uint64_t time, n_pkts, n_bytes;
3658 enum rte_color color_in, color_out;
3660 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
3662 m = instr_meter_idx_hbo(p, t, ip);
3663 rte_prefetch0(m->n_pkts);
3664 time = rte_get_tsc_cycles();
3665 length = instr_meter_length_nbo(t, ip);
3666 color_in = (enum rte_color)ip->meter.color_in_val;
3668 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3669 &m->profile->profile,
3674 color_out &= m->color_mask;
3676 n_pkts = m->n_pkts[color_out];
3677 n_bytes = m->n_bytes[color_out];
3679 instr_meter_color_out_hbo_set(t, ip, color_out);
3681 m->n_pkts[color_out] = n_pkts + 1;
3682 m->n_bytes[color_out] = n_bytes + length;
3686 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3689 uint64_t time, n_pkts, n_bytes;
3691 enum rte_color color_in, color_out;
3693 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
3695 m = instr_meter_idx_hbo(p, t, ip);
3696 rte_prefetch0(m->n_pkts);
3697 time = rte_get_tsc_cycles();
3698 length = instr_meter_length_hbo(t, ip);
3699 color_in = instr_meter_color_in_hbo(t, ip);
3701 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3702 &m->profile->profile,
3707 color_out &= m->color_mask;
3709 n_pkts = m->n_pkts[color_out];
3710 n_bytes = m->n_bytes[color_out];
3712 instr_meter_color_out_hbo_set(t, ip, color_out);
3714 m->n_pkts[color_out] = n_pkts + 1;
3715 m->n_bytes[color_out] = n_bytes + length;
3719 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3722 uint64_t time, n_pkts, n_bytes;
3724 enum rte_color color_in, color_out;
3726 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
3728 m = instr_meter_idx_hbo(p, t, ip);
3729 rte_prefetch0(m->n_pkts);
3730 time = rte_get_tsc_cycles();
3731 length = instr_meter_length_hbo(t, ip);
3732 color_in = (enum rte_color)ip->meter.color_in_val;
3734 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3735 &m->profile->profile,
3740 color_out &= m->color_mask;
3742 n_pkts = m->n_pkts[color_out];
3743 n_bytes = m->n_bytes[color_out];
3745 instr_meter_color_out_hbo_set(t, ip, color_out);
3747 m->n_pkts[color_out] = n_pkts + 1;
3748 m->n_bytes[color_out] = n_bytes + length;
3752 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3755 uint64_t time, n_pkts, n_bytes;
3757 enum rte_color color_in, color_out;
3759 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
3761 m = instr_meter_idx_imm(p, ip);
3762 rte_prefetch0(m->n_pkts);
3763 time = rte_get_tsc_cycles();
3764 length = instr_meter_length_nbo(t, ip);
3765 color_in = instr_meter_color_in_hbo(t, ip);
3767 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3768 &m->profile->profile,
3773 color_out &= m->color_mask;
3775 n_pkts = m->n_pkts[color_out];
3776 n_bytes = m->n_bytes[color_out];
3778 instr_meter_color_out_hbo_set(t, ip, color_out);
3780 m->n_pkts[color_out] = n_pkts + 1;
3781 m->n_bytes[color_out] = n_bytes + length;
3785 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3788 uint64_t time, n_pkts, n_bytes;
3790 enum rte_color color_in, color_out;
3792 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
3794 m = instr_meter_idx_imm(p, ip);
3795 rte_prefetch0(m->n_pkts);
3796 time = rte_get_tsc_cycles();
3797 length = instr_meter_length_nbo(t, ip);
3798 color_in = (enum rte_color)ip->meter.color_in_val;
3800 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3801 &m->profile->profile,
3806 color_out &= m->color_mask;
3808 n_pkts = m->n_pkts[color_out];
3809 n_bytes = m->n_bytes[color_out];
3811 instr_meter_color_out_hbo_set(t, ip, color_out);
3813 m->n_pkts[color_out] = n_pkts + 1;
3814 m->n_bytes[color_out] = n_bytes + length;
3818 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3821 uint64_t time, n_pkts, n_bytes;
3823 enum rte_color color_in, color_out;
3825 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
3827 m = instr_meter_idx_imm(p, ip);
3828 rte_prefetch0(m->n_pkts);
3829 time = rte_get_tsc_cycles();
3830 length = instr_meter_length_hbo(t, ip);
3831 color_in = instr_meter_color_in_hbo(t, ip);
3833 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3834 &m->profile->profile,
3839 color_out &= m->color_mask;
3841 n_pkts = m->n_pkts[color_out];
3842 n_bytes = m->n_bytes[color_out];
3844 instr_meter_color_out_hbo_set(t, ip, color_out);
3846 m->n_pkts[color_out] = n_pkts + 1;
3847 m->n_bytes[color_out] = n_bytes + length;
3851 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3854 uint64_t time, n_pkts, n_bytes;
3856 enum rte_color color_in, color_out;
3858 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
3860 m = instr_meter_idx_imm(p, ip);
3861 rte_prefetch0(m->n_pkts);
3862 time = rte_get_tsc_cycles();
3863 length = instr_meter_length_hbo(t, ip);
3864 color_in = (enum rte_color)ip->meter.color_in_val;
3866 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3867 &m->profile->profile,
3872 color_out &= m->color_mask;
3874 n_pkts = m->n_pkts[color_out];
3875 n_bytes = m->n_bytes[color_out];
3877 instr_meter_color_out_hbo_set(t, ip, color_out);
3879 m->n_pkts[color_out] = n_pkts + 1;
3880 m->n_bytes[color_out] = n_bytes + length;