1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
11 #include <rte_byteorder.h>
12 #include <rte_common.h>
13 #include <rte_cycles.h>
14 #include <rte_prefetch.h>
15 #include <rte_meter.h>
17 #include <rte_swx_table_selector.h>
18 #include <rte_swx_table_learner.h>
19 #include <rte_swx_pipeline.h>
20 #include <rte_swx_ctl.h>
27 #define TRACE(...) printf(__VA_ARGS__)
35 #define ntoh64(x) rte_be_to_cpu_64(x)
36 #define hton64(x) rte_cpu_to_be_64(x)
42 char name[RTE_SWX_NAME_SIZE];
49 TAILQ_ENTRY(struct_type) node;
50 char name[RTE_SWX_NAME_SIZE];
58 TAILQ_HEAD(struct_type_tailq, struct_type);
64 TAILQ_ENTRY(port_in_type) node;
65 char name[RTE_SWX_NAME_SIZE];
66 struct rte_swx_port_in_ops ops;
69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
72 TAILQ_ENTRY(port_in) node;
73 struct port_in_type *type;
78 TAILQ_HEAD(port_in_tailq, port_in);
80 struct port_in_runtime {
81 rte_swx_port_in_pkt_rx_t pkt_rx;
88 struct port_out_type {
89 TAILQ_ENTRY(port_out_type) node;
90 char name[RTE_SWX_NAME_SIZE];
91 struct rte_swx_port_out_ops ops;
94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
97 TAILQ_ENTRY(port_out) node;
98 struct port_out_type *type;
103 TAILQ_HEAD(port_out_tailq, port_out);
105 struct port_out_runtime {
106 rte_swx_port_out_pkt_tx_t pkt_tx;
107 rte_swx_port_out_flush_t flush;
114 struct extern_type_member_func {
115 TAILQ_ENTRY(extern_type_member_func) node;
116 char name[RTE_SWX_NAME_SIZE];
117 rte_swx_extern_type_member_func_t func;
121 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
124 TAILQ_ENTRY(extern_type) node;
125 char name[RTE_SWX_NAME_SIZE];
126 struct struct_type *mailbox_struct_type;
127 rte_swx_extern_type_constructor_t constructor;
128 rte_swx_extern_type_destructor_t destructor;
129 struct extern_type_member_func_tailq funcs;
133 TAILQ_HEAD(extern_type_tailq, extern_type);
136 TAILQ_ENTRY(extern_obj) node;
137 char name[RTE_SWX_NAME_SIZE];
138 struct extern_type *type;
144 TAILQ_HEAD(extern_obj_tailq, extern_obj);
146 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
147 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
150 struct extern_obj_runtime {
153 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
160 TAILQ_ENTRY(extern_func) node;
161 char name[RTE_SWX_NAME_SIZE];
162 struct struct_type *mailbox_struct_type;
163 rte_swx_extern_func_t func;
168 TAILQ_HEAD(extern_func_tailq, extern_func);
170 struct extern_func_runtime {
172 rte_swx_extern_func_t func;
179 TAILQ_ENTRY(header) node;
180 char name[RTE_SWX_NAME_SIZE];
181 struct struct_type *st;
186 TAILQ_HEAD(header_tailq, header);
188 struct header_runtime {
193 struct header_out_runtime {
203 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
204 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
205 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
206 * when transferred to packet meta-data and in NBO when transferred to packet
210 /* Notation conventions:
211 * -Header field: H = h.header.field (dst/src)
212 * -Meta-data field: M = m.field (dst/src)
213 * -Extern object mailbox field: E = e.field (dst/src)
214 * -Extern function mailbox field: F = f.field (dst/src)
215 * -Table action data field: T = t.field (src only)
216 * -Immediate value: I = 32-bit unsigned value (src only)
219 enum instruction_type {
226 INSTR_TX, /* port_out = M */
227 INSTR_TX_I, /* port_out = I */
229 /* extract h.header */
239 /* extract h.header m.last_field_size */
242 /* lookahead h.header */
256 /* validate h.header */
259 /* invalidate h.header */
260 INSTR_HDR_INVALIDATE,
264 * dst = HMEF, src = HMEFTI
266 INSTR_MOV, /* dst = MEF, src = MEFT */
267 INSTR_MOV_MH, /* dst = MEF, src = H */
268 INSTR_MOV_HM, /* dst = H, src = MEFT */
269 INSTR_MOV_HH, /* dst = H, src = H */
270 INSTR_MOV_I, /* dst = HMEF, src = I */
272 /* dma h.header t.field
273 * memcpy(h.header, t.field, sizeof(h.header))
286 * dst = HMEF, src = HMEFTI
288 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
289 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
290 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
291 INSTR_ALU_ADD_HH, /* dst = H, src = H */
292 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
293 INSTR_ALU_ADD_HI, /* dst = H, src = I */
297 * dst = HMEF, src = HMEFTI
299 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
300 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
301 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
302 INSTR_ALU_SUB_HH, /* dst = H, src = H */
303 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
304 INSTR_ALU_SUB_HI, /* dst = H, src = I */
307 * dst = dst '+ src[0:1] '+ src[2:3] + ...
308 * dst = H, src = {H, h.header}
310 INSTR_ALU_CKADD_FIELD, /* src = H */
311 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
312 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with any sizeof(header) */
318 INSTR_ALU_CKSUB_FIELD,
322 * dst = HMEF, src = HMEFTI
324 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
325 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
326 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
327 INSTR_ALU_AND_HH, /* dst = H, src = H */
328 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
332 * dst = HMEF, src = HMEFTI
334 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
335 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
336 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
337 INSTR_ALU_OR_HH, /* dst = H, src = H */
338 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
342 * dst = HMEF, src = HMEFTI
344 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
345 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
346 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
347 INSTR_ALU_XOR_HH, /* dst = H, src = H */
348 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
352 * dst = HMEF, src = HMEFTI
354 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
355 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
356 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
357 INSTR_ALU_SHL_HH, /* dst = H, src = H */
358 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
359 INSTR_ALU_SHL_HI, /* dst = H, src = I */
363 * dst = HMEF, src = HMEFTI
365 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
366 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
367 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
368 INSTR_ALU_SHR_HH, /* dst = H, src = H */
369 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
370 INSTR_ALU_SHR_HI, /* dst = H, src = I */
372 /* regprefetch REGARRAY index
373 * prefetch REGARRAY[index]
376 INSTR_REGPREFETCH_RH, /* index = H */
377 INSTR_REGPREFETCH_RM, /* index = MEFT */
378 INSTR_REGPREFETCH_RI, /* index = I */
380 /* regrd dst REGARRAY index
381 * dst = REGARRAY[index]
382 * dst = HMEF, index = HMEFTI
384 INSTR_REGRD_HRH, /* dst = H, index = H */
385 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
386 INSTR_REGRD_HRI, /* dst = H, index = I */
387 INSTR_REGRD_MRH, /* dst = MEF, index = H */
388 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
389 INSTR_REGRD_MRI, /* dst = MEF, index = I */
391 /* regwr REGARRAY index src
392 * REGARRAY[index] = src
393 * index = HMEFTI, src = HMEFTI
395 INSTR_REGWR_RHH, /* index = H, src = H */
396 INSTR_REGWR_RHM, /* index = H, src = MEFT */
397 INSTR_REGWR_RHI, /* index = H, src = I */
398 INSTR_REGWR_RMH, /* index = MEFT, src = H */
399 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
400 INSTR_REGWR_RMI, /* index = MEFT, src = I */
401 INSTR_REGWR_RIH, /* index = I, src = H */
402 INSTR_REGWR_RIM, /* index = I, src = MEFT */
403 INSTR_REGWR_RII, /* index = I, src = I */
405 /* regadd REGARRAY index src
406 * REGARRAY[index] += src
407 * index = HMEFTI, src = HMEFTI
409 INSTR_REGADD_RHH, /* index = H, src = H */
410 INSTR_REGADD_RHM, /* index = H, src = MEFT */
411 INSTR_REGADD_RHI, /* index = H, src = I */
412 INSTR_REGADD_RMH, /* index = MEFT, src = H */
413 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
414 INSTR_REGADD_RMI, /* index = MEFT, src = I */
415 INSTR_REGADD_RIH, /* index = I, src = H */
416 INSTR_REGADD_RIM, /* index = I, src = MEFT */
417 INSTR_REGADD_RII, /* index = I, src = I */
419 /* metprefetch METARRAY index
420 * prefetch METARRAY[index]
423 INSTR_METPREFETCH_H, /* index = H */
424 INSTR_METPREFETCH_M, /* index = MEFT */
425 INSTR_METPREFETCH_I, /* index = I */
427 /* meter METARRAY index length color_in color_out
428 * color_out = meter(METARRAY[index], length, color_in)
429 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
431 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
432 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
433 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
434 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
435 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
436 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
437 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
438 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
439 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
440 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
441 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
442 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
451 /* learn LEARNER ACTION_NAME [ m.action_first_arg ] */
455 INSTR_LEARNER_FORGET,
457 /* extern e.obj.func */
468 /* jmpv LABEL h.header
469 * Jump if header is valid
473 /* jmpnv LABEL h.header
474 * Jump if header is invalid
479 * Jump if table lookup hit
484 * Jump if table lookup miss
491 INSTR_JMP_ACTION_HIT,
493 /* jmpna LABEL ACTION
494 * Jump if action not run
496 INSTR_JMP_ACTION_MISS,
499 * Jump if a is equal to b
500 * a = HMEFT, b = HMEFTI
502 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
503 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
504 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
505 INSTR_JMP_EQ_HH, /* a = H, b = H */
506 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
509 * Jump if a is not equal to b
510 * a = HMEFT, b = HMEFTI
512 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
513 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
514 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
515 INSTR_JMP_NEQ_HH, /* a = H, b = H */
516 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
519 * Jump if a is less than b
520 * a = HMEFT, b = HMEFTI
522 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
523 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
524 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
525 INSTR_JMP_LT_HH, /* a = H, b = H */
526 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
527 INSTR_JMP_LT_HI, /* a = H, b = I */
530 * Jump if a is greater than b
531 * a = HMEFT, b = HMEFTI
533 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
534 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
535 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
536 INSTR_JMP_GT_HH, /* a = H, b = H */
537 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
538 INSTR_JMP_GT_HI, /* a = H, b = I */
545 /* Start of custom instructions. */
549 struct instr_operand {
570 uint8_t header_id[8];
571 uint8_t struct_id[8];
576 struct instr_hdr_validity {
589 struct instr_extern_obj {
594 struct instr_extern_func {
598 struct instr_dst_src {
599 struct instr_operand dst;
601 struct instr_operand src;
606 struct instr_regarray {
611 struct instr_operand idx;
616 struct instr_operand dstsrc;
626 struct instr_operand idx;
630 struct instr_operand length;
633 struct instr_operand color_in;
634 uint32_t color_in_val;
637 struct instr_operand color_out;
642 uint8_t header_id[8];
643 uint8_t struct_id[8];
654 struct instruction *ip;
657 struct instr_operand a;
663 struct instr_operand b;
669 enum instruction_type type;
672 struct instr_hdr_validity valid;
673 struct instr_dst_src mov;
674 struct instr_regarray regarray;
675 struct instr_meter meter;
676 struct instr_dma dma;
677 struct instr_dst_src alu;
678 struct instr_table table;
679 struct instr_learn learn;
680 struct instr_extern_obj ext_obj;
681 struct instr_extern_func ext_func;
682 struct instr_jmp jmp;
686 struct instruction_data {
687 char label[RTE_SWX_NAME_SIZE];
688 char jmp_label[RTE_SWX_NAME_SIZE];
689 uint32_t n_users; /* user = jmp instruction to this instruction. */
693 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
699 (*action_func_t)(struct rte_swx_pipeline *p);
702 TAILQ_ENTRY(action) node;
703 char name[RTE_SWX_NAME_SIZE];
704 struct struct_type *st;
705 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
706 struct instruction *instructions;
707 struct instruction_data *instruction_data;
708 uint32_t n_instructions;
712 TAILQ_HEAD(action_tailq, action);
718 TAILQ_ENTRY(table_type) node;
719 char name[RTE_SWX_NAME_SIZE];
720 enum rte_swx_table_match_type match_type;
721 struct rte_swx_table_ops ops;
724 TAILQ_HEAD(table_type_tailq, table_type);
727 enum rte_swx_table_match_type match_type;
732 TAILQ_ENTRY(table) node;
733 char name[RTE_SWX_NAME_SIZE];
734 char args[RTE_SWX_NAME_SIZE];
735 struct table_type *type; /* NULL when n_fields == 0. */
738 struct match_field *fields;
740 struct header *header; /* Only valid when n_fields > 0. */
743 struct action **actions;
744 struct action *default_action;
745 uint8_t *default_action_data;
747 int default_action_is_const;
748 uint32_t action_data_size_max;
749 int *action_is_for_table_entries;
750 int *action_is_for_default_entry;
756 TAILQ_HEAD(table_tailq, table);
758 struct table_runtime {
759 rte_swx_table_lookup_t func;
764 struct table_statistics {
765 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
766 uint64_t *n_pkts_action;
773 TAILQ_ENTRY(selector) node;
774 char name[RTE_SWX_NAME_SIZE];
776 struct field *group_id_field;
777 struct field **selector_fields;
778 uint32_t n_selector_fields;
779 struct header *selector_header;
780 struct field *member_id_field;
782 uint32_t n_groups_max;
783 uint32_t n_members_per_group_max;
788 TAILQ_HEAD(selector_tailq, selector);
790 struct selector_runtime {
792 uint8_t **group_id_buffer;
793 uint8_t **selector_buffer;
794 uint8_t **member_id_buffer;
797 struct selector_statistics {
805 TAILQ_ENTRY(learner) node;
806 char name[RTE_SWX_NAME_SIZE];
809 struct field **fields;
811 struct header *header;
814 struct action **actions;
815 struct action *default_action;
816 uint8_t *default_action_data;
818 int default_action_is_const;
819 uint32_t action_data_size_max;
820 int *action_is_for_table_entries;
821 int *action_is_for_default_entry;
828 TAILQ_HEAD(learner_tailq, learner);
830 struct learner_runtime {
835 struct learner_statistics {
836 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
837 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
838 uint64_t n_pkts_forget;
839 uint64_t *n_pkts_action;
846 TAILQ_ENTRY(regarray) node;
847 char name[RTE_SWX_NAME_SIZE];
853 TAILQ_HEAD(regarray_tailq, regarray);
855 struct regarray_runtime {
863 struct meter_profile {
864 TAILQ_ENTRY(meter_profile) node;
865 char name[RTE_SWX_NAME_SIZE];
866 struct rte_meter_trtcm_params params;
867 struct rte_meter_trtcm_profile profile;
871 TAILQ_HEAD(meter_profile_tailq, meter_profile);
874 TAILQ_ENTRY(metarray) node;
875 char name[RTE_SWX_NAME_SIZE];
880 TAILQ_HEAD(metarray_tailq, metarray);
883 struct rte_meter_trtcm m;
884 struct meter_profile *profile;
885 enum rte_color color_mask;
888 uint64_t n_pkts[RTE_COLORS];
889 uint64_t n_bytes[RTE_COLORS];
892 struct metarray_runtime {
893 struct meter *metarray;
902 struct rte_swx_pkt pkt;
908 /* Packet headers. */
909 struct header_runtime *headers; /* Extracted or generated headers. */
910 struct header_out_runtime *headers_out; /* Emitted headers. */
911 uint8_t *header_storage;
912 uint8_t *header_out_storage;
913 uint64_t valid_headers;
914 uint32_t n_headers_out;
916 /* Packet meta-data. */
920 struct table_runtime *tables;
921 struct selector_runtime *selectors;
922 struct learner_runtime *learners;
923 struct rte_swx_table_state *table_state;
925 int hit; /* 0 = Miss, 1 = Hit. */
929 /* Extern objects and functions. */
930 struct extern_obj_runtime *extern_objs;
931 struct extern_func_runtime *extern_funcs;
934 struct instruction *ip;
935 struct instruction *ret;
938 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
939 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
940 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
942 #define HEADER_VALID(thread, header_id) \
943 MASK64_BIT_GET((thread)->valid_headers, header_id)
945 static inline uint64_t
946 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
948 uint8_t *x_struct = t->structs[x->struct_id];
949 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
950 uint64_t x64 = *x64_ptr;
951 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
953 return x64 & x64_mask;
956 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
958 static inline uint64_t
959 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
961 uint8_t *x_struct = t->structs[x->struct_id];
962 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
963 uint64_t x64 = *x64_ptr;
965 return ntoh64(x64) >> (64 - x->n_bits);
970 #define instr_operand_nbo instr_operand_hbo
974 #define ALU(thread, ip, operator) \
976 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
977 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
978 uint64_t dst64 = *dst64_ptr; \
979 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
980 uint64_t dst = dst64 & dst64_mask; \
982 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
983 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
984 uint64_t src64 = *src64_ptr; \
985 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
986 uint64_t src = src64 & src64_mask; \
988 uint64_t result = dst operator src; \
990 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
993 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
995 #define ALU_MH(thread, ip, operator) \
997 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
998 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
999 uint64_t dst64 = *dst64_ptr; \
1000 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1001 uint64_t dst = dst64 & dst64_mask; \
1003 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1004 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1005 uint64_t src64 = *src64_ptr; \
1006 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1008 uint64_t result = dst operator src; \
1010 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1013 #define ALU_HM(thread, ip, operator) \
1015 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1016 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1017 uint64_t dst64 = *dst64_ptr; \
1018 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1019 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1021 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1022 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1023 uint64_t src64 = *src64_ptr; \
1024 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1025 uint64_t src = src64 & src64_mask; \
1027 uint64_t result = dst operator src; \
1028 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1030 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1033 #define ALU_HM_FAST(thread, ip, operator) \
1035 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1036 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1037 uint64_t dst64 = *dst64_ptr; \
1038 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1039 uint64_t dst = dst64 & dst64_mask; \
1041 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1042 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1043 uint64_t src64 = *src64_ptr; \
1044 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1045 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1047 uint64_t result = dst operator src; \
1049 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1052 #define ALU_HH(thread, ip, operator) \
1054 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1055 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1056 uint64_t dst64 = *dst64_ptr; \
1057 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1058 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1060 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1061 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1062 uint64_t src64 = *src64_ptr; \
1063 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1065 uint64_t result = dst operator src; \
1066 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1068 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1071 #define ALU_HH_FAST(thread, ip, operator) \
1073 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1074 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1075 uint64_t dst64 = *dst64_ptr; \
1076 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1077 uint64_t dst = dst64 & dst64_mask; \
1079 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1080 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1081 uint64_t src64 = *src64_ptr; \
1082 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1084 uint64_t result = dst operator src; \
1086 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1093 #define ALU_HM_FAST ALU
1095 #define ALU_HH_FAST ALU
1099 #define ALU_I(thread, ip, operator) \
1101 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1102 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1103 uint64_t dst64 = *dst64_ptr; \
1104 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1105 uint64_t dst = dst64 & dst64_mask; \
1107 uint64_t src = (ip)->alu.src_val; \
1109 uint64_t result = dst operator src; \
1111 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1114 #define ALU_MI ALU_I
1116 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1118 #define ALU_HI(thread, ip, operator) \
1120 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1121 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1122 uint64_t dst64 = *dst64_ptr; \
1123 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1124 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1126 uint64_t src = (ip)->alu.src_val; \
1128 uint64_t result = dst operator src; \
1129 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1131 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1136 #define ALU_HI ALU_I
1140 #define MOV(thread, ip) \
1142 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1143 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1144 uint64_t dst64 = *dst64_ptr; \
1145 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1147 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1148 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1149 uint64_t src64 = *src64_ptr; \
1150 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1151 uint64_t src = src64 & src64_mask; \
1153 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1156 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1158 #define MOV_MH(thread, ip) \
1160 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1161 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1162 uint64_t dst64 = *dst64_ptr; \
1163 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1165 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1166 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1167 uint64_t src64 = *src64_ptr; \
1168 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1170 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1173 #define MOV_HM(thread, ip) \
1175 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1176 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1177 uint64_t dst64 = *dst64_ptr; \
1178 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1180 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1181 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1182 uint64_t src64 = *src64_ptr; \
1183 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1184 uint64_t src = src64 & src64_mask; \
1186 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1187 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1190 #define MOV_HH(thread, ip) \
1192 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1193 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1194 uint64_t dst64 = *dst64_ptr; \
1195 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1197 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1198 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1199 uint64_t src64 = *src64_ptr; \
1201 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1202 src = src >> (64 - (ip)->mov.dst.n_bits); \
1203 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1214 #define MOV_I(thread, ip) \
1216 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1217 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1218 uint64_t dst64 = *dst64_ptr; \
1219 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1221 uint64_t src = (ip)->mov.src_val; \
1223 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1226 #define JMP_CMP(thread, ip, operator) \
1228 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1229 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1230 uint64_t a64 = *a64_ptr; \
1231 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1232 uint64_t a = a64 & a64_mask; \
1234 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1235 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1236 uint64_t b64 = *b64_ptr; \
1237 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1238 uint64_t b = b64 & b64_mask; \
1240 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1243 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1245 #define JMP_CMP_MH(thread, ip, operator) \
1247 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1248 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1249 uint64_t a64 = *a64_ptr; \
1250 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1251 uint64_t a = a64 & a64_mask; \
1253 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1254 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1255 uint64_t b64 = *b64_ptr; \
1256 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1258 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1261 #define JMP_CMP_HM(thread, ip, operator) \
1263 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1264 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1265 uint64_t a64 = *a64_ptr; \
1266 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1268 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1269 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1270 uint64_t b64 = *b64_ptr; \
1271 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1272 uint64_t b = b64 & b64_mask; \
1274 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1277 #define JMP_CMP_HH(thread, ip, operator) \
1279 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1280 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1281 uint64_t a64 = *a64_ptr; \
1282 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1284 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1285 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1286 uint64_t b64 = *b64_ptr; \
1287 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1289 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1292 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1294 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1295 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1296 uint64_t a64 = *a64_ptr; \
1297 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1299 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1300 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1301 uint64_t b64 = *b64_ptr; \
1302 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1304 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1309 #define JMP_CMP_MH JMP_CMP
1310 #define JMP_CMP_HM JMP_CMP
1311 #define JMP_CMP_HH JMP_CMP
1312 #define JMP_CMP_HH_FAST JMP_CMP
1316 #define JMP_CMP_I(thread, ip, operator) \
1318 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1319 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1320 uint64_t a64 = *a64_ptr; \
1321 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1322 uint64_t a = a64 & a64_mask; \
1324 uint64_t b = (ip)->jmp.b_val; \
1326 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1329 #define JMP_CMP_MI JMP_CMP_I
1331 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1333 #define JMP_CMP_HI(thread, ip, operator) \
1335 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1336 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1337 uint64_t a64 = *a64_ptr; \
1338 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1340 uint64_t b = (ip)->jmp.b_val; \
1342 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1347 #define JMP_CMP_HI JMP_CMP_I
1351 #define METADATA_READ(thread, offset, n_bits) \
1353 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1354 uint64_t m64 = *m64_ptr; \
1355 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1359 #define METADATA_WRITE(thread, offset, n_bits, value) \
1361 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1362 uint64_t m64 = *m64_ptr; \
1363 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1365 uint64_t m_new = value; \
1367 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1370 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1371 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1374 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1375 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
1378 struct rte_swx_pipeline {
1379 struct struct_type_tailq struct_types;
1380 struct port_in_type_tailq port_in_types;
1381 struct port_in_tailq ports_in;
1382 struct port_out_type_tailq port_out_types;
1383 struct port_out_tailq ports_out;
1384 struct extern_type_tailq extern_types;
1385 struct extern_obj_tailq extern_objs;
1386 struct extern_func_tailq extern_funcs;
1387 struct header_tailq headers;
1388 struct struct_type *metadata_st;
1389 uint32_t metadata_struct_id;
1390 struct action_tailq actions;
1391 struct table_type_tailq table_types;
1392 struct table_tailq tables;
1393 struct selector_tailq selectors;
1394 struct learner_tailq learners;
1395 struct regarray_tailq regarrays;
1396 struct meter_profile_tailq meter_profiles;
1397 struct metarray_tailq metarrays;
1399 struct port_in_runtime *in;
1400 struct port_out_runtime *out;
1401 struct instruction **action_instructions;
1402 action_func_t *action_funcs;
1403 struct rte_swx_table_state *table_state;
1404 struct table_statistics *table_stats;
1405 struct selector_statistics *selector_stats;
1406 struct learner_statistics *learner_stats;
1407 struct regarray_runtime *regarray_runtime;
1408 struct metarray_runtime *metarray_runtime;
1409 struct instruction *instructions;
1410 struct instruction_data *instruction_data;
1411 instr_exec_t *instruction_table;
1412 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1416 uint32_t n_ports_in;
1417 uint32_t n_ports_out;
1418 uint32_t n_extern_objs;
1419 uint32_t n_extern_funcs;
1422 uint32_t n_selectors;
1423 uint32_t n_learners;
1424 uint32_t n_regarrays;
1425 uint32_t n_metarrays;
1429 uint32_t n_instructions;
1438 pipeline_port_inc(struct rte_swx_pipeline *p)
1440 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1444 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1446 t->ip = p->instructions;
1450 thread_ip_set(struct thread *t, struct instruction *ip)
1456 thread_ip_action_call(struct rte_swx_pipeline *p,
1461 t->ip = p->action_instructions[action_id];
1465 thread_ip_inc(struct rte_swx_pipeline *p);
1468 thread_ip_inc(struct rte_swx_pipeline *p)
1470 struct thread *t = &p->threads[p->thread_id];
1476 thread_ip_inc_cond(struct thread *t, int cond)
1482 thread_yield(struct rte_swx_pipeline *p)
1484 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1488 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1490 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1497 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1499 struct port_in_runtime *port = &p->in[p->port_id];
1500 struct rte_swx_pkt *pkt = &t->pkt;
1504 pkt_received = port->pkt_rx(port->obj, pkt);
1505 t->ptr = &pkt->pkt[pkt->offset];
1506 rte_prefetch0(t->ptr);
1508 TRACE("[Thread %2u] rx %s from port %u\n",
1510 pkt_received ? "1 pkt" : "0 pkts",
1514 t->valid_headers = 0;
1515 t->n_headers_out = 0;
1518 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1521 t->table_state = p->table_state;
1524 pipeline_port_inc(p);
1526 return pkt_received;
1530 instr_rx_exec(struct rte_swx_pipeline *p)
1532 struct thread *t = &p->threads[p->thread_id];
1533 struct instruction *ip = t->ip;
1537 pkt_received = __instr_rx_exec(p, t, ip);
1540 thread_ip_inc_cond(t, pkt_received);
1548 emit_handler(struct thread *t)
1550 struct header_out_runtime *h0 = &t->headers_out[0];
1551 struct header_out_runtime *h1 = &t->headers_out[1];
1552 uint32_t offset = 0, i;
1554 /* No header change or header decapsulation. */
1555 if ((t->n_headers_out == 1) &&
1556 (h0->ptr + h0->n_bytes == t->ptr)) {
1557 TRACE("Emit handler: no header change or header decap.\n");
1559 t->pkt.offset -= h0->n_bytes;
1560 t->pkt.length += h0->n_bytes;
1565 /* Header encapsulation (optionally, with prior header decapsulation). */
1566 if ((t->n_headers_out == 2) &&
1567 (h1->ptr + h1->n_bytes == t->ptr) &&
1568 (h0->ptr == h0->ptr0)) {
1571 TRACE("Emit handler: header encapsulation.\n");
1573 offset = h0->n_bytes + h1->n_bytes;
1574 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1575 t->pkt.offset -= offset;
1576 t->pkt.length += offset;
1581 /* For any other case. */
1582 TRACE("Emit handler: complex case.\n");
1584 for (i = 0; i < t->n_headers_out; i++) {
1585 struct header_out_runtime *h = &t->headers_out[i];
1587 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1588 offset += h->n_bytes;
1592 memcpy(t->ptr - offset, t->header_out_storage, offset);
1593 t->pkt.offset -= offset;
1594 t->pkt.length += offset;
1599 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1601 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1602 struct port_out_runtime *port = &p->out[port_id];
1603 struct rte_swx_pkt *pkt = &t->pkt;
1605 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1613 port->pkt_tx(port->obj, pkt);
1617 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1619 uint64_t port_id = ip->io.io.val;
1620 struct port_out_runtime *port = &p->out[port_id];
1621 struct rte_swx_pkt *pkt = &t->pkt;
1623 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1631 port->pkt_tx(port->obj, pkt);
1638 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1640 const struct instruction *ip,
1643 uint64_t valid_headers = t->valid_headers;
1644 uint8_t *ptr = t->ptr;
1645 uint32_t offset = t->pkt.offset;
1646 uint32_t length = t->pkt.length;
1649 for (i = 0; i < n_extract; i++) {
1650 uint32_t header_id = ip->io.hdr.header_id[i];
1651 uint32_t struct_id = ip->io.hdr.struct_id[i];
1652 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1654 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1660 t->structs[struct_id] = ptr;
1661 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1670 t->valid_headers = valid_headers;
1673 t->pkt.offset = offset;
1674 t->pkt.length = length;
1679 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1681 const struct instruction *ip)
1683 __instr_hdr_extract_many_exec(p, t, ip, 1);
1687 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1689 const struct instruction *ip)
1691 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1693 __instr_hdr_extract_many_exec(p, t, ip, 2);
1697 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1699 const struct instruction *ip)
1701 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1703 __instr_hdr_extract_many_exec(p, t, ip, 3);
1707 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1709 const struct instruction *ip)
1711 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1713 __instr_hdr_extract_many_exec(p, t, ip, 4);
1717 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1719 const struct instruction *ip)
1721 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1723 __instr_hdr_extract_many_exec(p, t, ip, 5);
1727 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1729 const struct instruction *ip)
1731 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1733 __instr_hdr_extract_many_exec(p, t, ip, 6);
1737 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1739 const struct instruction *ip)
1741 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1743 __instr_hdr_extract_many_exec(p, t, ip, 7);
1747 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1749 const struct instruction *ip)
1751 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1753 __instr_hdr_extract_many_exec(p, t, ip, 8);
1757 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
1759 const struct instruction *ip)
1761 uint64_t valid_headers = t->valid_headers;
1762 uint8_t *ptr = t->ptr;
1763 uint32_t offset = t->pkt.offset;
1764 uint32_t length = t->pkt.length;
1766 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1767 uint32_t header_id = ip->io.hdr.header_id[0];
1768 uint32_t struct_id = ip->io.hdr.struct_id[0];
1769 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
1771 struct header_runtime *h = &t->headers[header_id];
1773 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
1779 n_bytes += n_bytes_last;
1782 t->structs[struct_id] = ptr;
1783 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1784 h->n_bytes = n_bytes;
1787 t->pkt.offset = offset + n_bytes;
1788 t->pkt.length = length - n_bytes;
1789 t->ptr = ptr + n_bytes;
1793 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
1795 const struct instruction *ip)
1797 uint64_t valid_headers = t->valid_headers;
1798 uint8_t *ptr = t->ptr;
1800 uint32_t header_id = ip->io.hdr.header_id[0];
1801 uint32_t struct_id = ip->io.hdr.struct_id[0];
1803 TRACE("[Thread %2u]: lookahead header %u\n",
1808 t->structs[struct_id] = ptr;
1809 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1816 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
1818 const struct instruction *ip,
1821 uint64_t valid_headers = t->valid_headers;
1822 uint32_t n_headers_out = t->n_headers_out;
1823 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1824 uint8_t *ho_ptr = NULL;
1825 uint32_t ho_nbytes = 0, first = 1, i;
1827 for (i = 0; i < n_emit; i++) {
1828 uint32_t header_id = ip->io.hdr.header_id[i];
1829 uint32_t struct_id = ip->io.hdr.struct_id[i];
1831 struct header_runtime *hi = &t->headers[header_id];
1832 uint8_t *hi_ptr0 = hi->ptr0;
1833 uint32_t n_bytes = hi->n_bytes;
1835 uint8_t *hi_ptr = t->structs[struct_id];
1837 if (!MASK64_BIT_GET(valid_headers, header_id))
1840 TRACE("[Thread %2u]: emit header %u\n",
1848 if (!t->n_headers_out) {
1849 ho = &t->headers_out[0];
1855 ho_nbytes = n_bytes;
1862 ho_nbytes = ho->n_bytes;
1866 if (ho_ptr + ho_nbytes == hi_ptr) {
1867 ho_nbytes += n_bytes;
1869 ho->n_bytes = ho_nbytes;
1876 ho_nbytes = n_bytes;
1882 ho->n_bytes = ho_nbytes;
1883 t->n_headers_out = n_headers_out;
1887 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
1889 const struct instruction *ip)
1891 __instr_hdr_emit_many_exec(p, t, ip, 1);
1895 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
1897 const struct instruction *ip)
1899 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1901 __instr_hdr_emit_many_exec(p, t, ip, 1);
1902 __instr_tx_exec(p, t, ip);
1906 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
1908 const struct instruction *ip)
1910 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1912 __instr_hdr_emit_many_exec(p, t, ip, 2);
1913 __instr_tx_exec(p, t, ip);
1917 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
1919 const struct instruction *ip)
1921 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1923 __instr_hdr_emit_many_exec(p, t, ip, 3);
1924 __instr_tx_exec(p, t, ip);
1928 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
1930 const struct instruction *ip)
1932 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1934 __instr_hdr_emit_many_exec(p, t, ip, 4);
1935 __instr_tx_exec(p, t, ip);
1939 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
1941 const struct instruction *ip)
1943 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1945 __instr_hdr_emit_many_exec(p, t, ip, 5);
1946 __instr_tx_exec(p, t, ip);
1950 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
1952 const struct instruction *ip)
1954 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1956 __instr_hdr_emit_many_exec(p, t, ip, 6);
1957 __instr_tx_exec(p, t, ip);
1961 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
1963 const struct instruction *ip)
1965 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1967 __instr_hdr_emit_many_exec(p, t, ip, 7);
1968 __instr_tx_exec(p, t, ip);
1972 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
1974 const struct instruction *ip)
1976 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
1978 __instr_hdr_emit_many_exec(p, t, ip, 8);
1979 __instr_tx_exec(p, t, ip);
1986 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
1988 const struct instruction *ip)
1990 uint32_t header_id = ip->valid.header_id;
1992 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
1995 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2002 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2004 const struct instruction *ip)
2006 uint32_t header_id = ip->valid.header_id;
2008 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2011 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2018 __instr_learn_exec(struct rte_swx_pipeline *p,
2020 const struct instruction *ip)
2022 uint64_t action_id = ip->learn.action_id;
2023 uint32_t mf_offset = ip->learn.mf_offset;
2024 uint32_t learner_id = t->learner_id;
2025 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2026 p->n_selectors + learner_id];
2027 struct learner_runtime *l = &t->learners[learner_id];
2028 struct learner_statistics *stats = &p->learner_stats[learner_id];
2032 status = rte_swx_table_learner_add(ts->obj,
2036 &t->metadata[mf_offset]);
2038 TRACE("[Thread %2u] learner %u learn %s\n",
2041 status ? "ok" : "error");
2043 stats->n_pkts_learn[status] += 1;
2050 __instr_forget_exec(struct rte_swx_pipeline *p,
2052 const struct instruction *ip __rte_unused)
2054 uint32_t learner_id = t->learner_id;
2055 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2056 p->n_selectors + learner_id];
2057 struct learner_runtime *l = &t->learners[learner_id];
2058 struct learner_statistics *stats = &p->learner_stats[learner_id];
2061 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2063 TRACE("[Thread %2u] learner %u forget\n",
2067 stats->n_pkts_forget += 1;
2073 static inline uint32_t
2074 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2076 const struct instruction *ip)
2078 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2079 uint32_t func_id = ip->ext_obj.func_id;
2080 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2081 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2084 TRACE("[Thread %2u] extern obj %u member func %u\n",
2089 done = func(obj->obj, obj->mailbox);
2094 static inline uint32_t
2095 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2097 const struct instruction *ip)
2099 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2100 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2101 rte_swx_extern_func_t func = ext_func->func;
2104 TRACE("[Thread %2u] extern func %u\n",
2108 done = func(ext_func->mailbox);
2117 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2119 const struct instruction *ip)
2121 TRACE("[Thread %2u] mov\n", p->thread_id);
2127 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2129 const struct instruction *ip)
2131 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2137 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2139 const struct instruction *ip)
2141 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2147 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2149 const struct instruction *ip)
2151 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2157 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2159 const struct instruction *ip)
2161 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2170 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2172 const struct instruction *ip,
2175 uint8_t *action_data = t->structs[0];
2176 uint64_t valid_headers = t->valid_headers;
2179 for (i = 0; i < n_dma; i++) {
2180 uint32_t header_id = ip->dma.dst.header_id[i];
2181 uint32_t struct_id = ip->dma.dst.struct_id[i];
2182 uint32_t offset = ip->dma.src.offset[i];
2183 uint32_t n_bytes = ip->dma.n_bytes[i];
2185 struct header_runtime *h = &t->headers[header_id];
2186 uint8_t *h_ptr0 = h->ptr0;
2187 uint8_t *h_ptr = t->structs[struct_id];
2189 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2191 void *src = &action_data[offset];
2193 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2196 memcpy(dst, src, n_bytes);
2197 t->structs[struct_id] = dst;
2198 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2201 t->valid_headers = valid_headers;
2205 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2207 __instr_dma_ht_many_exec(p, t, ip, 1);
2211 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2213 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2215 __instr_dma_ht_many_exec(p, t, ip, 2);
2219 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2221 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2223 __instr_dma_ht_many_exec(p, t, ip, 3);
2227 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2229 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2231 __instr_dma_ht_many_exec(p, t, ip, 4);
2235 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2237 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2239 __instr_dma_ht_many_exec(p, t, ip, 5);
2243 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2245 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2247 __instr_dma_ht_many_exec(p, t, ip, 6);
2251 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2253 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2255 __instr_dma_ht_many_exec(p, t, ip, 7);
2259 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2261 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2263 __instr_dma_ht_many_exec(p, t, ip, 8);
2270 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2272 const struct instruction *ip)
2274 TRACE("[Thread %2u] add\n", p->thread_id);
2280 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2282 const struct instruction *ip)
2284 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2290 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2292 const struct instruction *ip)
2294 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2300 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2302 const struct instruction *ip)
2304 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2310 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2312 const struct instruction *ip)
2314 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2320 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2322 const struct instruction *ip)
2324 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2330 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2332 const struct instruction *ip)
2334 TRACE("[Thread %2u] sub\n", p->thread_id);
2340 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2342 const struct instruction *ip)
2344 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2350 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2352 const struct instruction *ip)
2354 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2360 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2362 const struct instruction *ip)
2364 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2370 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2372 const struct instruction *ip)
2374 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2380 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2382 const struct instruction *ip)
2384 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2390 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2392 const struct instruction *ip)
2394 TRACE("[Thread %2u] shl\n", p->thread_id);
2400 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2402 const struct instruction *ip)
2404 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2410 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2412 const struct instruction *ip)
2414 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2420 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2422 const struct instruction *ip)
2424 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2430 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2432 const struct instruction *ip)
2434 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2440 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2442 const struct instruction *ip)
2444 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2450 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2452 const struct instruction *ip)
2454 TRACE("[Thread %2u] shr\n", p->thread_id);
2460 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2462 const struct instruction *ip)
2464 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2470 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2472 const struct instruction *ip)
2474 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2480 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2482 const struct instruction *ip)
2484 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2490 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2492 const struct instruction *ip)
2494 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2501 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2503 const struct instruction *ip)
2505 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2511 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2513 const struct instruction *ip)
2515 TRACE("[Thread %2u] and\n", p->thread_id);
2521 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2523 const struct instruction *ip)
2525 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2531 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2533 const struct instruction *ip)
2535 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2537 ALU_HM_FAST(t, ip, &);
2541 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2543 const struct instruction *ip)
2545 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2547 ALU_HH_FAST(t, ip, &);
2551 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2553 const struct instruction *ip)
2555 TRACE("[Thread %2u] and (i)\n", p->thread_id);
2561 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2563 const struct instruction *ip)
2565 TRACE("[Thread %2u] or\n", p->thread_id);
2571 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2573 const struct instruction *ip)
2575 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2581 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2583 const struct instruction *ip)
2585 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
2587 ALU_HM_FAST(t, ip, |);
2591 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2593 const struct instruction *ip)
2595 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
2597 ALU_HH_FAST(t, ip, |);
2601 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
2603 const struct instruction *ip)
2605 TRACE("[Thread %2u] or (i)\n", p->thread_id);
2611 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
2613 const struct instruction *ip)
2615 TRACE("[Thread %2u] xor\n", p->thread_id);
2621 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2623 const struct instruction *ip)
2625 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
2631 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2633 const struct instruction *ip)
2635 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
2637 ALU_HM_FAST(t, ip, ^);
2641 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2643 const struct instruction *ip)
2645 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
2647 ALU_HH_FAST(t, ip, ^);
2651 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
2653 const struct instruction *ip)
2655 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
2661 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
2663 const struct instruction *ip)
2665 uint8_t *dst_struct, *src_struct;
2666 uint16_t *dst16_ptr, dst;
2667 uint64_t *src64_ptr, src64, src64_mask, src;
2670 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
2673 dst_struct = t->structs[ip->alu.dst.struct_id];
2674 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2677 src_struct = t->structs[ip->alu.src.struct_id];
2678 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2680 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2681 src = src64 & src64_mask;
2686 /* The first input (r) is a 16-bit number. The second and the third
2687 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
2688 * three numbers (output r) is a 34-bit number.
2690 r += (src >> 32) + (src & 0xFFFFFFFF);
2692 /* The first input is a 16-bit number. The second input is an 18-bit
2693 * number. In the worst case scenario, the sum of the two numbers is a
2696 r = (r & 0xFFFF) + (r >> 16);
2698 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2699 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
2701 r = (r & 0xFFFF) + (r >> 16);
2703 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2704 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2705 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
2706 * therefore the output r is always a 16-bit number.
2708 r = (r & 0xFFFF) + (r >> 16);
2713 *dst16_ptr = (uint16_t)r;
2717 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
2719 const struct instruction *ip)
2721 uint8_t *dst_struct, *src_struct;
2722 uint16_t *dst16_ptr, dst;
2723 uint64_t *src64_ptr, src64, src64_mask, src;
2726 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
2729 dst_struct = t->structs[ip->alu.dst.struct_id];
2730 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2733 src_struct = t->structs[ip->alu.src.struct_id];
2734 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2736 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2737 src = src64 & src64_mask;
2742 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
2743 * the following sequence of operations in 2's complement arithmetic:
2744 * a '- b = (a - b) % 0xFFFF.
2746 * In order to prevent an underflow for the below subtraction, in which
2747 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
2748 * minuend), we first add a multiple of the 0xFFFF modulus to the
2749 * minuend. The number we add to the minuend needs to be a 34-bit number
2750 * or higher, so for readability reasons we picked the 36-bit multiple.
2751 * We are effectively turning the 16-bit minuend into a 36-bit number:
2752 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
2754 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
2756 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
2757 * result (the output r) is a 36-bit number.
2759 r -= (src >> 32) + (src & 0xFFFFFFFF);
2761 /* The first input is a 16-bit number. The second input is a 20-bit
2762 * number. Their sum is a 21-bit number.
2764 r = (r & 0xFFFF) + (r >> 16);
2766 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2767 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
2769 r = (r & 0xFFFF) + (r >> 16);
2771 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2772 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2773 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
2774 * generated, therefore the output r is always a 16-bit number.
2776 r = (r & 0xFFFF) + (r >> 16);
2781 *dst16_ptr = (uint16_t)r;
2785 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
2787 const struct instruction *ip)
2789 uint8_t *dst_struct, *src_struct;
2790 uint16_t *dst16_ptr;
2791 uint32_t *src32_ptr;
2794 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
2797 dst_struct = t->structs[ip->alu.dst.struct_id];
2798 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2800 src_struct = t->structs[ip->alu.src.struct_id];
2801 src32_ptr = (uint32_t *)&src_struct[0];
2803 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
2804 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
2805 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
2806 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
2807 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
2809 /* The first input is a 16-bit number. The second input is a 19-bit
2810 * number. Their sum is a 20-bit number.
2812 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2814 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2815 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
2817 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2819 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2820 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2821 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
2822 * generated, therefore the output r is always a 16-bit number.
2824 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2827 r0 = r0 ? r0 : 0xFFFF;
2829 *dst16_ptr = (uint16_t)r0;
2833 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
2835 const struct instruction *ip)
2837 uint8_t *dst_struct, *src_struct;
2838 uint16_t *dst16_ptr;
2839 uint32_t *src32_ptr;
2843 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
2846 dst_struct = t->structs[ip->alu.dst.struct_id];
2847 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2849 src_struct = t->structs[ip->alu.src.struct_id];
2850 src32_ptr = (uint32_t *)&src_struct[0];
2852 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
2853 * Therefore, in the worst case scenario, a 35-bit number is added to a
2854 * 16-bit number (the input r), so the output r is 36-bit number.
2856 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
2859 /* The first input is a 16-bit number. The second input is a 20-bit
2860 * number. Their sum is a 21-bit number.
2862 r = (r & 0xFFFF) + (r >> 16);
2864 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2865 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
2867 r = (r & 0xFFFF) + (r >> 16);
2869 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2870 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2871 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
2872 * generated, therefore the output r is always a 16-bit number.
2874 r = (r & 0xFFFF) + (r >> 16);
2879 *dst16_ptr = (uint16_t)r;
2885 static inline uint64_t *
2886 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
2888 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2892 static inline uint64_t
2893 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2895 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2897 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
2898 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
2899 uint64_t idx64 = *idx64_ptr;
2900 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
2901 uint64_t idx = idx64 & idx64_mask & r->size_mask;
2906 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2908 static inline uint64_t
2909 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2911 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2913 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
2914 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
2915 uint64_t idx64 = *idx64_ptr;
2916 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
2923 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
2927 static inline uint64_t
2928 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
2930 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2932 uint64_t idx = ip->regarray.idx_val & r->size_mask;
2937 static inline uint64_t
2938 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
2940 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
2941 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
2942 uint64_t src64 = *src64_ptr;
2943 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2944 uint64_t src = src64 & src64_mask;
2949 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2951 static inline uint64_t
2952 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
2954 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
2955 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
2956 uint64_t src64 = *src64_ptr;
2957 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
2964 #define instr_regarray_src_nbo instr_regarray_src_hbo
2969 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
2971 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
2972 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
2973 uint64_t dst64 = *dst64_ptr;
2974 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2976 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
2980 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2983 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
2985 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
2986 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
2987 uint64_t dst64 = *dst64_ptr;
2988 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2990 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
2991 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
2996 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3001 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3003 const struct instruction *ip)
3005 uint64_t *regarray, idx;
3007 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3009 regarray = instr_regarray_regarray(p, ip);
3010 idx = instr_regarray_idx_nbo(p, t, ip);
3011 rte_prefetch0(®array[idx]);
3015 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3017 const struct instruction *ip)
3019 uint64_t *regarray, idx;
3021 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3023 regarray = instr_regarray_regarray(p, ip);
3024 idx = instr_regarray_idx_hbo(p, t, ip);
3025 rte_prefetch0(®array[idx]);
3029 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3030 struct thread *t __rte_unused,
3031 const struct instruction *ip)
3033 uint64_t *regarray, idx;
3035 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3037 regarray = instr_regarray_regarray(p, ip);
3038 idx = instr_regarray_idx_imm(p, ip);
3039 rte_prefetch0(®array[idx]);
3043 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3045 const struct instruction *ip)
3047 uint64_t *regarray, idx;
3049 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3051 regarray = instr_regarray_regarray(p, ip);
3052 idx = instr_regarray_idx_nbo(p, t, ip);
3053 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3057 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3059 const struct instruction *ip)
3061 uint64_t *regarray, idx;
3063 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3066 regarray = instr_regarray_regarray(p, ip);
3067 idx = instr_regarray_idx_hbo(p, t, ip);
3068 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3072 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3074 uint64_t *regarray, idx;
3076 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3078 regarray = instr_regarray_regarray(p, ip);
3079 idx = instr_regarray_idx_nbo(p, t, ip);
3080 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3084 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3086 uint64_t *regarray, idx;
3088 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3090 regarray = instr_regarray_regarray(p, ip);
3091 idx = instr_regarray_idx_hbo(p, t, ip);
3092 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3096 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3098 uint64_t *regarray, idx;
3100 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3102 regarray = instr_regarray_regarray(p, ip);
3103 idx = instr_regarray_idx_imm(p, ip);
3104 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3108 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3110 uint64_t *regarray, idx;
3112 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3114 regarray = instr_regarray_regarray(p, ip);
3115 idx = instr_regarray_idx_imm(p, ip);
3116 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3120 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3122 uint64_t *regarray, idx, src;
3124 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3126 regarray = instr_regarray_regarray(p, ip);
3127 idx = instr_regarray_idx_nbo(p, t, ip);
3128 src = instr_regarray_src_nbo(t, ip);
3129 regarray[idx] = src;
3133 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3135 uint64_t *regarray, idx, src;
3137 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3139 regarray = instr_regarray_regarray(p, ip);
3140 idx = instr_regarray_idx_nbo(p, t, ip);
3141 src = instr_regarray_src_hbo(t, ip);
3142 regarray[idx] = src;
3146 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3148 uint64_t *regarray, idx, src;
3150 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3152 regarray = instr_regarray_regarray(p, ip);
3153 idx = instr_regarray_idx_hbo(p, t, ip);
3154 src = instr_regarray_src_nbo(t, ip);
3155 regarray[idx] = src;
3159 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3161 uint64_t *regarray, idx, src;
3163 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3165 regarray = instr_regarray_regarray(p, ip);
3166 idx = instr_regarray_idx_hbo(p, t, ip);
3167 src = instr_regarray_src_hbo(t, ip);
3168 regarray[idx] = src;
3172 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3174 uint64_t *regarray, idx, src;
3176 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3178 regarray = instr_regarray_regarray(p, ip);
3179 idx = instr_regarray_idx_nbo(p, t, ip);
3180 src = ip->regarray.dstsrc_val;
3181 regarray[idx] = src;
3185 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3187 uint64_t *regarray, idx, src;
3189 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3191 regarray = instr_regarray_regarray(p, ip);
3192 idx = instr_regarray_idx_hbo(p, t, ip);
3193 src = ip->regarray.dstsrc_val;
3194 regarray[idx] = src;
3198 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3200 uint64_t *regarray, idx, src;
3202 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3204 regarray = instr_regarray_regarray(p, ip);
3205 idx = instr_regarray_idx_imm(p, ip);
3206 src = instr_regarray_src_nbo(t, ip);
3207 regarray[idx] = src;
3211 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3213 uint64_t *regarray, idx, src;
3215 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3217 regarray = instr_regarray_regarray(p, ip);
3218 idx = instr_regarray_idx_imm(p, ip);
3219 src = instr_regarray_src_hbo(t, ip);
3220 regarray[idx] = src;
3224 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3225 struct thread *t __rte_unused,
3226 const struct instruction *ip)
3228 uint64_t *regarray, idx, src;
3230 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3232 regarray = instr_regarray_regarray(p, ip);
3233 idx = instr_regarray_idx_imm(p, ip);
3234 src = ip->regarray.dstsrc_val;
3235 regarray[idx] = src;
3239 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3241 uint64_t *regarray, idx, src;
3243 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3245 regarray = instr_regarray_regarray(p, ip);
3246 idx = instr_regarray_idx_nbo(p, t, ip);
3247 src = instr_regarray_src_nbo(t, ip);
3248 regarray[idx] += src;
3252 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3254 uint64_t *regarray, idx, src;
3256 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3258 regarray = instr_regarray_regarray(p, ip);
3259 idx = instr_regarray_idx_nbo(p, t, ip);
3260 src = instr_regarray_src_hbo(t, ip);
3261 regarray[idx] += src;
3265 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3267 uint64_t *regarray, idx, src;
3269 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3271 regarray = instr_regarray_regarray(p, ip);
3272 idx = instr_regarray_idx_hbo(p, t, ip);
3273 src = instr_regarray_src_nbo(t, ip);
3274 regarray[idx] += src;
3278 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3280 uint64_t *regarray, idx, src;
3282 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3284 regarray = instr_regarray_regarray(p, ip);
3285 idx = instr_regarray_idx_hbo(p, t, ip);
3286 src = instr_regarray_src_hbo(t, ip);
3287 regarray[idx] += src;
3291 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3293 uint64_t *regarray, idx, src;
3295 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3297 regarray = instr_regarray_regarray(p, ip);
3298 idx = instr_regarray_idx_nbo(p, t, ip);
3299 src = ip->regarray.dstsrc_val;
3300 regarray[idx] += src;
3304 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3306 uint64_t *regarray, idx, src;
3308 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3310 regarray = instr_regarray_regarray(p, ip);
3311 idx = instr_regarray_idx_hbo(p, t, ip);
3312 src = ip->regarray.dstsrc_val;
3313 regarray[idx] += src;
3317 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3319 uint64_t *regarray, idx, src;
3321 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3323 regarray = instr_regarray_regarray(p, ip);
3324 idx = instr_regarray_idx_imm(p, ip);
3325 src = instr_regarray_src_nbo(t, ip);
3326 regarray[idx] += src;
3330 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3332 uint64_t *regarray, idx, src;
3334 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3336 regarray = instr_regarray_regarray(p, ip);
3337 idx = instr_regarray_idx_imm(p, ip);
3338 src = instr_regarray_src_hbo(t, ip);
3339 regarray[idx] += src;
3343 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3344 struct thread *t __rte_unused,
3345 const struct instruction *ip)
3347 uint64_t *regarray, idx, src;
3349 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3351 regarray = instr_regarray_regarray(p, ip);
3352 idx = instr_regarray_idx_imm(p, ip);
3353 src = ip->regarray.dstsrc_val;
3354 regarray[idx] += src;
3360 static inline struct meter *
3361 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3363 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3365 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3366 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3367 uint64_t idx64 = *idx64_ptr;
3368 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3369 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3371 return &r->metarray[idx];
3374 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3376 static inline struct meter *
3377 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3379 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3381 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3382 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3383 uint64_t idx64 = *idx64_ptr;
3384 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3386 return &r->metarray[idx];
3391 #define instr_meter_idx_nbo instr_meter_idx_hbo
3395 static inline struct meter *
3396 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3398 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3400 uint64_t idx = ip->meter.idx_val & r->size_mask;
3402 return &r->metarray[idx];
3405 static inline uint32_t
3406 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3408 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3409 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3410 uint64_t src64 = *src64_ptr;
3411 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3412 uint64_t src = src64 & src64_mask;
3414 return (uint32_t)src;
3417 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3419 static inline uint32_t
3420 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3422 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3423 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3424 uint64_t src64 = *src64_ptr;
3425 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3427 return (uint32_t)src;
3432 #define instr_meter_length_nbo instr_meter_length_hbo
3436 static inline enum rte_color
3437 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3439 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3440 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3441 uint64_t src64 = *src64_ptr;
3442 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3443 uint64_t src = src64 & src64_mask;
3445 return (enum rte_color)src;
3449 instr_meter_color_out_hbo_set(struct thread *t,
3450 const struct instruction *ip,
3451 enum rte_color color_out)
3453 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3454 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3455 uint64_t dst64 = *dst64_ptr;
3456 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3458 uint64_t src = (uint64_t)color_out;
3460 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3464 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3466 const struct instruction *ip)
3470 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3472 m = instr_meter_idx_nbo(p, t, ip);
3477 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3479 const struct instruction *ip)
3483 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3485 m = instr_meter_idx_hbo(p, t, ip);
3490 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3491 struct thread *t __rte_unused,
3492 const struct instruction *ip)
3496 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3498 m = instr_meter_idx_imm(p, ip);
3503 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3506 uint64_t time, n_pkts, n_bytes;
3508 enum rte_color color_in, color_out;
3510 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3512 m = instr_meter_idx_nbo(p, t, ip);
3513 rte_prefetch0(m->n_pkts);
3514 time = rte_get_tsc_cycles();
3515 length = instr_meter_length_nbo(t, ip);
3516 color_in = instr_meter_color_in_hbo(t, ip);
3518 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3519 &m->profile->profile,
3524 color_out &= m->color_mask;
3526 n_pkts = m->n_pkts[color_out];
3527 n_bytes = m->n_bytes[color_out];
3529 instr_meter_color_out_hbo_set(t, ip, color_out);
3531 m->n_pkts[color_out] = n_pkts + 1;
3532 m->n_bytes[color_out] = n_bytes + length;
3536 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3539 uint64_t time, n_pkts, n_bytes;
3541 enum rte_color color_in, color_out;
3543 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3545 m = instr_meter_idx_nbo(p, t, ip);
3546 rte_prefetch0(m->n_pkts);
3547 time = rte_get_tsc_cycles();
3548 length = instr_meter_length_nbo(t, ip);
3549 color_in = (enum rte_color)ip->meter.color_in_val;
3551 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3552 &m->profile->profile,
3557 color_out &= m->color_mask;
3559 n_pkts = m->n_pkts[color_out];
3560 n_bytes = m->n_bytes[color_out];
3562 instr_meter_color_out_hbo_set(t, ip, color_out);
3564 m->n_pkts[color_out] = n_pkts + 1;
3565 m->n_bytes[color_out] = n_bytes + length;
3569 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3572 uint64_t time, n_pkts, n_bytes;
3574 enum rte_color color_in, color_out;
3576 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
3578 m = instr_meter_idx_nbo(p, t, ip);
3579 rte_prefetch0(m->n_pkts);
3580 time = rte_get_tsc_cycles();
3581 length = instr_meter_length_hbo(t, ip);
3582 color_in = instr_meter_color_in_hbo(t, ip);
3584 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3585 &m->profile->profile,
3590 color_out &= m->color_mask;
3592 n_pkts = m->n_pkts[color_out];
3593 n_bytes = m->n_bytes[color_out];
3595 instr_meter_color_out_hbo_set(t, ip, color_out);
3597 m->n_pkts[color_out] = n_pkts + 1;
3598 m->n_bytes[color_out] = n_bytes + length;
3602 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3605 uint64_t time, n_pkts, n_bytes;
3607 enum rte_color color_in, color_out;
3609 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
3611 m = instr_meter_idx_nbo(p, t, ip);
3612 rte_prefetch0(m->n_pkts);
3613 time = rte_get_tsc_cycles();
3614 length = instr_meter_length_hbo(t, ip);
3615 color_in = (enum rte_color)ip->meter.color_in_val;
3617 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3618 &m->profile->profile,
3623 color_out &= m->color_mask;
3625 n_pkts = m->n_pkts[color_out];
3626 n_bytes = m->n_bytes[color_out];
3628 instr_meter_color_out_hbo_set(t, ip, color_out);
3630 m->n_pkts[color_out] = n_pkts + 1;
3631 m->n_bytes[color_out] = n_bytes + length;
3635 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3638 uint64_t time, n_pkts, n_bytes;
3640 enum rte_color color_in, color_out;
3642 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
3644 m = instr_meter_idx_hbo(p, t, ip);
3645 rte_prefetch0(m->n_pkts);
3646 time = rte_get_tsc_cycles();
3647 length = instr_meter_length_nbo(t, ip);
3648 color_in = instr_meter_color_in_hbo(t, ip);
3650 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3651 &m->profile->profile,
3656 color_out &= m->color_mask;
3658 n_pkts = m->n_pkts[color_out];
3659 n_bytes = m->n_bytes[color_out];
3661 instr_meter_color_out_hbo_set(t, ip, color_out);
3663 m->n_pkts[color_out] = n_pkts + 1;
3664 m->n_bytes[color_out] = n_bytes + length;
3668 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3671 uint64_t time, n_pkts, n_bytes;
3673 enum rte_color color_in, color_out;
3675 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
3677 m = instr_meter_idx_hbo(p, t, ip);
3678 rte_prefetch0(m->n_pkts);
3679 time = rte_get_tsc_cycles();
3680 length = instr_meter_length_nbo(t, ip);
3681 color_in = (enum rte_color)ip->meter.color_in_val;
3683 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3684 &m->profile->profile,
3689 color_out &= m->color_mask;
3691 n_pkts = m->n_pkts[color_out];
3692 n_bytes = m->n_bytes[color_out];
3694 instr_meter_color_out_hbo_set(t, ip, color_out);
3696 m->n_pkts[color_out] = n_pkts + 1;
3697 m->n_bytes[color_out] = n_bytes + length;
3701 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3704 uint64_t time, n_pkts, n_bytes;
3706 enum rte_color color_in, color_out;
3708 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
3710 m = instr_meter_idx_hbo(p, t, ip);
3711 rte_prefetch0(m->n_pkts);
3712 time = rte_get_tsc_cycles();
3713 length = instr_meter_length_hbo(t, ip);
3714 color_in = instr_meter_color_in_hbo(t, ip);
3716 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3717 &m->profile->profile,
3722 color_out &= m->color_mask;
3724 n_pkts = m->n_pkts[color_out];
3725 n_bytes = m->n_bytes[color_out];
3727 instr_meter_color_out_hbo_set(t, ip, color_out);
3729 m->n_pkts[color_out] = n_pkts + 1;
3730 m->n_bytes[color_out] = n_bytes + length;
3734 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3737 uint64_t time, n_pkts, n_bytes;
3739 enum rte_color color_in, color_out;
3741 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
3743 m = instr_meter_idx_hbo(p, t, ip);
3744 rte_prefetch0(m->n_pkts);
3745 time = rte_get_tsc_cycles();
3746 length = instr_meter_length_hbo(t, ip);
3747 color_in = (enum rte_color)ip->meter.color_in_val;
3749 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3750 &m->profile->profile,
3755 color_out &= m->color_mask;
3757 n_pkts = m->n_pkts[color_out];
3758 n_bytes = m->n_bytes[color_out];
3760 instr_meter_color_out_hbo_set(t, ip, color_out);
3762 m->n_pkts[color_out] = n_pkts + 1;
3763 m->n_bytes[color_out] = n_bytes + length;
3767 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3770 uint64_t time, n_pkts, n_bytes;
3772 enum rte_color color_in, color_out;
3774 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
3776 m = instr_meter_idx_imm(p, ip);
3777 rte_prefetch0(m->n_pkts);
3778 time = rte_get_tsc_cycles();
3779 length = instr_meter_length_nbo(t, ip);
3780 color_in = instr_meter_color_in_hbo(t, ip);
3782 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3783 &m->profile->profile,
3788 color_out &= m->color_mask;
3790 n_pkts = m->n_pkts[color_out];
3791 n_bytes = m->n_bytes[color_out];
3793 instr_meter_color_out_hbo_set(t, ip, color_out);
3795 m->n_pkts[color_out] = n_pkts + 1;
3796 m->n_bytes[color_out] = n_bytes + length;
3800 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3803 uint64_t time, n_pkts, n_bytes;
3805 enum rte_color color_in, color_out;
3807 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
3809 m = instr_meter_idx_imm(p, ip);
3810 rte_prefetch0(m->n_pkts);
3811 time = rte_get_tsc_cycles();
3812 length = instr_meter_length_nbo(t, ip);
3813 color_in = (enum rte_color)ip->meter.color_in_val;
3815 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3816 &m->profile->profile,
3821 color_out &= m->color_mask;
3823 n_pkts = m->n_pkts[color_out];
3824 n_bytes = m->n_bytes[color_out];
3826 instr_meter_color_out_hbo_set(t, ip, color_out);
3828 m->n_pkts[color_out] = n_pkts + 1;
3829 m->n_bytes[color_out] = n_bytes + length;
3833 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3836 uint64_t time, n_pkts, n_bytes;
3838 enum rte_color color_in, color_out;
3840 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
3842 m = instr_meter_idx_imm(p, ip);
3843 rte_prefetch0(m->n_pkts);
3844 time = rte_get_tsc_cycles();
3845 length = instr_meter_length_hbo(t, ip);
3846 color_in = instr_meter_color_in_hbo(t, ip);
3848 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3849 &m->profile->profile,
3854 color_out &= m->color_mask;
3856 n_pkts = m->n_pkts[color_out];
3857 n_bytes = m->n_bytes[color_out];
3859 instr_meter_color_out_hbo_set(t, ip, color_out);
3861 m->n_pkts[color_out] = n_pkts + 1;
3862 m->n_bytes[color_out] = n_bytes + length;
3866 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3869 uint64_t time, n_pkts, n_bytes;
3871 enum rte_color color_in, color_out;
3873 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
3875 m = instr_meter_idx_imm(p, ip);
3876 rte_prefetch0(m->n_pkts);
3877 time = rte_get_tsc_cycles();
3878 length = instr_meter_length_hbo(t, ip);
3879 color_in = (enum rte_color)ip->meter.color_in_val;
3881 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3882 &m->profile->profile,
3887 color_out &= m->color_mask;
3889 n_pkts = m->n_pkts[color_out];
3890 n_bytes = m->n_bytes[color_out];
3892 instr_meter_color_out_hbo_set(t, ip, color_out);
3894 m->n_pkts[color_out] = n_pkts + 1;
3895 m->n_bytes[color_out] = n_bytes + length;