1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
11 #include <rte_byteorder.h>
12 #include <rte_common.h>
13 #include <rte_cycles.h>
14 #include <rte_prefetch.h>
15 #include <rte_meter.h>
17 #include <rte_swx_table_selector.h>
18 #include <rte_swx_table_learner.h>
19 #include <rte_swx_pipeline.h>
20 #include <rte_swx_ctl.h>
27 #define TRACE(...) printf(__VA_ARGS__)
35 #define ntoh64(x) rte_be_to_cpu_64(x)
36 #define hton64(x) rte_cpu_to_be_64(x)
42 char name[RTE_SWX_NAME_SIZE];
49 TAILQ_ENTRY(struct_type) node;
50 char name[RTE_SWX_NAME_SIZE];
58 TAILQ_HEAD(struct_type_tailq, struct_type);
64 TAILQ_ENTRY(port_in_type) node;
65 char name[RTE_SWX_NAME_SIZE];
66 struct rte_swx_port_in_ops ops;
69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
72 TAILQ_ENTRY(port_in) node;
73 struct port_in_type *type;
78 TAILQ_HEAD(port_in_tailq, port_in);
80 struct port_in_runtime {
81 rte_swx_port_in_pkt_rx_t pkt_rx;
88 struct port_out_type {
89 TAILQ_ENTRY(port_out_type) node;
90 char name[RTE_SWX_NAME_SIZE];
91 struct rte_swx_port_out_ops ops;
94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
97 TAILQ_ENTRY(port_out) node;
98 struct port_out_type *type;
103 TAILQ_HEAD(port_out_tailq, port_out);
105 struct port_out_runtime {
106 rte_swx_port_out_pkt_tx_t pkt_tx;
107 rte_swx_port_out_flush_t flush;
114 struct extern_type_member_func {
115 TAILQ_ENTRY(extern_type_member_func) node;
116 char name[RTE_SWX_NAME_SIZE];
117 rte_swx_extern_type_member_func_t func;
121 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
124 TAILQ_ENTRY(extern_type) node;
125 char name[RTE_SWX_NAME_SIZE];
126 struct struct_type *mailbox_struct_type;
127 rte_swx_extern_type_constructor_t constructor;
128 rte_swx_extern_type_destructor_t destructor;
129 struct extern_type_member_func_tailq funcs;
133 TAILQ_HEAD(extern_type_tailq, extern_type);
136 TAILQ_ENTRY(extern_obj) node;
137 char name[RTE_SWX_NAME_SIZE];
138 struct extern_type *type;
144 TAILQ_HEAD(extern_obj_tailq, extern_obj);
146 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
147 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
150 struct extern_obj_runtime {
153 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
160 TAILQ_ENTRY(extern_func) node;
161 char name[RTE_SWX_NAME_SIZE];
162 struct struct_type *mailbox_struct_type;
163 rte_swx_extern_func_t func;
168 TAILQ_HEAD(extern_func_tailq, extern_func);
170 struct extern_func_runtime {
172 rte_swx_extern_func_t func;
179 TAILQ_ENTRY(header) node;
180 char name[RTE_SWX_NAME_SIZE];
181 struct struct_type *st;
186 TAILQ_HEAD(header_tailq, header);
188 struct header_runtime {
193 struct header_out_runtime {
203 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
204 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
205 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
206 * when transferred to packet meta-data and in NBO when transferred to packet
210 /* Notation conventions:
211 * -Header field: H = h.header.field (dst/src)
212 * -Meta-data field: M = m.field (dst/src)
213 * -Extern object mailbox field: E = e.field (dst/src)
214 * -Extern function mailbox field: F = f.field (dst/src)
215 * -Table action data field: T = t.field (src only)
216 * -Immediate value: I = 32-bit unsigned value (src only)
219 enum instruction_type {
226 INSTR_TX, /* port_out = M */
227 INSTR_TX_I, /* port_out = I */
229 /* extract h.header */
239 /* extract h.header m.last_field_size */
242 /* lookahead h.header */
256 /* validate h.header */
259 /* invalidate h.header */
260 INSTR_HDR_INVALIDATE,
264 * dst = HMEF, src = HMEFTI
266 INSTR_MOV, /* dst = MEF, src = MEFT */
267 INSTR_MOV_MH, /* dst = MEF, src = H */
268 INSTR_MOV_HM, /* dst = H, src = MEFT */
269 INSTR_MOV_HH, /* dst = H, src = H */
270 INSTR_MOV_I, /* dst = HMEF, src = I */
272 /* dma h.header t.field
273 * memcpy(h.header, t.field, sizeof(h.header))
286 * dst = HMEF, src = HMEFTI
288 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
289 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
290 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
291 INSTR_ALU_ADD_HH, /* dst = H, src = H */
292 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
293 INSTR_ALU_ADD_HI, /* dst = H, src = I */
297 * dst = HMEF, src = HMEFTI
299 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
300 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
301 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
302 INSTR_ALU_SUB_HH, /* dst = H, src = H */
303 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
304 INSTR_ALU_SUB_HI, /* dst = H, src = I */
307 * dst = dst '+ src[0:1] '+ src[2:3] + ...
308 * dst = H, src = {H, h.header}
310 INSTR_ALU_CKADD_FIELD, /* src = H */
311 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
312 INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
318 INSTR_ALU_CKSUB_FIELD,
322 * dst = HMEF, src = HMEFTI
324 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
325 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
326 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
327 INSTR_ALU_AND_HH, /* dst = H, src = H */
328 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
332 * dst = HMEF, src = HMEFTI
334 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
335 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
336 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
337 INSTR_ALU_OR_HH, /* dst = H, src = H */
338 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
342 * dst = HMEF, src = HMEFTI
344 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
345 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
346 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
347 INSTR_ALU_XOR_HH, /* dst = H, src = H */
348 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
352 * dst = HMEF, src = HMEFTI
354 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
355 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
356 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
357 INSTR_ALU_SHL_HH, /* dst = H, src = H */
358 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
359 INSTR_ALU_SHL_HI, /* dst = H, src = I */
363 * dst = HMEF, src = HMEFTI
365 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
366 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
367 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
368 INSTR_ALU_SHR_HH, /* dst = H, src = H */
369 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
370 INSTR_ALU_SHR_HI, /* dst = H, src = I */
372 /* regprefetch REGARRAY index
373 * prefetch REGARRAY[index]
376 INSTR_REGPREFETCH_RH, /* index = H */
377 INSTR_REGPREFETCH_RM, /* index = MEFT */
378 INSTR_REGPREFETCH_RI, /* index = I */
380 /* regrd dst REGARRAY index
381 * dst = REGARRAY[index]
382 * dst = HMEF, index = HMEFTI
384 INSTR_REGRD_HRH, /* dst = H, index = H */
385 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
386 INSTR_REGRD_HRI, /* dst = H, index = I */
387 INSTR_REGRD_MRH, /* dst = MEF, index = H */
388 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
389 INSTR_REGRD_MRI, /* dst = MEF, index = I */
391 /* regwr REGARRAY index src
392 * REGARRAY[index] = src
393 * index = HMEFTI, src = HMEFTI
395 INSTR_REGWR_RHH, /* index = H, src = H */
396 INSTR_REGWR_RHM, /* index = H, src = MEFT */
397 INSTR_REGWR_RHI, /* index = H, src = I */
398 INSTR_REGWR_RMH, /* index = MEFT, src = H */
399 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
400 INSTR_REGWR_RMI, /* index = MEFT, src = I */
401 INSTR_REGWR_RIH, /* index = I, src = H */
402 INSTR_REGWR_RIM, /* index = I, src = MEFT */
403 INSTR_REGWR_RII, /* index = I, src = I */
405 /* regadd REGARRAY index src
406 * REGARRAY[index] += src
407 * index = HMEFTI, src = HMEFTI
409 INSTR_REGADD_RHH, /* index = H, src = H */
410 INSTR_REGADD_RHM, /* index = H, src = MEFT */
411 INSTR_REGADD_RHI, /* index = H, src = I */
412 INSTR_REGADD_RMH, /* index = MEFT, src = H */
413 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
414 INSTR_REGADD_RMI, /* index = MEFT, src = I */
415 INSTR_REGADD_RIH, /* index = I, src = H */
416 INSTR_REGADD_RIM, /* index = I, src = MEFT */
417 INSTR_REGADD_RII, /* index = I, src = I */
419 /* metprefetch METARRAY index
420 * prefetch METARRAY[index]
423 INSTR_METPREFETCH_H, /* index = H */
424 INSTR_METPREFETCH_M, /* index = MEFT */
425 INSTR_METPREFETCH_I, /* index = I */
427 /* meter METARRAY index length color_in color_out
428 * color_out = meter(METARRAY[index], length, color_in)
429 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
431 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
432 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
433 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
434 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
435 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
436 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
437 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
438 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
439 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
440 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
441 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
442 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
451 /* learn LEARNER ACTION_NAME [ m.action_first_arg ] */
455 INSTR_LEARNER_FORGET,
457 /* extern e.obj.func */
468 /* jmpv LABEL h.header
469 * Jump if header is valid
473 /* jmpnv LABEL h.header
474 * Jump if header is invalid
479 * Jump if table lookup hit
484 * Jump if table lookup miss
491 INSTR_JMP_ACTION_HIT,
493 /* jmpna LABEL ACTION
494 * Jump if action not run
496 INSTR_JMP_ACTION_MISS,
499 * Jump if a is equal to b
500 * a = HMEFT, b = HMEFTI
502 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
503 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
504 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
505 INSTR_JMP_EQ_HH, /* a = H, b = H */
506 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
509 * Jump if a is not equal to b
510 * a = HMEFT, b = HMEFTI
512 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
513 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
514 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
515 INSTR_JMP_NEQ_HH, /* a = H, b = H */
516 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
519 * Jump if a is less than b
520 * a = HMEFT, b = HMEFTI
522 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
523 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
524 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
525 INSTR_JMP_LT_HH, /* a = H, b = H */
526 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
527 INSTR_JMP_LT_HI, /* a = H, b = I */
530 * Jump if a is greater than b
531 * a = HMEFT, b = HMEFTI
533 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
534 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
535 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
536 INSTR_JMP_GT_HH, /* a = H, b = H */
537 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
538 INSTR_JMP_GT_HI, /* a = H, b = I */
545 /* Start of custom instructions. */
549 struct instr_operand {
570 uint8_t header_id[8];
571 uint8_t struct_id[8];
576 struct instr_hdr_validity {
589 struct instr_extern_obj {
594 struct instr_extern_func {
598 struct instr_dst_src {
599 struct instr_operand dst;
601 struct instr_operand src;
606 struct instr_regarray {
611 struct instr_operand idx;
616 struct instr_operand dstsrc;
626 struct instr_operand idx;
630 struct instr_operand length;
633 struct instr_operand color_in;
634 uint32_t color_in_val;
637 struct instr_operand color_out;
642 uint8_t header_id[8];
643 uint8_t struct_id[8];
654 struct instruction *ip;
657 struct instr_operand a;
663 struct instr_operand b;
669 enum instruction_type type;
672 struct instr_hdr_validity valid;
673 struct instr_dst_src mov;
674 struct instr_regarray regarray;
675 struct instr_meter meter;
676 struct instr_dma dma;
677 struct instr_dst_src alu;
678 struct instr_table table;
679 struct instr_learn learn;
680 struct instr_extern_obj ext_obj;
681 struct instr_extern_func ext_func;
682 struct instr_jmp jmp;
686 struct instruction_data {
687 char label[RTE_SWX_NAME_SIZE];
688 char jmp_label[RTE_SWX_NAME_SIZE];
689 uint32_t n_users; /* user = jmp instruction to this instruction. */
693 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
699 (*action_func_t)(struct rte_swx_pipeline *p);
702 TAILQ_ENTRY(action) node;
703 char name[RTE_SWX_NAME_SIZE];
704 struct struct_type *st;
705 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
706 struct instruction *instructions;
707 struct instruction_data *instruction_data;
708 uint32_t n_instructions;
712 TAILQ_HEAD(action_tailq, action);
718 TAILQ_ENTRY(table_type) node;
719 char name[RTE_SWX_NAME_SIZE];
720 enum rte_swx_table_match_type match_type;
721 struct rte_swx_table_ops ops;
724 TAILQ_HEAD(table_type_tailq, table_type);
727 enum rte_swx_table_match_type match_type;
732 TAILQ_ENTRY(table) node;
733 char name[RTE_SWX_NAME_SIZE];
734 char args[RTE_SWX_NAME_SIZE];
735 struct table_type *type; /* NULL when n_fields == 0. */
738 struct match_field *fields;
740 struct header *header; /* Only valid when n_fields > 0. */
743 struct action **actions;
744 struct action *default_action;
745 uint8_t *default_action_data;
747 int default_action_is_const;
748 uint32_t action_data_size_max;
754 TAILQ_HEAD(table_tailq, table);
756 struct table_runtime {
757 rte_swx_table_lookup_t func;
762 struct table_statistics {
763 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
764 uint64_t *n_pkts_action;
771 TAILQ_ENTRY(selector) node;
772 char name[RTE_SWX_NAME_SIZE];
774 struct field *group_id_field;
775 struct field **selector_fields;
776 uint32_t n_selector_fields;
777 struct header *selector_header;
778 struct field *member_id_field;
780 uint32_t n_groups_max;
781 uint32_t n_members_per_group_max;
786 TAILQ_HEAD(selector_tailq, selector);
788 struct selector_runtime {
790 uint8_t **group_id_buffer;
791 uint8_t **selector_buffer;
792 uint8_t **member_id_buffer;
795 struct selector_statistics {
803 TAILQ_ENTRY(learner) node;
804 char name[RTE_SWX_NAME_SIZE];
807 struct field **fields;
809 struct header *header;
812 struct action **actions;
813 struct action *default_action;
814 uint8_t *default_action_data;
816 int default_action_is_const;
817 uint32_t action_data_size_max;
824 TAILQ_HEAD(learner_tailq, learner);
826 struct learner_runtime {
831 struct learner_statistics {
832 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
833 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
834 uint64_t n_pkts_forget;
835 uint64_t *n_pkts_action;
842 TAILQ_ENTRY(regarray) node;
843 char name[RTE_SWX_NAME_SIZE];
849 TAILQ_HEAD(regarray_tailq, regarray);
851 struct regarray_runtime {
859 struct meter_profile {
860 TAILQ_ENTRY(meter_profile) node;
861 char name[RTE_SWX_NAME_SIZE];
862 struct rte_meter_trtcm_params params;
863 struct rte_meter_trtcm_profile profile;
867 TAILQ_HEAD(meter_profile_tailq, meter_profile);
870 TAILQ_ENTRY(metarray) node;
871 char name[RTE_SWX_NAME_SIZE];
876 TAILQ_HEAD(metarray_tailq, metarray);
879 struct rte_meter_trtcm m;
880 struct meter_profile *profile;
881 enum rte_color color_mask;
884 uint64_t n_pkts[RTE_COLORS];
885 uint64_t n_bytes[RTE_COLORS];
888 struct metarray_runtime {
889 struct meter *metarray;
898 struct rte_swx_pkt pkt;
904 /* Packet headers. */
905 struct header_runtime *headers; /* Extracted or generated headers. */
906 struct header_out_runtime *headers_out; /* Emitted headers. */
907 uint8_t *header_storage;
908 uint8_t *header_out_storage;
909 uint64_t valid_headers;
910 uint32_t n_headers_out;
912 /* Packet meta-data. */
916 struct table_runtime *tables;
917 struct selector_runtime *selectors;
918 struct learner_runtime *learners;
919 struct rte_swx_table_state *table_state;
921 int hit; /* 0 = Miss, 1 = Hit. */
925 /* Extern objects and functions. */
926 struct extern_obj_runtime *extern_objs;
927 struct extern_func_runtime *extern_funcs;
930 struct instruction *ip;
931 struct instruction *ret;
934 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
935 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
936 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
938 #define HEADER_VALID(thread, header_id) \
939 MASK64_BIT_GET((thread)->valid_headers, header_id)
941 static inline uint64_t
942 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
944 uint8_t *x_struct = t->structs[x->struct_id];
945 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
946 uint64_t x64 = *x64_ptr;
947 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
949 return x64 & x64_mask;
952 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
954 static inline uint64_t
955 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
957 uint8_t *x_struct = t->structs[x->struct_id];
958 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
959 uint64_t x64 = *x64_ptr;
961 return ntoh64(x64) >> (64 - x->n_bits);
966 #define instr_operand_nbo instr_operand_hbo
970 #define ALU(thread, ip, operator) \
972 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
973 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
974 uint64_t dst64 = *dst64_ptr; \
975 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
976 uint64_t dst = dst64 & dst64_mask; \
978 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
979 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
980 uint64_t src64 = *src64_ptr; \
981 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
982 uint64_t src = src64 & src64_mask; \
984 uint64_t result = dst operator src; \
986 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
989 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
991 #define ALU_MH(thread, ip, operator) \
993 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
994 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
995 uint64_t dst64 = *dst64_ptr; \
996 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
997 uint64_t dst = dst64 & dst64_mask; \
999 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1000 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1001 uint64_t src64 = *src64_ptr; \
1002 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1004 uint64_t result = dst operator src; \
1006 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1009 #define ALU_HM(thread, ip, operator) \
1011 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1012 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1013 uint64_t dst64 = *dst64_ptr; \
1014 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1015 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1017 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1018 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1019 uint64_t src64 = *src64_ptr; \
1020 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1021 uint64_t src = src64 & src64_mask; \
1023 uint64_t result = dst operator src; \
1024 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1026 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1029 #define ALU_HM_FAST(thread, ip, operator) \
1031 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1032 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1033 uint64_t dst64 = *dst64_ptr; \
1034 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1035 uint64_t dst = dst64 & dst64_mask; \
1037 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1038 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1039 uint64_t src64 = *src64_ptr; \
1040 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1041 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1043 uint64_t result = dst operator src; \
1045 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1048 #define ALU_HH(thread, ip, operator) \
1050 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1051 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1052 uint64_t dst64 = *dst64_ptr; \
1053 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1054 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1056 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1057 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1058 uint64_t src64 = *src64_ptr; \
1059 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1061 uint64_t result = dst operator src; \
1062 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1064 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1067 #define ALU_HH_FAST(thread, ip, operator) \
1069 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1070 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1071 uint64_t dst64 = *dst64_ptr; \
1072 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1073 uint64_t dst = dst64 & dst64_mask; \
1075 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1076 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1077 uint64_t src64 = *src64_ptr; \
1078 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1080 uint64_t result = dst operator src; \
1082 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1089 #define ALU_HM_FAST ALU
1091 #define ALU_HH_FAST ALU
1095 #define ALU_I(thread, ip, operator) \
1097 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1098 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1099 uint64_t dst64 = *dst64_ptr; \
1100 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1101 uint64_t dst = dst64 & dst64_mask; \
1103 uint64_t src = (ip)->alu.src_val; \
1105 uint64_t result = dst operator src; \
1107 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1110 #define ALU_MI ALU_I
1112 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1114 #define ALU_HI(thread, ip, operator) \
1116 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1117 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1118 uint64_t dst64 = *dst64_ptr; \
1119 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1120 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1122 uint64_t src = (ip)->alu.src_val; \
1124 uint64_t result = dst operator src; \
1125 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1127 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1132 #define ALU_HI ALU_I
1136 #define MOV(thread, ip) \
1138 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1139 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1140 uint64_t dst64 = *dst64_ptr; \
1141 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1143 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1144 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1145 uint64_t src64 = *src64_ptr; \
1146 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1147 uint64_t src = src64 & src64_mask; \
1149 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1152 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1154 #define MOV_MH(thread, ip) \
1156 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1157 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1158 uint64_t dst64 = *dst64_ptr; \
1159 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1161 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1162 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1163 uint64_t src64 = *src64_ptr; \
1164 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1166 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1169 #define MOV_HM(thread, ip) \
1171 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1172 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1173 uint64_t dst64 = *dst64_ptr; \
1174 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1176 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1177 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1178 uint64_t src64 = *src64_ptr; \
1179 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1180 uint64_t src = src64 & src64_mask; \
1182 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1183 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1186 #define MOV_HH(thread, ip) \
1188 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1189 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1190 uint64_t dst64 = *dst64_ptr; \
1191 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1193 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1194 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1195 uint64_t src64 = *src64_ptr; \
1197 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1198 src = src >> (64 - (ip)->mov.dst.n_bits); \
1199 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1210 #define MOV_I(thread, ip) \
1212 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1213 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1214 uint64_t dst64 = *dst64_ptr; \
1215 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1217 uint64_t src = (ip)->mov.src_val; \
1219 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1222 #define JMP_CMP(thread, ip, operator) \
1224 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1225 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1226 uint64_t a64 = *a64_ptr; \
1227 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1228 uint64_t a = a64 & a64_mask; \
1230 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1231 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1232 uint64_t b64 = *b64_ptr; \
1233 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1234 uint64_t b = b64 & b64_mask; \
1236 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1239 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1241 #define JMP_CMP_MH(thread, ip, operator) \
1243 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1244 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1245 uint64_t a64 = *a64_ptr; \
1246 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1247 uint64_t a = a64 & a64_mask; \
1249 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1250 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1251 uint64_t b64 = *b64_ptr; \
1252 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1254 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1257 #define JMP_CMP_HM(thread, ip, operator) \
1259 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1260 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1261 uint64_t a64 = *a64_ptr; \
1262 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1264 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1265 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1266 uint64_t b64 = *b64_ptr; \
1267 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1268 uint64_t b = b64 & b64_mask; \
1270 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1273 #define JMP_CMP_HH(thread, ip, operator) \
1275 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1276 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1277 uint64_t a64 = *a64_ptr; \
1278 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1280 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1281 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1282 uint64_t b64 = *b64_ptr; \
1283 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1285 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1288 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1290 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1291 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1292 uint64_t a64 = *a64_ptr; \
1293 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1295 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1296 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1297 uint64_t b64 = *b64_ptr; \
1298 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1300 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1305 #define JMP_CMP_MH JMP_CMP
1306 #define JMP_CMP_HM JMP_CMP
1307 #define JMP_CMP_HH JMP_CMP
1308 #define JMP_CMP_HH_FAST JMP_CMP
1312 #define JMP_CMP_I(thread, ip, operator) \
1314 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1315 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1316 uint64_t a64 = *a64_ptr; \
1317 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1318 uint64_t a = a64 & a64_mask; \
1320 uint64_t b = (ip)->jmp.b_val; \
1322 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1325 #define JMP_CMP_MI JMP_CMP_I
1327 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1329 #define JMP_CMP_HI(thread, ip, operator) \
1331 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1332 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1333 uint64_t a64 = *a64_ptr; \
1334 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1336 uint64_t b = (ip)->jmp.b_val; \
1338 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1343 #define JMP_CMP_HI JMP_CMP_I
1347 #define METADATA_READ(thread, offset, n_bits) \
1349 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1350 uint64_t m64 = *m64_ptr; \
1351 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1355 #define METADATA_WRITE(thread, offset, n_bits, value) \
1357 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1358 uint64_t m64 = *m64_ptr; \
1359 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1361 uint64_t m_new = value; \
1363 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1366 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1367 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1370 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1371 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
1374 struct rte_swx_pipeline {
1375 struct struct_type_tailq struct_types;
1376 struct port_in_type_tailq port_in_types;
1377 struct port_in_tailq ports_in;
1378 struct port_out_type_tailq port_out_types;
1379 struct port_out_tailq ports_out;
1380 struct extern_type_tailq extern_types;
1381 struct extern_obj_tailq extern_objs;
1382 struct extern_func_tailq extern_funcs;
1383 struct header_tailq headers;
1384 struct struct_type *metadata_st;
1385 uint32_t metadata_struct_id;
1386 struct action_tailq actions;
1387 struct table_type_tailq table_types;
1388 struct table_tailq tables;
1389 struct selector_tailq selectors;
1390 struct learner_tailq learners;
1391 struct regarray_tailq regarrays;
1392 struct meter_profile_tailq meter_profiles;
1393 struct metarray_tailq metarrays;
1395 struct port_in_runtime *in;
1396 struct port_out_runtime *out;
1397 struct instruction **action_instructions;
1398 action_func_t *action_funcs;
1399 struct rte_swx_table_state *table_state;
1400 struct table_statistics *table_stats;
1401 struct selector_statistics *selector_stats;
1402 struct learner_statistics *learner_stats;
1403 struct regarray_runtime *regarray_runtime;
1404 struct metarray_runtime *metarray_runtime;
1405 struct instruction *instructions;
1406 struct instruction_data *instruction_data;
1407 instr_exec_t *instruction_table;
1408 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1412 uint32_t n_ports_in;
1413 uint32_t n_ports_out;
1414 uint32_t n_extern_objs;
1415 uint32_t n_extern_funcs;
1418 uint32_t n_selectors;
1419 uint32_t n_learners;
1420 uint32_t n_regarrays;
1421 uint32_t n_metarrays;
1425 uint32_t n_instructions;
1434 pipeline_port_inc(struct rte_swx_pipeline *p)
1436 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1440 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1442 t->ip = p->instructions;
1446 thread_ip_set(struct thread *t, struct instruction *ip)
1452 thread_ip_action_call(struct rte_swx_pipeline *p,
1457 t->ip = p->action_instructions[action_id];
1461 thread_ip_inc(struct rte_swx_pipeline *p);
1464 thread_ip_inc(struct rte_swx_pipeline *p)
1466 struct thread *t = &p->threads[p->thread_id];
1472 thread_ip_inc_cond(struct thread *t, int cond)
1478 thread_yield(struct rte_swx_pipeline *p)
1480 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1484 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1486 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1493 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1495 struct port_in_runtime *port = &p->in[p->port_id];
1496 struct rte_swx_pkt *pkt = &t->pkt;
1500 pkt_received = port->pkt_rx(port->obj, pkt);
1501 t->ptr = &pkt->pkt[pkt->offset];
1502 rte_prefetch0(t->ptr);
1504 TRACE("[Thread %2u] rx %s from port %u\n",
1506 pkt_received ? "1 pkt" : "0 pkts",
1510 t->valid_headers = 0;
1511 t->n_headers_out = 0;
1514 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1517 t->table_state = p->table_state;
1520 pipeline_port_inc(p);
1522 return pkt_received;
1526 instr_rx_exec(struct rte_swx_pipeline *p)
1528 struct thread *t = &p->threads[p->thread_id];
1529 struct instruction *ip = t->ip;
1533 pkt_received = __instr_rx_exec(p, t, ip);
1536 thread_ip_inc_cond(t, pkt_received);
1544 emit_handler(struct thread *t)
1546 struct header_out_runtime *h0 = &t->headers_out[0];
1547 struct header_out_runtime *h1 = &t->headers_out[1];
1548 uint32_t offset = 0, i;
1550 /* No header change or header decapsulation. */
1551 if ((t->n_headers_out == 1) &&
1552 (h0->ptr + h0->n_bytes == t->ptr)) {
1553 TRACE("Emit handler: no header change or header decap.\n");
1555 t->pkt.offset -= h0->n_bytes;
1556 t->pkt.length += h0->n_bytes;
1561 /* Header encapsulation (optionally, with prior header decasulation). */
1562 if ((t->n_headers_out == 2) &&
1563 (h1->ptr + h1->n_bytes == t->ptr) &&
1564 (h0->ptr == h0->ptr0)) {
1567 TRACE("Emit handler: header encapsulation.\n");
1569 offset = h0->n_bytes + h1->n_bytes;
1570 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1571 t->pkt.offset -= offset;
1572 t->pkt.length += offset;
1577 /* For any other case. */
1578 TRACE("Emit handler: complex case.\n");
1580 for (i = 0; i < t->n_headers_out; i++) {
1581 struct header_out_runtime *h = &t->headers_out[i];
1583 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1584 offset += h->n_bytes;
1588 memcpy(t->ptr - offset, t->header_out_storage, offset);
1589 t->pkt.offset -= offset;
1590 t->pkt.length += offset;
1595 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1597 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1598 struct port_out_runtime *port = &p->out[port_id];
1599 struct rte_swx_pkt *pkt = &t->pkt;
1601 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1609 port->pkt_tx(port->obj, pkt);
1613 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1615 uint64_t port_id = ip->io.io.val;
1616 struct port_out_runtime *port = &p->out[port_id];
1617 struct rte_swx_pkt *pkt = &t->pkt;
1619 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1627 port->pkt_tx(port->obj, pkt);
1634 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1636 const struct instruction *ip,
1639 uint64_t valid_headers = t->valid_headers;
1640 uint8_t *ptr = t->ptr;
1641 uint32_t offset = t->pkt.offset;
1642 uint32_t length = t->pkt.length;
1645 for (i = 0; i < n_extract; i++) {
1646 uint32_t header_id = ip->io.hdr.header_id[i];
1647 uint32_t struct_id = ip->io.hdr.struct_id[i];
1648 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1650 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1656 t->structs[struct_id] = ptr;
1657 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1666 t->valid_headers = valid_headers;
1669 t->pkt.offset = offset;
1670 t->pkt.length = length;
1675 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1677 const struct instruction *ip)
1679 __instr_hdr_extract_many_exec(p, t, ip, 1);
1683 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1685 const struct instruction *ip)
1687 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1689 __instr_hdr_extract_many_exec(p, t, ip, 2);
1693 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1695 const struct instruction *ip)
1697 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1699 __instr_hdr_extract_many_exec(p, t, ip, 3);
1703 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1705 const struct instruction *ip)
1707 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1709 __instr_hdr_extract_many_exec(p, t, ip, 4);
1713 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1715 const struct instruction *ip)
1717 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1719 __instr_hdr_extract_many_exec(p, t, ip, 5);
1723 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1725 const struct instruction *ip)
1727 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1729 __instr_hdr_extract_many_exec(p, t, ip, 6);
1733 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1735 const struct instruction *ip)
1737 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1739 __instr_hdr_extract_many_exec(p, t, ip, 7);
1743 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1745 const struct instruction *ip)
1747 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1749 __instr_hdr_extract_many_exec(p, t, ip, 8);
1753 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
1755 const struct instruction *ip)
1757 uint64_t valid_headers = t->valid_headers;
1758 uint8_t *ptr = t->ptr;
1759 uint32_t offset = t->pkt.offset;
1760 uint32_t length = t->pkt.length;
1762 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1763 uint32_t header_id = ip->io.hdr.header_id[0];
1764 uint32_t struct_id = ip->io.hdr.struct_id[0];
1765 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
1767 struct header_runtime *h = &t->headers[header_id];
1769 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
1775 n_bytes += n_bytes_last;
1778 t->structs[struct_id] = ptr;
1779 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1780 h->n_bytes = n_bytes;
1783 t->pkt.offset = offset + n_bytes;
1784 t->pkt.length = length - n_bytes;
1785 t->ptr = ptr + n_bytes;
1789 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
1791 const struct instruction *ip)
1793 uint64_t valid_headers = t->valid_headers;
1794 uint8_t *ptr = t->ptr;
1796 uint32_t header_id = ip->io.hdr.header_id[0];
1797 uint32_t struct_id = ip->io.hdr.struct_id[0];
1799 TRACE("[Thread %2u]: lookahead header %u\n",
1804 t->structs[struct_id] = ptr;
1805 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1812 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
1814 const struct instruction *ip,
1817 uint64_t valid_headers = t->valid_headers;
1818 uint32_t n_headers_out = t->n_headers_out;
1819 struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
1820 uint8_t *ho_ptr = NULL;
1821 uint32_t ho_nbytes = 0, first = 1, i;
1823 for (i = 0; i < n_emit; i++) {
1824 uint32_t header_id = ip->io.hdr.header_id[i];
1825 uint32_t struct_id = ip->io.hdr.struct_id[i];
1827 struct header_runtime *hi = &t->headers[header_id];
1828 uint8_t *hi_ptr0 = hi->ptr0;
1829 uint32_t n_bytes = hi->n_bytes;
1831 uint8_t *hi_ptr = t->structs[struct_id];
1833 if (!MASK64_BIT_GET(valid_headers, header_id))
1836 TRACE("[Thread %2u]: emit header %u\n",
1844 if (!t->n_headers_out) {
1845 ho = &t->headers_out[0];
1851 ho_nbytes = n_bytes;
1858 ho_nbytes = ho->n_bytes;
1862 if (ho_ptr + ho_nbytes == hi_ptr) {
1863 ho_nbytes += n_bytes;
1865 ho->n_bytes = ho_nbytes;
1872 ho_nbytes = n_bytes;
1878 ho->n_bytes = ho_nbytes;
1879 t->n_headers_out = n_headers_out;
1883 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
1885 const struct instruction *ip)
1887 __instr_hdr_emit_many_exec(p, t, ip, 1);
1891 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
1893 const struct instruction *ip)
1895 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1897 __instr_hdr_emit_many_exec(p, t, ip, 1);
1898 __instr_tx_exec(p, t, ip);
1902 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
1904 const struct instruction *ip)
1906 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1908 __instr_hdr_emit_many_exec(p, t, ip, 2);
1909 __instr_tx_exec(p, t, ip);
1913 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
1915 const struct instruction *ip)
1917 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1919 __instr_hdr_emit_many_exec(p, t, ip, 3);
1920 __instr_tx_exec(p, t, ip);
1924 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
1926 const struct instruction *ip)
1928 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1930 __instr_hdr_emit_many_exec(p, t, ip, 4);
1931 __instr_tx_exec(p, t, ip);
1935 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
1937 const struct instruction *ip)
1939 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1941 __instr_hdr_emit_many_exec(p, t, ip, 5);
1942 __instr_tx_exec(p, t, ip);
1946 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
1948 const struct instruction *ip)
1950 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1952 __instr_hdr_emit_many_exec(p, t, ip, 6);
1953 __instr_tx_exec(p, t, ip);
1957 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
1959 const struct instruction *ip)
1961 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1963 __instr_hdr_emit_many_exec(p, t, ip, 7);
1964 __instr_tx_exec(p, t, ip);
1968 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
1970 const struct instruction *ip)
1972 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
1974 __instr_hdr_emit_many_exec(p, t, ip, 8);
1975 __instr_tx_exec(p, t, ip);
1982 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
1984 const struct instruction *ip)
1986 uint32_t header_id = ip->valid.header_id;
1988 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
1991 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
1998 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2000 const struct instruction *ip)
2002 uint32_t header_id = ip->valid.header_id;
2004 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2007 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2014 __instr_learn_exec(struct rte_swx_pipeline *p,
2016 const struct instruction *ip)
2018 uint64_t action_id = ip->learn.action_id;
2019 uint32_t mf_offset = ip->learn.mf_offset;
2020 uint32_t learner_id = t->learner_id;
2021 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2022 p->n_selectors + learner_id];
2023 struct learner_runtime *l = &t->learners[learner_id];
2024 struct learner_statistics *stats = &p->learner_stats[learner_id];
2028 status = rte_swx_table_learner_add(ts->obj,
2032 &t->metadata[mf_offset]);
2034 TRACE("[Thread %2u] learner %u learn %s\n",
2037 status ? "ok" : "error");
2039 stats->n_pkts_learn[status] += 1;
2046 __instr_forget_exec(struct rte_swx_pipeline *p,
2048 const struct instruction *ip __rte_unused)
2050 uint32_t learner_id = t->learner_id;
2051 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2052 p->n_selectors + learner_id];
2053 struct learner_runtime *l = &t->learners[learner_id];
2054 struct learner_statistics *stats = &p->learner_stats[learner_id];
2057 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2059 TRACE("[Thread %2u] learner %u forget\n",
2063 stats->n_pkts_forget += 1;
2069 static inline uint32_t
2070 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2072 const struct instruction *ip)
2074 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2075 uint32_t func_id = ip->ext_obj.func_id;
2076 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2077 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2080 TRACE("[Thread %2u] extern obj %u member func %u\n",
2085 done = func(obj->obj, obj->mailbox);
2090 static inline uint32_t
2091 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2093 const struct instruction *ip)
2095 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2096 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2097 rte_swx_extern_func_t func = ext_func->func;
2100 TRACE("[Thread %2u] extern func %u\n",
2104 done = func(ext_func->mailbox);
2113 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2115 const struct instruction *ip)
2117 TRACE("[Thread %2u] mov\n", p->thread_id);
2123 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2125 const struct instruction *ip)
2127 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2133 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2135 const struct instruction *ip)
2137 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2143 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2145 const struct instruction *ip)
2147 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2153 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2155 const struct instruction *ip)
2157 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2166 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2168 const struct instruction *ip,
2171 uint8_t *action_data = t->structs[0];
2172 uint64_t valid_headers = t->valid_headers;
2175 for (i = 0; i < n_dma; i++) {
2176 uint32_t header_id = ip->dma.dst.header_id[i];
2177 uint32_t struct_id = ip->dma.dst.struct_id[i];
2178 uint32_t offset = ip->dma.src.offset[i];
2179 uint32_t n_bytes = ip->dma.n_bytes[i];
2181 struct header_runtime *h = &t->headers[header_id];
2182 uint8_t *h_ptr0 = h->ptr0;
2183 uint8_t *h_ptr = t->structs[struct_id];
2185 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2187 void *src = &action_data[offset];
2189 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2192 memcpy(dst, src, n_bytes);
2193 t->structs[struct_id] = dst;
2194 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2197 t->valid_headers = valid_headers;
2201 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2203 __instr_dma_ht_many_exec(p, t, ip, 1);
2207 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2209 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2211 __instr_dma_ht_many_exec(p, t, ip, 2);
2215 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2217 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2219 __instr_dma_ht_many_exec(p, t, ip, 3);
2223 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2225 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2227 __instr_dma_ht_many_exec(p, t, ip, 4);
2231 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2233 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2235 __instr_dma_ht_many_exec(p, t, ip, 5);
2239 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2241 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2243 __instr_dma_ht_many_exec(p, t, ip, 6);
2247 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2249 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2251 __instr_dma_ht_many_exec(p, t, ip, 7);
2255 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2257 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2259 __instr_dma_ht_many_exec(p, t, ip, 8);
2266 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2268 const struct instruction *ip)
2270 TRACE("[Thread %2u] add\n", p->thread_id);
2276 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2278 const struct instruction *ip)
2280 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2286 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2288 const struct instruction *ip)
2290 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2296 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2298 const struct instruction *ip)
2300 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2306 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2308 const struct instruction *ip)
2310 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2316 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2318 const struct instruction *ip)
2320 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2326 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2328 const struct instruction *ip)
2330 TRACE("[Thread %2u] sub\n", p->thread_id);
2336 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2338 const struct instruction *ip)
2340 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2346 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2348 const struct instruction *ip)
2350 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2356 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2358 const struct instruction *ip)
2360 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2366 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2368 const struct instruction *ip)
2370 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2376 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2378 const struct instruction *ip)
2380 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2386 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2388 const struct instruction *ip)
2390 TRACE("[Thread %2u] shl\n", p->thread_id);
2396 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2398 const struct instruction *ip)
2400 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2406 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2408 const struct instruction *ip)
2410 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2416 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2418 const struct instruction *ip)
2420 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2426 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2428 const struct instruction *ip)
2430 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2436 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2438 const struct instruction *ip)
2440 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2446 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2448 const struct instruction *ip)
2450 TRACE("[Thread %2u] shr\n", p->thread_id);
2456 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2458 const struct instruction *ip)
2460 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2466 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2468 const struct instruction *ip)
2470 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2476 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2478 const struct instruction *ip)
2480 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2486 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2488 const struct instruction *ip)
2490 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2497 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2499 const struct instruction *ip)
2501 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2507 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2509 const struct instruction *ip)
2511 TRACE("[Thread %2u] and\n", p->thread_id);
2517 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2519 const struct instruction *ip)
2521 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2527 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2529 const struct instruction *ip)
2531 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2533 ALU_HM_FAST(t, ip, &);
2537 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2539 const struct instruction *ip)
2541 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2543 ALU_HH_FAST(t, ip, &);
2547 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2549 const struct instruction *ip)
2551 TRACE("[Thread %2u] and (i)\n", p->thread_id);
2557 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2559 const struct instruction *ip)
2561 TRACE("[Thread %2u] or\n", p->thread_id);
2567 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2569 const struct instruction *ip)
2571 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2577 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2579 const struct instruction *ip)
2581 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
2583 ALU_HM_FAST(t, ip, |);
2587 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2589 const struct instruction *ip)
2591 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
2593 ALU_HH_FAST(t, ip, |);
2597 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
2599 const struct instruction *ip)
2601 TRACE("[Thread %2u] or (i)\n", p->thread_id);
2607 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
2609 const struct instruction *ip)
2611 TRACE("[Thread %2u] xor\n", p->thread_id);
2617 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2619 const struct instruction *ip)
2621 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
2627 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2629 const struct instruction *ip)
2631 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
2633 ALU_HM_FAST(t, ip, ^);
2637 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2639 const struct instruction *ip)
2641 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
2643 ALU_HH_FAST(t, ip, ^);
2647 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
2649 const struct instruction *ip)
2651 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
2657 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
2659 const struct instruction *ip)
2661 uint8_t *dst_struct, *src_struct;
2662 uint16_t *dst16_ptr, dst;
2663 uint64_t *src64_ptr, src64, src64_mask, src;
2666 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
2669 dst_struct = t->structs[ip->alu.dst.struct_id];
2670 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2673 src_struct = t->structs[ip->alu.src.struct_id];
2674 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2676 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2677 src = src64 & src64_mask;
2682 /* The first input (r) is a 16-bit number. The second and the third
2683 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
2684 * three numbers (output r) is a 34-bit number.
2686 r += (src >> 32) + (src & 0xFFFFFFFF);
2688 /* The first input is a 16-bit number. The second input is an 18-bit
2689 * number. In the worst case scenario, the sum of the two numbers is a
2692 r = (r & 0xFFFF) + (r >> 16);
2694 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2695 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
2697 r = (r & 0xFFFF) + (r >> 16);
2699 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2700 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2701 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
2702 * therefore the output r is always a 16-bit number.
2704 r = (r & 0xFFFF) + (r >> 16);
2709 *dst16_ptr = (uint16_t)r;
2713 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
2715 const struct instruction *ip)
2717 uint8_t *dst_struct, *src_struct;
2718 uint16_t *dst16_ptr, dst;
2719 uint64_t *src64_ptr, src64, src64_mask, src;
2722 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
2725 dst_struct = t->structs[ip->alu.dst.struct_id];
2726 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2729 src_struct = t->structs[ip->alu.src.struct_id];
2730 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2732 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2733 src = src64 & src64_mask;
2738 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
2739 * the following sequence of operations in 2's complement arithmetic:
2740 * a '- b = (a - b) % 0xFFFF.
2742 * In order to prevent an underflow for the below subtraction, in which
2743 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
2744 * minuend), we first add a multiple of the 0xFFFF modulus to the
2745 * minuend. The number we add to the minuend needs to be a 34-bit number
2746 * or higher, so for readability reasons we picked the 36-bit multiple.
2747 * We are effectively turning the 16-bit minuend into a 36-bit number:
2748 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
2750 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
2752 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
2753 * result (the output r) is a 36-bit number.
2755 r -= (src >> 32) + (src & 0xFFFFFFFF);
2757 /* The first input is a 16-bit number. The second input is a 20-bit
2758 * number. Their sum is a 21-bit number.
2760 r = (r & 0xFFFF) + (r >> 16);
2762 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2763 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
2765 r = (r & 0xFFFF) + (r >> 16);
2767 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2768 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2769 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
2770 * generated, therefore the output r is always a 16-bit number.
2772 r = (r & 0xFFFF) + (r >> 16);
2777 *dst16_ptr = (uint16_t)r;
2781 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
2783 const struct instruction *ip)
2785 uint8_t *dst_struct, *src_struct;
2786 uint16_t *dst16_ptr;
2787 uint32_t *src32_ptr;
2790 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
2793 dst_struct = t->structs[ip->alu.dst.struct_id];
2794 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2796 src_struct = t->structs[ip->alu.src.struct_id];
2797 src32_ptr = (uint32_t *)&src_struct[0];
2799 r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
2800 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
2801 r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
2802 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
2803 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
2805 /* The first input is a 16-bit number. The second input is a 19-bit
2806 * number. Their sum is a 20-bit number.
2808 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2810 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2811 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
2813 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2815 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2816 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2817 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
2818 * generated, therefore the output r is always a 16-bit number.
2820 r0 = (r0 & 0xFFFF) + (r0 >> 16);
2823 r0 = r0 ? r0 : 0xFFFF;
2825 *dst16_ptr = (uint16_t)r0;
2829 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
2831 const struct instruction *ip)
2833 uint8_t *dst_struct, *src_struct;
2834 uint16_t *dst16_ptr;
2835 uint32_t *src32_ptr;
2839 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
2842 dst_struct = t->structs[ip->alu.dst.struct_id];
2843 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2845 src_struct = t->structs[ip->alu.src.struct_id];
2846 src32_ptr = (uint32_t *)&src_struct[0];
2848 /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
2849 * Therefore, in the worst case scenario, a 35-bit number is added to a
2850 * 16-bit number (the input r), so the output r is 36-bit number.
2852 for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
2855 /* The first input is a 16-bit number. The second input is a 20-bit
2856 * number. Their sum is a 21-bit number.
2858 r = (r & 0xFFFF) + (r >> 16);
2860 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
2861 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
2863 r = (r & 0xFFFF) + (r >> 16);
2865 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
2866 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
2867 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
2868 * generated, therefore the output r is always a 16-bit number.
2870 r = (r & 0xFFFF) + (r >> 16);
2875 *dst16_ptr = (uint16_t)r;
2881 static inline uint64_t *
2882 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
2884 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2888 static inline uint64_t
2889 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2891 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2893 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
2894 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
2895 uint64_t idx64 = *idx64_ptr;
2896 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
2897 uint64_t idx = idx64 & idx64_mask & r->size_mask;
2902 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2904 static inline uint64_t
2905 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2907 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2909 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
2910 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
2911 uint64_t idx64 = *idx64_ptr;
2912 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
2919 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
2923 static inline uint64_t
2924 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
2926 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
2928 uint64_t idx = ip->regarray.idx_val & r->size_mask;
2933 static inline uint64_t
2934 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
2936 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
2937 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
2938 uint64_t src64 = *src64_ptr;
2939 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2940 uint64_t src = src64 & src64_mask;
2945 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2947 static inline uint64_t
2948 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
2950 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
2951 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
2952 uint64_t src64 = *src64_ptr;
2953 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
2960 #define instr_regarray_src_nbo instr_regarray_src_hbo
2965 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
2967 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
2968 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
2969 uint64_t dst64 = *dst64_ptr;
2970 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2972 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
2976 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
2979 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
2981 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
2982 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
2983 uint64_t dst64 = *dst64_ptr;
2984 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
2986 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
2987 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
2992 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
2997 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
2999 const struct instruction *ip)
3001 uint64_t *regarray, idx;
3003 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3005 regarray = instr_regarray_regarray(p, ip);
3006 idx = instr_regarray_idx_nbo(p, t, ip);
3007 rte_prefetch0(®array[idx]);
3011 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3013 const struct instruction *ip)
3015 uint64_t *regarray, idx;
3017 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3019 regarray = instr_regarray_regarray(p, ip);
3020 idx = instr_regarray_idx_hbo(p, t, ip);
3021 rte_prefetch0(®array[idx]);
3025 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3026 struct thread *t __rte_unused,
3027 const struct instruction *ip)
3029 uint64_t *regarray, idx;
3031 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3033 regarray = instr_regarray_regarray(p, ip);
3034 idx = instr_regarray_idx_imm(p, ip);
3035 rte_prefetch0(®array[idx]);
3039 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3041 const struct instruction *ip)
3043 uint64_t *regarray, idx;
3045 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3047 regarray = instr_regarray_regarray(p, ip);
3048 idx = instr_regarray_idx_nbo(p, t, ip);
3049 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3053 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3055 const struct instruction *ip)
3057 uint64_t *regarray, idx;
3059 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3062 regarray = instr_regarray_regarray(p, ip);
3063 idx = instr_regarray_idx_hbo(p, t, ip);
3064 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3068 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3070 uint64_t *regarray, idx;
3072 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3074 regarray = instr_regarray_regarray(p, ip);
3075 idx = instr_regarray_idx_nbo(p, t, ip);
3076 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3080 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3082 uint64_t *regarray, idx;
3084 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3086 regarray = instr_regarray_regarray(p, ip);
3087 idx = instr_regarray_idx_hbo(p, t, ip);
3088 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3092 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3094 uint64_t *regarray, idx;
3096 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3098 regarray = instr_regarray_regarray(p, ip);
3099 idx = instr_regarray_idx_imm(p, ip);
3100 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3104 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3106 uint64_t *regarray, idx;
3108 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3110 regarray = instr_regarray_regarray(p, ip);
3111 idx = instr_regarray_idx_imm(p, ip);
3112 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3116 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3118 uint64_t *regarray, idx, src;
3120 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3122 regarray = instr_regarray_regarray(p, ip);
3123 idx = instr_regarray_idx_nbo(p, t, ip);
3124 src = instr_regarray_src_nbo(t, ip);
3125 regarray[idx] = src;
3129 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3131 uint64_t *regarray, idx, src;
3133 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3135 regarray = instr_regarray_regarray(p, ip);
3136 idx = instr_regarray_idx_nbo(p, t, ip);
3137 src = instr_regarray_src_hbo(t, ip);
3138 regarray[idx] = src;
3142 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3144 uint64_t *regarray, idx, src;
3146 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3148 regarray = instr_regarray_regarray(p, ip);
3149 idx = instr_regarray_idx_hbo(p, t, ip);
3150 src = instr_regarray_src_nbo(t, ip);
3151 regarray[idx] = src;
3155 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3157 uint64_t *regarray, idx, src;
3159 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3161 regarray = instr_regarray_regarray(p, ip);
3162 idx = instr_regarray_idx_hbo(p, t, ip);
3163 src = instr_regarray_src_hbo(t, ip);
3164 regarray[idx] = src;
3168 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3170 uint64_t *regarray, idx, src;
3172 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3174 regarray = instr_regarray_regarray(p, ip);
3175 idx = instr_regarray_idx_nbo(p, t, ip);
3176 src = ip->regarray.dstsrc_val;
3177 regarray[idx] = src;
3181 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3183 uint64_t *regarray, idx, src;
3185 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3187 regarray = instr_regarray_regarray(p, ip);
3188 idx = instr_regarray_idx_hbo(p, t, ip);
3189 src = ip->regarray.dstsrc_val;
3190 regarray[idx] = src;
3194 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3196 uint64_t *regarray, idx, src;
3198 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3200 regarray = instr_regarray_regarray(p, ip);
3201 idx = instr_regarray_idx_imm(p, ip);
3202 src = instr_regarray_src_nbo(t, ip);
3203 regarray[idx] = src;
3207 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3209 uint64_t *regarray, idx, src;
3211 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3213 regarray = instr_regarray_regarray(p, ip);
3214 idx = instr_regarray_idx_imm(p, ip);
3215 src = instr_regarray_src_hbo(t, ip);
3216 regarray[idx] = src;
3220 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3221 struct thread *t __rte_unused,
3222 const struct instruction *ip)
3224 uint64_t *regarray, idx, src;
3226 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3228 regarray = instr_regarray_regarray(p, ip);
3229 idx = instr_regarray_idx_imm(p, ip);
3230 src = ip->regarray.dstsrc_val;
3231 regarray[idx] = src;
3235 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3237 uint64_t *regarray, idx, src;
3239 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3241 regarray = instr_regarray_regarray(p, ip);
3242 idx = instr_regarray_idx_nbo(p, t, ip);
3243 src = instr_regarray_src_nbo(t, ip);
3244 regarray[idx] += src;
3248 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3250 uint64_t *regarray, idx, src;
3252 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3254 regarray = instr_regarray_regarray(p, ip);
3255 idx = instr_regarray_idx_nbo(p, t, ip);
3256 src = instr_regarray_src_hbo(t, ip);
3257 regarray[idx] += src;
3261 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3263 uint64_t *regarray, idx, src;
3265 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3267 regarray = instr_regarray_regarray(p, ip);
3268 idx = instr_regarray_idx_hbo(p, t, ip);
3269 src = instr_regarray_src_nbo(t, ip);
3270 regarray[idx] += src;
3274 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3276 uint64_t *regarray, idx, src;
3278 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3280 regarray = instr_regarray_regarray(p, ip);
3281 idx = instr_regarray_idx_hbo(p, t, ip);
3282 src = instr_regarray_src_hbo(t, ip);
3283 regarray[idx] += src;
3287 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3289 uint64_t *regarray, idx, src;
3291 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3293 regarray = instr_regarray_regarray(p, ip);
3294 idx = instr_regarray_idx_nbo(p, t, ip);
3295 src = ip->regarray.dstsrc_val;
3296 regarray[idx] += src;
3300 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3302 uint64_t *regarray, idx, src;
3304 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3306 regarray = instr_regarray_regarray(p, ip);
3307 idx = instr_regarray_idx_hbo(p, t, ip);
3308 src = ip->regarray.dstsrc_val;
3309 regarray[idx] += src;
3313 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3315 uint64_t *regarray, idx, src;
3317 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3319 regarray = instr_regarray_regarray(p, ip);
3320 idx = instr_regarray_idx_imm(p, ip);
3321 src = instr_regarray_src_nbo(t, ip);
3322 regarray[idx] += src;
3326 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3328 uint64_t *regarray, idx, src;
3330 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3332 regarray = instr_regarray_regarray(p, ip);
3333 idx = instr_regarray_idx_imm(p, ip);
3334 src = instr_regarray_src_hbo(t, ip);
3335 regarray[idx] += src;
3339 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3340 struct thread *t __rte_unused,
3341 const struct instruction *ip)
3343 uint64_t *regarray, idx, src;
3345 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3347 regarray = instr_regarray_regarray(p, ip);
3348 idx = instr_regarray_idx_imm(p, ip);
3349 src = ip->regarray.dstsrc_val;
3350 regarray[idx] += src;
3356 static inline struct meter *
3357 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3359 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3361 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3362 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3363 uint64_t idx64 = *idx64_ptr;
3364 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3365 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3367 return &r->metarray[idx];
3370 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3372 static inline struct meter *
3373 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3375 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3377 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3378 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3379 uint64_t idx64 = *idx64_ptr;
3380 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3382 return &r->metarray[idx];
3387 #define instr_meter_idx_nbo instr_meter_idx_hbo
3391 static inline struct meter *
3392 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3394 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3396 uint64_t idx = ip->meter.idx_val & r->size_mask;
3398 return &r->metarray[idx];
3401 static inline uint32_t
3402 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3404 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3405 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3406 uint64_t src64 = *src64_ptr;
3407 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3408 uint64_t src = src64 & src64_mask;
3410 return (uint32_t)src;
3413 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3415 static inline uint32_t
3416 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3418 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3419 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3420 uint64_t src64 = *src64_ptr;
3421 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3423 return (uint32_t)src;
3428 #define instr_meter_length_nbo instr_meter_length_hbo
3432 static inline enum rte_color
3433 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3435 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3436 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3437 uint64_t src64 = *src64_ptr;
3438 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3439 uint64_t src = src64 & src64_mask;
3441 return (enum rte_color)src;
3445 instr_meter_color_out_hbo_set(struct thread *t,
3446 const struct instruction *ip,
3447 enum rte_color color_out)
3449 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3450 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3451 uint64_t dst64 = *dst64_ptr;
3452 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3454 uint64_t src = (uint64_t)color_out;
3456 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3460 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3462 const struct instruction *ip)
3466 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3468 m = instr_meter_idx_nbo(p, t, ip);
3473 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3475 const struct instruction *ip)
3479 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3481 m = instr_meter_idx_hbo(p, t, ip);
3486 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3487 struct thread *t __rte_unused,
3488 const struct instruction *ip)
3492 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3494 m = instr_meter_idx_imm(p, ip);
3499 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3502 uint64_t time, n_pkts, n_bytes;
3504 enum rte_color color_in, color_out;
3506 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3508 m = instr_meter_idx_nbo(p, t, ip);
3509 rte_prefetch0(m->n_pkts);
3510 time = rte_get_tsc_cycles();
3511 length = instr_meter_length_nbo(t, ip);
3512 color_in = instr_meter_color_in_hbo(t, ip);
3514 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3515 &m->profile->profile,
3520 color_out &= m->color_mask;
3522 n_pkts = m->n_pkts[color_out];
3523 n_bytes = m->n_bytes[color_out];
3525 instr_meter_color_out_hbo_set(t, ip, color_out);
3527 m->n_pkts[color_out] = n_pkts + 1;
3528 m->n_bytes[color_out] = n_bytes + length;
3532 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3535 uint64_t time, n_pkts, n_bytes;
3537 enum rte_color color_in, color_out;
3539 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3541 m = instr_meter_idx_nbo(p, t, ip);
3542 rte_prefetch0(m->n_pkts);
3543 time = rte_get_tsc_cycles();
3544 length = instr_meter_length_nbo(t, ip);
3545 color_in = (enum rte_color)ip->meter.color_in_val;
3547 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3548 &m->profile->profile,
3553 color_out &= m->color_mask;
3555 n_pkts = m->n_pkts[color_out];
3556 n_bytes = m->n_bytes[color_out];
3558 instr_meter_color_out_hbo_set(t, ip, color_out);
3560 m->n_pkts[color_out] = n_pkts + 1;
3561 m->n_bytes[color_out] = n_bytes + length;
3565 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3568 uint64_t time, n_pkts, n_bytes;
3570 enum rte_color color_in, color_out;
3572 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
3574 m = instr_meter_idx_nbo(p, t, ip);
3575 rte_prefetch0(m->n_pkts);
3576 time = rte_get_tsc_cycles();
3577 length = instr_meter_length_hbo(t, ip);
3578 color_in = instr_meter_color_in_hbo(t, ip);
3580 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3581 &m->profile->profile,
3586 color_out &= m->color_mask;
3588 n_pkts = m->n_pkts[color_out];
3589 n_bytes = m->n_bytes[color_out];
3591 instr_meter_color_out_hbo_set(t, ip, color_out);
3593 m->n_pkts[color_out] = n_pkts + 1;
3594 m->n_bytes[color_out] = n_bytes + length;
3598 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3601 uint64_t time, n_pkts, n_bytes;
3603 enum rte_color color_in, color_out;
3605 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
3607 m = instr_meter_idx_nbo(p, t, ip);
3608 rte_prefetch0(m->n_pkts);
3609 time = rte_get_tsc_cycles();
3610 length = instr_meter_length_hbo(t, ip);
3611 color_in = (enum rte_color)ip->meter.color_in_val;
3613 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3614 &m->profile->profile,
3619 color_out &= m->color_mask;
3621 n_pkts = m->n_pkts[color_out];
3622 n_bytes = m->n_bytes[color_out];
3624 instr_meter_color_out_hbo_set(t, ip, color_out);
3626 m->n_pkts[color_out] = n_pkts + 1;
3627 m->n_bytes[color_out] = n_bytes + length;
3631 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3634 uint64_t time, n_pkts, n_bytes;
3636 enum rte_color color_in, color_out;
3638 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
3640 m = instr_meter_idx_hbo(p, t, ip);
3641 rte_prefetch0(m->n_pkts);
3642 time = rte_get_tsc_cycles();
3643 length = instr_meter_length_nbo(t, ip);
3644 color_in = instr_meter_color_in_hbo(t, ip);
3646 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3647 &m->profile->profile,
3652 color_out &= m->color_mask;
3654 n_pkts = m->n_pkts[color_out];
3655 n_bytes = m->n_bytes[color_out];
3657 instr_meter_color_out_hbo_set(t, ip, color_out);
3659 m->n_pkts[color_out] = n_pkts + 1;
3660 m->n_bytes[color_out] = n_bytes + length;
3664 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3667 uint64_t time, n_pkts, n_bytes;
3669 enum rte_color color_in, color_out;
3671 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
3673 m = instr_meter_idx_hbo(p, t, ip);
3674 rte_prefetch0(m->n_pkts);
3675 time = rte_get_tsc_cycles();
3676 length = instr_meter_length_nbo(t, ip);
3677 color_in = (enum rte_color)ip->meter.color_in_val;
3679 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3680 &m->profile->profile,
3685 color_out &= m->color_mask;
3687 n_pkts = m->n_pkts[color_out];
3688 n_bytes = m->n_bytes[color_out];
3690 instr_meter_color_out_hbo_set(t, ip, color_out);
3692 m->n_pkts[color_out] = n_pkts + 1;
3693 m->n_bytes[color_out] = n_bytes + length;
3697 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3700 uint64_t time, n_pkts, n_bytes;
3702 enum rte_color color_in, color_out;
3704 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
3706 m = instr_meter_idx_hbo(p, t, ip);
3707 rte_prefetch0(m->n_pkts);
3708 time = rte_get_tsc_cycles();
3709 length = instr_meter_length_hbo(t, ip);
3710 color_in = instr_meter_color_in_hbo(t, ip);
3712 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3713 &m->profile->profile,
3718 color_out &= m->color_mask;
3720 n_pkts = m->n_pkts[color_out];
3721 n_bytes = m->n_bytes[color_out];
3723 instr_meter_color_out_hbo_set(t, ip, color_out);
3725 m->n_pkts[color_out] = n_pkts + 1;
3726 m->n_bytes[color_out] = n_bytes + length;
3730 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3733 uint64_t time, n_pkts, n_bytes;
3735 enum rte_color color_in, color_out;
3737 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
3739 m = instr_meter_idx_hbo(p, t, ip);
3740 rte_prefetch0(m->n_pkts);
3741 time = rte_get_tsc_cycles();
3742 length = instr_meter_length_hbo(t, ip);
3743 color_in = (enum rte_color)ip->meter.color_in_val;
3745 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3746 &m->profile->profile,
3751 color_out &= m->color_mask;
3753 n_pkts = m->n_pkts[color_out];
3754 n_bytes = m->n_bytes[color_out];
3756 instr_meter_color_out_hbo_set(t, ip, color_out);
3758 m->n_pkts[color_out] = n_pkts + 1;
3759 m->n_bytes[color_out] = n_bytes + length;
3763 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3766 uint64_t time, n_pkts, n_bytes;
3768 enum rte_color color_in, color_out;
3770 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
3772 m = instr_meter_idx_imm(p, ip);
3773 rte_prefetch0(m->n_pkts);
3774 time = rte_get_tsc_cycles();
3775 length = instr_meter_length_nbo(t, ip);
3776 color_in = instr_meter_color_in_hbo(t, ip);
3778 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3779 &m->profile->profile,
3784 color_out &= m->color_mask;
3786 n_pkts = m->n_pkts[color_out];
3787 n_bytes = m->n_bytes[color_out];
3789 instr_meter_color_out_hbo_set(t, ip, color_out);
3791 m->n_pkts[color_out] = n_pkts + 1;
3792 m->n_bytes[color_out] = n_bytes + length;
3796 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3799 uint64_t time, n_pkts, n_bytes;
3801 enum rte_color color_in, color_out;
3803 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
3805 m = instr_meter_idx_imm(p, ip);
3806 rte_prefetch0(m->n_pkts);
3807 time = rte_get_tsc_cycles();
3808 length = instr_meter_length_nbo(t, ip);
3809 color_in = (enum rte_color)ip->meter.color_in_val;
3811 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3812 &m->profile->profile,
3817 color_out &= m->color_mask;
3819 n_pkts = m->n_pkts[color_out];
3820 n_bytes = m->n_bytes[color_out];
3822 instr_meter_color_out_hbo_set(t, ip, color_out);
3824 m->n_pkts[color_out] = n_pkts + 1;
3825 m->n_bytes[color_out] = n_bytes + length;
3829 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3832 uint64_t time, n_pkts, n_bytes;
3834 enum rte_color color_in, color_out;
3836 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
3838 m = instr_meter_idx_imm(p, ip);
3839 rte_prefetch0(m->n_pkts);
3840 time = rte_get_tsc_cycles();
3841 length = instr_meter_length_hbo(t, ip);
3842 color_in = instr_meter_color_in_hbo(t, ip);
3844 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3845 &m->profile->profile,
3850 color_out &= m->color_mask;
3852 n_pkts = m->n_pkts[color_out];
3853 n_bytes = m->n_bytes[color_out];
3855 instr_meter_color_out_hbo_set(t, ip, color_out);
3857 m->n_pkts[color_out] = n_pkts + 1;
3858 m->n_bytes[color_out] = n_bytes + length;
3862 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3865 uint64_t time, n_pkts, n_bytes;
3867 enum rte_color color_in, color_out;
3869 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
3871 m = instr_meter_idx_imm(p, ip);
3872 rte_prefetch0(m->n_pkts);
3873 time = rte_get_tsc_cycles();
3874 length = instr_meter_length_hbo(t, ip);
3875 color_in = (enum rte_color)ip->meter.color_in_val;
3877 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3878 &m->profile->profile,
3883 color_out &= m->color_mask;
3885 n_pkts = m->n_pkts[color_out];
3886 n_bytes = m->n_bytes[color_out];
3888 instr_meter_color_out_hbo_set(t, ip, color_out);
3890 m->n_pkts[color_out] = n_pkts + 1;
3891 m->n_bytes[color_out] = n_bytes + length;