1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
4 #ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
5 #define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
11 #include <rte_byteorder.h>
12 #include <rte_common.h>
13 #include <rte_cycles.h>
14 #include <rte_prefetch.h>
15 #include <rte_meter.h>
17 #include <rte_swx_table_selector.h>
18 #include <rte_swx_table_learner.h>
19 #include <rte_swx_pipeline.h>
20 #include <rte_swx_ctl.h>
27 #define TRACE(...) printf(__VA_ARGS__)
35 #define ntoh64(x) rte_be_to_cpu_64(x)
36 #define hton64(x) rte_cpu_to_be_64(x)
42 char name[RTE_SWX_NAME_SIZE];
49 TAILQ_ENTRY(struct_type) node;
50 char name[RTE_SWX_NAME_SIZE];
58 TAILQ_HEAD(struct_type_tailq, struct_type);
64 TAILQ_ENTRY(port_in_type) node;
65 char name[RTE_SWX_NAME_SIZE];
66 struct rte_swx_port_in_ops ops;
69 TAILQ_HEAD(port_in_type_tailq, port_in_type);
72 TAILQ_ENTRY(port_in) node;
73 struct port_in_type *type;
78 TAILQ_HEAD(port_in_tailq, port_in);
80 struct port_in_runtime {
81 rte_swx_port_in_pkt_rx_t pkt_rx;
88 struct port_out_type {
89 TAILQ_ENTRY(port_out_type) node;
90 char name[RTE_SWX_NAME_SIZE];
91 struct rte_swx_port_out_ops ops;
94 TAILQ_HEAD(port_out_type_tailq, port_out_type);
97 TAILQ_ENTRY(port_out) node;
98 struct port_out_type *type;
103 TAILQ_HEAD(port_out_tailq, port_out);
105 struct port_out_runtime {
106 rte_swx_port_out_pkt_tx_t pkt_tx;
107 rte_swx_port_out_pkt_fast_clone_tx_t pkt_fast_clone_tx;
108 rte_swx_port_out_pkt_clone_tx_t pkt_clone_tx;
109 rte_swx_port_out_flush_t flush;
116 struct mirroring_session {
119 uint32_t truncation_length;
125 struct extern_type_member_func {
126 TAILQ_ENTRY(extern_type_member_func) node;
127 char name[RTE_SWX_NAME_SIZE];
128 rte_swx_extern_type_member_func_t func;
132 TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
135 TAILQ_ENTRY(extern_type) node;
136 char name[RTE_SWX_NAME_SIZE];
137 struct struct_type *mailbox_struct_type;
138 rte_swx_extern_type_constructor_t constructor;
139 rte_swx_extern_type_destructor_t destructor;
140 struct extern_type_member_func_tailq funcs;
144 TAILQ_HEAD(extern_type_tailq, extern_type);
147 TAILQ_ENTRY(extern_obj) node;
148 char name[RTE_SWX_NAME_SIZE];
149 struct extern_type *type;
155 TAILQ_HEAD(extern_obj_tailq, extern_obj);
157 #ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
158 #define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
161 struct extern_obj_runtime {
164 rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
171 TAILQ_ENTRY(extern_func) node;
172 char name[RTE_SWX_NAME_SIZE];
173 struct struct_type *mailbox_struct_type;
174 rte_swx_extern_func_t func;
179 TAILQ_HEAD(extern_func_tailq, extern_func);
181 struct extern_func_runtime {
183 rte_swx_extern_func_t func;
190 TAILQ_ENTRY(hash_func) node;
191 char name[RTE_SWX_NAME_SIZE];
192 rte_swx_hash_func_t func;
196 TAILQ_HEAD(hash_func_tailq, hash_func);
198 struct hash_func_runtime {
199 rte_swx_hash_func_t func;
206 TAILQ_ENTRY(header) node;
207 char name[RTE_SWX_NAME_SIZE];
208 struct struct_type *st;
213 TAILQ_HEAD(header_tailq, header);
215 struct header_runtime {
220 struct header_out_runtime {
230 /* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
231 * Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
232 * Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
233 * when transferred to packet meta-data and in NBO when transferred to packet
237 /* Notation conventions:
238 * -Header field: H = h.header.field (dst/src)
239 * -Meta-data field: M = m.field (dst/src)
240 * -Extern object mailbox field: E = e.field (dst/src)
241 * -Extern function mailbox field: F = f.field (dst/src)
242 * -Table action data field: T = t.field (src only)
243 * -Immediate value: I = 32-bit unsigned value (src only)
246 enum instruction_type {
253 INSTR_TX, /* port_out = M */
254 INSTR_TX_I, /* port_out = I */
258 * mirror slot_id session_id
268 /* recircid m.recirc_pass_id
269 * Read the internal recirculation pass ID into the specified meta-data field.
273 /* extract h.header */
283 /* extract h.header m.last_field_size */
286 /* lookahead h.header */
300 /* validate h.header */
303 /* invalidate h.header */
304 INSTR_HDR_INVALIDATE,
308 * dst = HMEF, src = HMEFTI
310 INSTR_MOV, /* dst = MEF, src = MEFT */
311 INSTR_MOV_MH, /* dst = MEF, src = H */
312 INSTR_MOV_HM, /* dst = H, src = MEFT */
313 INSTR_MOV_HH, /* dst = H, src = H */
314 INSTR_MOV_I, /* dst = HMEF, src = I */
316 /* dma h.header t.field
317 * memcpy(h.header, t.field, sizeof(h.header))
330 * dst = HMEF, src = HMEFTI
332 INSTR_ALU_ADD, /* dst = MEF, src = MEF */
333 INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
334 INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
335 INSTR_ALU_ADD_HH, /* dst = H, src = H */
336 INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
337 INSTR_ALU_ADD_HI, /* dst = H, src = I */
341 * dst = HMEF, src = HMEFTI
343 INSTR_ALU_SUB, /* dst = MEF, src = MEF */
344 INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
345 INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
346 INSTR_ALU_SUB_HH, /* dst = H, src = H */
347 INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
348 INSTR_ALU_SUB_HI, /* dst = H, src = I */
351 * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
352 * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
354 INSTR_ALU_CKADD_FIELD, /* src = H */
355 INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
356 INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
360 * dst = H, src = H, '- = 1's complement subtraction operator
362 INSTR_ALU_CKSUB_FIELD,
366 * dst = HMEF, src = HMEFTI
368 INSTR_ALU_AND, /* dst = MEF, src = MEFT */
369 INSTR_ALU_AND_MH, /* dst = MEF, src = H */
370 INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
371 INSTR_ALU_AND_HH, /* dst = H, src = H */
372 INSTR_ALU_AND_I, /* dst = HMEF, src = I */
376 * dst = HMEF, src = HMEFTI
378 INSTR_ALU_OR, /* dst = MEF, src = MEFT */
379 INSTR_ALU_OR_MH, /* dst = MEF, src = H */
380 INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
381 INSTR_ALU_OR_HH, /* dst = H, src = H */
382 INSTR_ALU_OR_I, /* dst = HMEF, src = I */
386 * dst = HMEF, src = HMEFTI
388 INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
389 INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
390 INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
391 INSTR_ALU_XOR_HH, /* dst = H, src = H */
392 INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
396 * dst = HMEF, src = HMEFTI
398 INSTR_ALU_SHL, /* dst = MEF, src = MEF */
399 INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
400 INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
401 INSTR_ALU_SHL_HH, /* dst = H, src = H */
402 INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
403 INSTR_ALU_SHL_HI, /* dst = H, src = I */
407 * dst = HMEF, src = HMEFTI
409 INSTR_ALU_SHR, /* dst = MEF, src = MEF */
410 INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
411 INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
412 INSTR_ALU_SHR_HH, /* dst = H, src = H */
413 INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
414 INSTR_ALU_SHR_HI, /* dst = H, src = I */
416 /* regprefetch REGARRAY index
417 * prefetch REGARRAY[index]
420 INSTR_REGPREFETCH_RH, /* index = H */
421 INSTR_REGPREFETCH_RM, /* index = MEFT */
422 INSTR_REGPREFETCH_RI, /* index = I */
424 /* regrd dst REGARRAY index
425 * dst = REGARRAY[index]
426 * dst = HMEF, index = HMEFTI
428 INSTR_REGRD_HRH, /* dst = H, index = H */
429 INSTR_REGRD_HRM, /* dst = H, index = MEFT */
430 INSTR_REGRD_HRI, /* dst = H, index = I */
431 INSTR_REGRD_MRH, /* dst = MEF, index = H */
432 INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
433 INSTR_REGRD_MRI, /* dst = MEF, index = I */
435 /* regwr REGARRAY index src
436 * REGARRAY[index] = src
437 * index = HMEFTI, src = HMEFTI
439 INSTR_REGWR_RHH, /* index = H, src = H */
440 INSTR_REGWR_RHM, /* index = H, src = MEFT */
441 INSTR_REGWR_RHI, /* index = H, src = I */
442 INSTR_REGWR_RMH, /* index = MEFT, src = H */
443 INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
444 INSTR_REGWR_RMI, /* index = MEFT, src = I */
445 INSTR_REGWR_RIH, /* index = I, src = H */
446 INSTR_REGWR_RIM, /* index = I, src = MEFT */
447 INSTR_REGWR_RII, /* index = I, src = I */
449 /* regadd REGARRAY index src
450 * REGARRAY[index] += src
451 * index = HMEFTI, src = HMEFTI
453 INSTR_REGADD_RHH, /* index = H, src = H */
454 INSTR_REGADD_RHM, /* index = H, src = MEFT */
455 INSTR_REGADD_RHI, /* index = H, src = I */
456 INSTR_REGADD_RMH, /* index = MEFT, src = H */
457 INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
458 INSTR_REGADD_RMI, /* index = MEFT, src = I */
459 INSTR_REGADD_RIH, /* index = I, src = H */
460 INSTR_REGADD_RIM, /* index = I, src = MEFT */
461 INSTR_REGADD_RII, /* index = I, src = I */
463 /* metprefetch METARRAY index
464 * prefetch METARRAY[index]
467 INSTR_METPREFETCH_H, /* index = H */
468 INSTR_METPREFETCH_M, /* index = MEFT */
469 INSTR_METPREFETCH_I, /* index = I */
471 /* meter METARRAY index length color_in color_out
472 * color_out = meter(METARRAY[index], length, color_in)
473 * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
475 INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
476 INSTR_METER_HHI, /* index = H, length = H, color_in = I */
477 INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
478 INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
479 INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
480 INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
481 INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
482 INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
483 INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
484 INSTR_METER_IHI, /* index = I, length = H, color_in = I */
485 INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
486 INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
495 /* learn ACTION_NAME [ m.action_first_arg ] m.timeout_id */
498 /* rearm [ m.timeout_id ] */
500 INSTR_LEARNER_REARM_NEW,
503 INSTR_LEARNER_FORGET,
505 /* extern e.obj.func */
511 /* hash HASH_FUNC_NAME dst src_first src_last
512 * Compute hash value over range of struct fields.
516 * src_first and src_last must be fields within the same struct
525 /* jmpv LABEL h.header
526 * Jump if header is valid
530 /* jmpnv LABEL h.header
531 * Jump if header is invalid
536 * Jump if table lookup hit
541 * Jump if table lookup miss
548 INSTR_JMP_ACTION_HIT,
550 /* jmpna LABEL ACTION
551 * Jump if action not run
553 INSTR_JMP_ACTION_MISS,
556 * Jump if a is equal to b
557 * a = HMEFT, b = HMEFTI
559 INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
560 INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
561 INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
562 INSTR_JMP_EQ_HH, /* a = H, b = H */
563 INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
566 * Jump if a is not equal to b
567 * a = HMEFT, b = HMEFTI
569 INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
570 INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
571 INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
572 INSTR_JMP_NEQ_HH, /* a = H, b = H */
573 INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
576 * Jump if a is less than b
577 * a = HMEFT, b = HMEFTI
579 INSTR_JMP_LT, /* a = MEFT, b = MEFT */
580 INSTR_JMP_LT_MH, /* a = MEFT, b = H */
581 INSTR_JMP_LT_HM, /* a = H, b = MEFT */
582 INSTR_JMP_LT_HH, /* a = H, b = H */
583 INSTR_JMP_LT_MI, /* a = MEFT, b = I */
584 INSTR_JMP_LT_HI, /* a = H, b = I */
587 * Jump if a is greater than b
588 * a = HMEFT, b = HMEFTI
590 INSTR_JMP_GT, /* a = MEFT, b = MEFT */
591 INSTR_JMP_GT_MH, /* a = MEFT, b = H */
592 INSTR_JMP_GT_HM, /* a = H, b = MEFT */
593 INSTR_JMP_GT_HH, /* a = H, b = H */
594 INSTR_JMP_GT_MI, /* a = MEFT, b = I */
595 INSTR_JMP_GT_HI, /* a = H, b = I */
602 /* Start of custom instructions. */
606 struct instr_operand {
627 uint8_t header_id[8];
628 uint8_t struct_id[8];
633 struct instr_hdr_validity {
643 uint8_t mf_first_arg_offset;
644 uint8_t mf_timeout_id_offset;
645 uint8_t mf_timeout_id_n_bits;
648 struct instr_extern_obj {
653 struct instr_extern_func {
657 struct instr_hash_func {
658 uint8_t hash_func_id;
672 struct instr_dst_src {
673 struct instr_operand dst;
675 struct instr_operand src;
680 struct instr_regarray {
685 struct instr_operand idx;
690 struct instr_operand dstsrc;
700 struct instr_operand idx;
704 struct instr_operand length;
707 struct instr_operand color_in;
708 uint32_t color_in_val;
711 struct instr_operand color_out;
716 uint8_t header_id[8];
717 uint8_t struct_id[8];
728 struct instruction *ip;
731 struct instr_operand a;
737 struct instr_operand b;
743 enum instruction_type type;
746 struct instr_dst_src mirror;
747 struct instr_hdr_validity valid;
748 struct instr_dst_src mov;
749 struct instr_regarray regarray;
750 struct instr_meter meter;
751 struct instr_dma dma;
752 struct instr_dst_src alu;
753 struct instr_table table;
754 struct instr_learn learn;
755 struct instr_extern_obj ext_obj;
756 struct instr_extern_func ext_func;
757 struct instr_hash_func hash_func;
758 struct instr_jmp jmp;
762 struct instruction_data {
763 char label[RTE_SWX_NAME_SIZE];
764 char jmp_label[RTE_SWX_NAME_SIZE];
765 uint32_t n_users; /* user = jmp instruction to this instruction. */
769 typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
775 (*action_func_t)(struct rte_swx_pipeline *p);
778 TAILQ_ENTRY(action) node;
779 char name[RTE_SWX_NAME_SIZE];
780 struct struct_type *st;
781 int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
782 struct instruction *instructions;
783 struct instruction_data *instruction_data;
784 uint32_t n_instructions;
788 TAILQ_HEAD(action_tailq, action);
794 TAILQ_ENTRY(table_type) node;
795 char name[RTE_SWX_NAME_SIZE];
796 enum rte_swx_table_match_type match_type;
797 struct rte_swx_table_ops ops;
800 TAILQ_HEAD(table_type_tailq, table_type);
803 enum rte_swx_table_match_type match_type;
808 TAILQ_ENTRY(table) node;
809 char name[RTE_SWX_NAME_SIZE];
810 char args[RTE_SWX_NAME_SIZE];
811 struct table_type *type; /* NULL when n_fields == 0. */
814 struct match_field *fields;
816 struct header *header; /* Only valid when n_fields > 0. */
819 struct action **actions;
820 struct action *default_action;
821 uint8_t *default_action_data;
823 int default_action_is_const;
824 uint32_t action_data_size_max;
825 int *action_is_for_table_entries;
826 int *action_is_for_default_entry;
832 TAILQ_HEAD(table_tailq, table);
834 struct table_runtime {
835 rte_swx_table_lookup_t func;
840 struct table_statistics {
841 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
842 uint64_t *n_pkts_action;
849 TAILQ_ENTRY(selector) node;
850 char name[RTE_SWX_NAME_SIZE];
852 struct field *group_id_field;
853 struct field **selector_fields;
854 uint32_t n_selector_fields;
855 struct header *selector_header;
856 struct field *member_id_field;
858 uint32_t n_groups_max;
859 uint32_t n_members_per_group_max;
864 TAILQ_HEAD(selector_tailq, selector);
866 struct selector_runtime {
868 uint8_t **group_id_buffer;
869 uint8_t **selector_buffer;
870 uint8_t **member_id_buffer;
873 struct selector_statistics {
881 TAILQ_ENTRY(learner) node;
882 char name[RTE_SWX_NAME_SIZE];
885 struct field **fields;
887 struct header *header;
890 struct action **actions;
891 struct action *default_action;
892 uint8_t *default_action_data;
894 int default_action_is_const;
895 uint32_t action_data_size_max;
896 int *action_is_for_table_entries;
897 int *action_is_for_default_entry;
900 uint32_t timeout[RTE_SWX_TABLE_LEARNER_N_KEY_TIMEOUTS_MAX];
905 TAILQ_HEAD(learner_tailq, learner);
907 struct learner_runtime {
912 struct learner_statistics {
913 uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
914 uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
915 uint64_t n_pkts_rearm;
916 uint64_t n_pkts_forget;
917 uint64_t *n_pkts_action;
924 TAILQ_ENTRY(regarray) node;
925 char name[RTE_SWX_NAME_SIZE];
931 TAILQ_HEAD(regarray_tailq, regarray);
933 struct regarray_runtime {
941 struct meter_profile {
942 TAILQ_ENTRY(meter_profile) node;
943 char name[RTE_SWX_NAME_SIZE];
944 struct rte_meter_trtcm_params params;
945 struct rte_meter_trtcm_profile profile;
949 TAILQ_HEAD(meter_profile_tailq, meter_profile);
952 TAILQ_ENTRY(metarray) node;
953 char name[RTE_SWX_NAME_SIZE];
958 TAILQ_HEAD(metarray_tailq, metarray);
961 struct rte_meter_trtcm m;
962 struct meter_profile *profile;
963 enum rte_color color_mask;
966 uint64_t n_pkts[RTE_COLORS];
967 uint64_t n_bytes[RTE_COLORS];
970 struct metarray_runtime {
971 struct meter *metarray;
980 struct rte_swx_pkt pkt;
982 uint32_t *mirroring_slots;
983 uint64_t mirroring_slots_mask;
985 uint32_t recirc_pass_id;
990 /* Packet headers. */
991 struct header_runtime *headers; /* Extracted or generated headers. */
992 struct header_out_runtime *headers_out; /* Emitted headers. */
993 uint8_t *header_storage;
994 uint8_t *header_out_storage;
995 uint64_t valid_headers;
996 uint32_t n_headers_out;
998 /* Packet meta-data. */
1002 struct table_runtime *tables;
1003 struct selector_runtime *selectors;
1004 struct learner_runtime *learners;
1005 struct rte_swx_table_state *table_state;
1007 int hit; /* 0 = Miss, 1 = Hit. */
1008 uint32_t learner_id;
1011 /* Extern objects and functions. */
1012 struct extern_obj_runtime *extern_objs;
1013 struct extern_func_runtime *extern_funcs;
1016 struct instruction *ip;
1017 struct instruction *ret;
1020 #define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
1021 #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
1022 #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
1024 #define HEADER_VALID(thread, header_id) \
1025 MASK64_BIT_GET((thread)->valid_headers, header_id)
1027 static inline uint64_t
1028 instr_operand_hbo(struct thread *t, const struct instr_operand *x)
1030 uint8_t *x_struct = t->structs[x->struct_id];
1031 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1032 uint64_t x64 = *x64_ptr;
1033 uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
1035 return x64 & x64_mask;
1038 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1040 static inline uint64_t
1041 instr_operand_nbo(struct thread *t, const struct instr_operand *x)
1043 uint8_t *x_struct = t->structs[x->struct_id];
1044 uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
1045 uint64_t x64 = *x64_ptr;
1047 return ntoh64(x64) >> (64 - x->n_bits);
1052 #define instr_operand_nbo instr_operand_hbo
1056 #define ALU(thread, ip, operator) \
1058 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1059 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1060 uint64_t dst64 = *dst64_ptr; \
1061 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1062 uint64_t dst = dst64 & dst64_mask; \
1064 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1065 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1066 uint64_t src64 = *src64_ptr; \
1067 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1068 uint64_t src = src64 & src64_mask; \
1070 uint64_t result = dst operator src; \
1072 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1075 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1077 #define ALU_MH(thread, ip, operator) \
1079 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1080 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1081 uint64_t dst64 = *dst64_ptr; \
1082 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1083 uint64_t dst = dst64 & dst64_mask; \
1085 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1086 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1087 uint64_t src64 = *src64_ptr; \
1088 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1090 uint64_t result = dst operator src; \
1092 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1095 #define ALU_HM(thread, ip, operator) \
1097 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1098 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1099 uint64_t dst64 = *dst64_ptr; \
1100 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1101 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1103 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1104 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1105 uint64_t src64 = *src64_ptr; \
1106 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1107 uint64_t src = src64 & src64_mask; \
1109 uint64_t result = dst operator src; \
1110 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1112 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1115 #define ALU_HM_FAST(thread, ip, operator) \
1117 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1118 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1119 uint64_t dst64 = *dst64_ptr; \
1120 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1121 uint64_t dst = dst64 & dst64_mask; \
1123 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1124 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1125 uint64_t src64 = *src64_ptr; \
1126 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
1127 uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
1129 uint64_t result = dst operator src; \
1131 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1134 #define ALU_HH(thread, ip, operator) \
1136 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1137 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1138 uint64_t dst64 = *dst64_ptr; \
1139 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1140 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1142 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1143 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1144 uint64_t src64 = *src64_ptr; \
1145 uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
1147 uint64_t result = dst operator src; \
1148 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1150 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1153 #define ALU_HH_FAST(thread, ip, operator) \
1155 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1156 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1157 uint64_t dst64 = *dst64_ptr; \
1158 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1159 uint64_t dst = dst64 & dst64_mask; \
1161 uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
1162 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
1163 uint64_t src64 = *src64_ptr; \
1164 uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
1166 uint64_t result = dst operator src; \
1168 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1175 #define ALU_HM_FAST ALU
1177 #define ALU_HH_FAST ALU
1181 #define ALU_I(thread, ip, operator) \
1183 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1184 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1185 uint64_t dst64 = *dst64_ptr; \
1186 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1187 uint64_t dst = dst64 & dst64_mask; \
1189 uint64_t src = (ip)->alu.src_val; \
1191 uint64_t result = dst operator src; \
1193 *dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
1196 #define ALU_MI ALU_I
1198 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1200 #define ALU_HI(thread, ip, operator) \
1202 uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
1203 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
1204 uint64_t dst64 = *dst64_ptr; \
1205 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
1206 uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
1208 uint64_t src = (ip)->alu.src_val; \
1210 uint64_t result = dst operator src; \
1211 result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
1213 *dst64_ptr = (dst64 & ~dst64_mask) | result; \
1218 #define ALU_HI ALU_I
1222 #define MOV(thread, ip) \
1224 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1225 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1226 uint64_t dst64 = *dst64_ptr; \
1227 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1229 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1230 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1231 uint64_t src64 = *src64_ptr; \
1232 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1233 uint64_t src = src64 & src64_mask; \
1235 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1238 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1240 #define MOV_MH(thread, ip) \
1242 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1243 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1244 uint64_t dst64 = *dst64_ptr; \
1245 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1247 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1248 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1249 uint64_t src64 = *src64_ptr; \
1250 uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
1252 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1255 #define MOV_HM(thread, ip) \
1257 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1258 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1259 uint64_t dst64 = *dst64_ptr; \
1260 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1262 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1263 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1264 uint64_t src64 = *src64_ptr; \
1265 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
1266 uint64_t src = src64 & src64_mask; \
1268 src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
1269 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1272 #define MOV_HH(thread, ip) \
1274 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1275 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1276 uint64_t dst64 = *dst64_ptr; \
1277 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1279 uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
1280 uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
1281 uint64_t src64 = *src64_ptr; \
1283 uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
1284 src = src >> (64 - (ip)->mov.dst.n_bits); \
1285 *dst64_ptr = (dst64 & ~dst64_mask) | src; \
1296 #define MOV_I(thread, ip) \
1298 uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
1299 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
1300 uint64_t dst64 = *dst64_ptr; \
1301 uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
1303 uint64_t src = (ip)->mov.src_val; \
1305 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
1308 #define JMP_CMP(thread, ip, operator) \
1310 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1311 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1312 uint64_t a64 = *a64_ptr; \
1313 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1314 uint64_t a = a64 & a64_mask; \
1316 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1317 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1318 uint64_t b64 = *b64_ptr; \
1319 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1320 uint64_t b = b64 & b64_mask; \
1322 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1325 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1327 #define JMP_CMP_MH(thread, ip, operator) \
1329 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1330 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1331 uint64_t a64 = *a64_ptr; \
1332 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1333 uint64_t a = a64 & a64_mask; \
1335 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1336 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1337 uint64_t b64 = *b64_ptr; \
1338 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1340 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1343 #define JMP_CMP_HM(thread, ip, operator) \
1345 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1346 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1347 uint64_t a64 = *a64_ptr; \
1348 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1350 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1351 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1352 uint64_t b64 = *b64_ptr; \
1353 uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
1354 uint64_t b = b64 & b64_mask; \
1356 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1359 #define JMP_CMP_HH(thread, ip, operator) \
1361 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1362 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1363 uint64_t a64 = *a64_ptr; \
1364 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1366 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1367 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1368 uint64_t b64 = *b64_ptr; \
1369 uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
1371 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1374 #define JMP_CMP_HH_FAST(thread, ip, operator) \
1376 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1377 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1378 uint64_t a64 = *a64_ptr; \
1379 uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
1381 uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
1382 uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
1383 uint64_t b64 = *b64_ptr; \
1384 uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
1386 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1391 #define JMP_CMP_MH JMP_CMP
1392 #define JMP_CMP_HM JMP_CMP
1393 #define JMP_CMP_HH JMP_CMP
1394 #define JMP_CMP_HH_FAST JMP_CMP
1398 #define JMP_CMP_I(thread, ip, operator) \
1400 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1401 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1402 uint64_t a64 = *a64_ptr; \
1403 uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
1404 uint64_t a = a64 & a64_mask; \
1406 uint64_t b = (ip)->jmp.b_val; \
1408 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1411 #define JMP_CMP_MI JMP_CMP_I
1413 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1415 #define JMP_CMP_HI(thread, ip, operator) \
1417 uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
1418 uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
1419 uint64_t a64 = *a64_ptr; \
1420 uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
1422 uint64_t b = (ip)->jmp.b_val; \
1424 (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
1429 #define JMP_CMP_HI JMP_CMP_I
1433 #define METADATA_READ(thread, offset, n_bits) \
1435 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1436 uint64_t m64 = *m64_ptr; \
1437 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1441 #define METADATA_WRITE(thread, offset, n_bits, value) \
1443 uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
1444 uint64_t m64 = *m64_ptr; \
1445 uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
1447 uint64_t m_new = value; \
1449 *m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
1452 #ifndef RTE_SWX_PIPELINE_THREADS_MAX
1453 #define RTE_SWX_PIPELINE_THREADS_MAX 16
1456 #ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
1457 #define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
1460 struct rte_swx_pipeline {
1461 struct struct_type_tailq struct_types;
1462 struct port_in_type_tailq port_in_types;
1463 struct port_in_tailq ports_in;
1464 struct port_out_type_tailq port_out_types;
1465 struct port_out_tailq ports_out;
1466 struct extern_type_tailq extern_types;
1467 struct extern_obj_tailq extern_objs;
1468 struct extern_func_tailq extern_funcs;
1469 struct hash_func_tailq hash_funcs;
1470 struct header_tailq headers;
1471 struct struct_type *metadata_st;
1472 uint32_t metadata_struct_id;
1473 struct action_tailq actions;
1474 struct table_type_tailq table_types;
1475 struct table_tailq tables;
1476 struct selector_tailq selectors;
1477 struct learner_tailq learners;
1478 struct regarray_tailq regarrays;
1479 struct meter_profile_tailq meter_profiles;
1480 struct metarray_tailq metarrays;
1482 struct port_in_runtime *in;
1483 struct port_out_runtime *out;
1484 struct mirroring_session *mirroring_sessions;
1485 struct instruction **action_instructions;
1486 action_func_t *action_funcs;
1487 struct rte_swx_table_state *table_state;
1488 struct table_statistics *table_stats;
1489 struct selector_statistics *selector_stats;
1490 struct learner_statistics *learner_stats;
1491 struct hash_func_runtime *hash_func_runtime;
1492 struct regarray_runtime *regarray_runtime;
1493 struct metarray_runtime *metarray_runtime;
1494 struct instruction *instructions;
1495 struct instruction_data *instruction_data;
1496 instr_exec_t *instruction_table;
1497 struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
1501 uint32_t n_ports_in;
1502 uint32_t n_ports_out;
1503 uint32_t n_mirroring_slots;
1504 uint32_t n_mirroring_sessions;
1505 uint32_t n_extern_objs;
1506 uint32_t n_extern_funcs;
1507 uint32_t n_hash_funcs;
1510 uint32_t n_selectors;
1511 uint32_t n_learners;
1512 uint32_t n_regarrays;
1513 uint32_t n_metarrays;
1517 uint32_t n_instructions;
1526 pipeline_port_inc(struct rte_swx_pipeline *p)
1528 p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
1532 thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
1534 t->ip = p->instructions;
1538 thread_ip_set(struct thread *t, struct instruction *ip)
1544 thread_ip_action_call(struct rte_swx_pipeline *p,
1549 t->ip = p->action_instructions[action_id];
1553 thread_ip_inc(struct rte_swx_pipeline *p);
1556 thread_ip_inc(struct rte_swx_pipeline *p)
1558 struct thread *t = &p->threads[p->thread_id];
1564 thread_ip_inc_cond(struct thread *t, int cond)
1570 thread_yield(struct rte_swx_pipeline *p)
1572 p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1576 thread_yield_cond(struct rte_swx_pipeline *p, int cond)
1578 p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
1585 __instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1587 struct port_in_runtime *port = &p->in[p->port_id];
1588 struct rte_swx_pkt *pkt = &t->pkt;
1591 /* Recirculation: keep the current packet. */
1592 if (t->recirculate) {
1593 TRACE("[Thread %2u] rx - recirculate (pass %u)\n",
1595 t->recirc_pass_id + 1);
1598 t->ptr = &pkt->pkt[pkt->offset];
1599 t->mirroring_slots_mask = 0;
1601 t->recirc_pass_id++;
1604 t->valid_headers = 0;
1605 t->n_headers_out = 0;
1608 t->table_state = p->table_state;
1614 pkt_received = port->pkt_rx(port->obj, pkt);
1615 t->ptr = &pkt->pkt[pkt->offset];
1616 rte_prefetch0(t->ptr);
1618 TRACE("[Thread %2u] rx %s from port %u\n",
1620 pkt_received ? "1 pkt" : "0 pkts",
1623 t->mirroring_slots_mask = 0;
1624 t->recirc_pass_id = 0;
1627 t->valid_headers = 0;
1628 t->n_headers_out = 0;
1631 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
1634 t->table_state = p->table_state;
1637 pipeline_port_inc(p);
1639 return pkt_received;
1643 instr_rx_exec(struct rte_swx_pipeline *p)
1645 struct thread *t = &p->threads[p->thread_id];
1646 struct instruction *ip = t->ip;
1650 pkt_received = __instr_rx_exec(p, t, ip);
1653 thread_ip_inc_cond(t, pkt_received);
1661 emit_handler(struct thread *t)
1663 struct header_out_runtime *h0 = &t->headers_out[0];
1664 struct header_out_runtime *h1 = &t->headers_out[1];
1665 uint32_t offset = 0, i;
1667 /* No header change or header decapsulation. */
1668 if ((t->n_headers_out == 1) &&
1669 (h0->ptr + h0->n_bytes == t->ptr)) {
1670 TRACE("Emit handler: no header change or header decap.\n");
1672 t->pkt.offset -= h0->n_bytes;
1673 t->pkt.length += h0->n_bytes;
1678 /* Header encapsulation (optionally, with prior header decapsulation). */
1679 if ((t->n_headers_out == 2) &&
1680 (h1->ptr + h1->n_bytes == t->ptr) &&
1681 (h0->ptr == h0->ptr0)) {
1684 TRACE("Emit handler: header encapsulation.\n");
1686 offset = h0->n_bytes + h1->n_bytes;
1687 memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
1688 t->pkt.offset -= offset;
1689 t->pkt.length += offset;
1694 /* For any other case. */
1695 TRACE("Emit handler: complex case.\n");
1697 for (i = 0; i < t->n_headers_out; i++) {
1698 struct header_out_runtime *h = &t->headers_out[i];
1700 memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
1701 offset += h->n_bytes;
1705 memcpy(t->ptr - offset, t->header_out_storage, offset);
1706 t->pkt.offset -= offset;
1707 t->pkt.length += offset;
1712 mirroring_handler(struct rte_swx_pipeline *p, struct thread *t, struct rte_swx_pkt *pkt)
1714 uint64_t slots_mask = t->mirroring_slots_mask, slot_mask;
1717 for (slot_id = 0, slot_mask = 1LLU ; slots_mask; slot_id++, slot_mask <<= 1)
1718 if (slot_mask & slots_mask) {
1719 struct port_out_runtime *port;
1720 struct mirroring_session *session;
1721 uint32_t port_id, session_id;
1723 session_id = t->mirroring_slots[slot_id];
1724 session = &p->mirroring_sessions[session_id];
1726 port_id = session->port_id;
1727 port = &p->out[port_id];
1729 if (session->fast_clone)
1730 port->pkt_fast_clone_tx(port->obj, pkt);
1732 port->pkt_clone_tx(port->obj, pkt, session->truncation_length);
1734 slots_mask &= ~slot_mask;
1739 __instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1741 uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
1742 struct port_out_runtime *port = &p->out[port_id];
1743 struct rte_swx_pkt *pkt = &t->pkt;
1745 /* Recirculation: keep the current packet. */
1746 if (t->recirculate) {
1747 TRACE("[Thread %2u]: tx 1 pkt - recirculate\n",
1754 mirroring_handler(p, t, pkt);
1759 TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
1767 mirroring_handler(p, t, pkt);
1768 port->pkt_tx(port->obj, pkt);
1772 __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
1774 uint64_t port_id = ip->io.io.val;
1775 struct port_out_runtime *port = &p->out[port_id];
1776 struct rte_swx_pkt *pkt = &t->pkt;
1778 /* Recirculation: keep the current packet. */
1779 if (t->recirculate) {
1780 TRACE("[Thread %2u]: tx (i) 1 pkt - recirculate\n",
1787 mirroring_handler(p, t, pkt);
1792 TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
1800 mirroring_handler(p, t, pkt);
1801 port->pkt_tx(port->obj, pkt);
1805 __instr_drop_exec(struct rte_swx_pipeline *p,
1807 const struct instruction *ip __rte_unused)
1809 uint64_t port_id = p->n_ports_out - 1;
1810 struct port_out_runtime *port = &p->out[port_id];
1811 struct rte_swx_pkt *pkt = &t->pkt;
1813 TRACE("[Thread %2u]: drop 1 pkt\n",
1820 mirroring_handler(p, t, pkt);
1821 port->pkt_tx(port->obj, pkt);
1825 __instr_mirror_exec(struct rte_swx_pipeline *p,
1827 const struct instruction *ip)
1829 uint64_t slot_id = instr_operand_hbo(t, &ip->mirror.dst);
1830 uint64_t session_id = instr_operand_hbo(t, &ip->mirror.src);
1832 slot_id &= p->n_mirroring_slots - 1;
1833 session_id &= p->n_mirroring_sessions - 1;
1835 TRACE("[Thread %2u]: mirror pkt (slot = %u, session = %u)\n",
1838 (uint32_t)session_id);
1840 t->mirroring_slots[slot_id] = session_id;
1841 t->mirroring_slots_mask |= 1LLU << slot_id;
1845 __instr_recirculate_exec(struct rte_swx_pipeline *p __rte_unused,
1847 const struct instruction *ip __rte_unused)
1849 TRACE("[Thread %2u]: recirculate\n",
1856 __instr_recircid_exec(struct rte_swx_pipeline *p __rte_unused,
1858 const struct instruction *ip)
1860 TRACE("[Thread %2u]: recircid (pass %u)\n",
1865 METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, t->recirc_pass_id);
1872 __instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
1874 const struct instruction *ip,
1877 uint64_t valid_headers = t->valid_headers;
1878 uint8_t *ptr = t->ptr;
1879 uint32_t offset = t->pkt.offset;
1880 uint32_t length = t->pkt.length;
1883 for (i = 0; i < n_extract; i++) {
1884 uint32_t header_id = ip->io.hdr.header_id[i];
1885 uint32_t struct_id = ip->io.hdr.struct_id[i];
1886 uint32_t n_bytes = ip->io.hdr.n_bytes[i];
1888 TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
1894 t->structs[struct_id] = ptr;
1895 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
1904 t->valid_headers = valid_headers;
1907 t->pkt.offset = offset;
1908 t->pkt.length = length;
1913 __instr_hdr_extract_exec(struct rte_swx_pipeline *p,
1915 const struct instruction *ip)
1917 __instr_hdr_extract_many_exec(p, t, ip, 1);
1921 __instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
1923 const struct instruction *ip)
1925 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
1927 __instr_hdr_extract_many_exec(p, t, ip, 2);
1931 __instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
1933 const struct instruction *ip)
1935 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
1937 __instr_hdr_extract_many_exec(p, t, ip, 3);
1941 __instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
1943 const struct instruction *ip)
1945 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
1947 __instr_hdr_extract_many_exec(p, t, ip, 4);
1951 __instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
1953 const struct instruction *ip)
1955 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
1957 __instr_hdr_extract_many_exec(p, t, ip, 5);
1961 __instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
1963 const struct instruction *ip)
1965 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
1967 __instr_hdr_extract_many_exec(p, t, ip, 6);
1971 __instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
1973 const struct instruction *ip)
1975 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
1977 __instr_hdr_extract_many_exec(p, t, ip, 7);
1981 __instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
1983 const struct instruction *ip)
1985 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
1987 __instr_hdr_extract_many_exec(p, t, ip, 8);
1991 __instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
1993 const struct instruction *ip)
1995 uint64_t valid_headers = t->valid_headers;
1996 uint8_t *ptr = t->ptr;
1997 uint32_t offset = t->pkt.offset;
1998 uint32_t length = t->pkt.length;
2000 uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
2001 uint32_t header_id = ip->io.hdr.header_id[0];
2002 uint32_t struct_id = ip->io.hdr.struct_id[0];
2003 uint32_t n_bytes = ip->io.hdr.n_bytes[0];
2005 struct header_runtime *h = &t->headers[header_id];
2007 TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
2013 n_bytes += n_bytes_last;
2016 t->structs[struct_id] = ptr;
2017 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2018 h->n_bytes = n_bytes;
2021 t->pkt.offset = offset + n_bytes;
2022 t->pkt.length = length - n_bytes;
2023 t->ptr = ptr + n_bytes;
2027 __instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
2029 const struct instruction *ip)
2031 uint64_t valid_headers = t->valid_headers;
2032 uint8_t *ptr = t->ptr;
2034 uint32_t header_id = ip->io.hdr.header_id[0];
2035 uint32_t struct_id = ip->io.hdr.struct_id[0];
2037 TRACE("[Thread %2u]: lookahead header %u\n",
2042 t->structs[struct_id] = ptr;
2043 t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2050 __instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
2052 const struct instruction *ip,
2055 uint64_t valid_headers = t->valid_headers;
2056 uint32_t n_headers_out = t->n_headers_out;
2057 struct header_out_runtime *ho = NULL;
2058 uint8_t *ho_ptr = NULL;
2059 uint32_t ho_nbytes = 0, i;
2061 for (i = 0; i < n_emit; i++) {
2062 uint32_t header_id = ip->io.hdr.header_id[i];
2063 uint32_t struct_id = ip->io.hdr.struct_id[i];
2065 struct header_runtime *hi = &t->headers[header_id];
2066 uint8_t *hi_ptr0 = hi->ptr0;
2067 uint32_t n_bytes = hi->n_bytes;
2069 uint8_t *hi_ptr = t->structs[struct_id];
2071 if (!MASK64_BIT_GET(valid_headers, header_id)) {
2072 TRACE("[Thread %2u]: emit header %u (invalid)\n",
2079 TRACE("[Thread %2u]: emit header %u (valid)\n",
2085 if (!n_headers_out) {
2086 ho = &t->headers_out[0];
2092 ho_nbytes = n_bytes;
2098 ho = &t->headers_out[n_headers_out - 1];
2101 ho_nbytes = ho->n_bytes;
2105 if (ho_ptr + ho_nbytes == hi_ptr) {
2106 ho_nbytes += n_bytes;
2108 ho->n_bytes = ho_nbytes;
2115 ho_nbytes = n_bytes;
2122 ho->n_bytes = ho_nbytes;
2123 t->n_headers_out = n_headers_out;
2127 __instr_hdr_emit_exec(struct rte_swx_pipeline *p,
2129 const struct instruction *ip)
2131 __instr_hdr_emit_many_exec(p, t, ip, 1);
2135 __instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
2137 const struct instruction *ip)
2139 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2141 __instr_hdr_emit_many_exec(p, t, ip, 1);
2142 __instr_tx_exec(p, t, ip);
2146 __instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
2148 const struct instruction *ip)
2150 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2152 __instr_hdr_emit_many_exec(p, t, ip, 2);
2153 __instr_tx_exec(p, t, ip);
2157 __instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
2159 const struct instruction *ip)
2161 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2163 __instr_hdr_emit_many_exec(p, t, ip, 3);
2164 __instr_tx_exec(p, t, ip);
2168 __instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
2170 const struct instruction *ip)
2172 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2174 __instr_hdr_emit_many_exec(p, t, ip, 4);
2175 __instr_tx_exec(p, t, ip);
2179 __instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
2181 const struct instruction *ip)
2183 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2185 __instr_hdr_emit_many_exec(p, t, ip, 5);
2186 __instr_tx_exec(p, t, ip);
2190 __instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
2192 const struct instruction *ip)
2194 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2196 __instr_hdr_emit_many_exec(p, t, ip, 6);
2197 __instr_tx_exec(p, t, ip);
2201 __instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
2203 const struct instruction *ip)
2205 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2207 __instr_hdr_emit_many_exec(p, t, ip, 7);
2208 __instr_tx_exec(p, t, ip);
2212 __instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
2214 const struct instruction *ip)
2216 TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
2218 __instr_hdr_emit_many_exec(p, t, ip, 8);
2219 __instr_tx_exec(p, t, ip);
2226 __instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
2228 const struct instruction *ip)
2230 uint32_t header_id = ip->valid.header_id;
2232 TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
2235 t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
2242 __instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
2244 const struct instruction *ip)
2246 uint32_t header_id = ip->valid.header_id;
2248 TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
2251 t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
2258 __instr_learn_exec(struct rte_swx_pipeline *p,
2260 const struct instruction *ip)
2262 uint64_t action_id = ip->learn.action_id;
2263 uint32_t mf_first_arg_offset = ip->learn.mf_first_arg_offset;
2264 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2265 ip->learn.mf_timeout_id_n_bits);
2266 uint32_t learner_id = t->learner_id;
2267 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2268 p->n_selectors + learner_id];
2269 struct learner_runtime *l = &t->learners[learner_id];
2270 struct learner_statistics *stats = &p->learner_stats[learner_id];
2274 status = rte_swx_table_learner_add(ts->obj,
2278 &t->metadata[mf_first_arg_offset],
2281 TRACE("[Thread %2u] learner %u learn %s\n",
2284 status ? "ok" : "error");
2286 stats->n_pkts_learn[status] += 1;
2293 __instr_rearm_exec(struct rte_swx_pipeline *p,
2295 const struct instruction *ip __rte_unused)
2297 uint32_t learner_id = t->learner_id;
2298 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2299 p->n_selectors + learner_id];
2300 struct learner_runtime *l = &t->learners[learner_id];
2301 struct learner_statistics *stats = &p->learner_stats[learner_id];
2304 rte_swx_table_learner_rearm(ts->obj, l->mailbox, t->time);
2306 TRACE("[Thread %2u] learner %u rearm\n",
2310 stats->n_pkts_rearm += 1;
2314 __instr_rearm_new_exec(struct rte_swx_pipeline *p,
2316 const struct instruction *ip)
2318 uint32_t timeout_id = METADATA_READ(t, ip->learn.mf_timeout_id_offset,
2319 ip->learn.mf_timeout_id_n_bits);
2320 uint32_t learner_id = t->learner_id;
2321 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2322 p->n_selectors + learner_id];
2323 struct learner_runtime *l = &t->learners[learner_id];
2324 struct learner_statistics *stats = &p->learner_stats[learner_id];
2327 rte_swx_table_learner_rearm_new(ts->obj, l->mailbox, t->time, timeout_id);
2329 TRACE("[Thread %2u] learner %u rearm with timeout ID %u\n",
2334 stats->n_pkts_rearm += 1;
2341 __instr_forget_exec(struct rte_swx_pipeline *p,
2343 const struct instruction *ip __rte_unused)
2345 uint32_t learner_id = t->learner_id;
2346 struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
2347 p->n_selectors + learner_id];
2348 struct learner_runtime *l = &t->learners[learner_id];
2349 struct learner_statistics *stats = &p->learner_stats[learner_id];
2352 rte_swx_table_learner_delete(ts->obj, l->mailbox);
2354 TRACE("[Thread %2u] learner %u forget\n",
2358 stats->n_pkts_forget += 1;
2364 static inline uint32_t
2365 __instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
2367 const struct instruction *ip)
2369 uint32_t obj_id = ip->ext_obj.ext_obj_id;
2370 uint32_t func_id = ip->ext_obj.func_id;
2371 struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
2372 rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
2375 TRACE("[Thread %2u] extern obj %u member func %u\n",
2380 done = func(obj->obj, obj->mailbox);
2385 static inline uint32_t
2386 __instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
2388 const struct instruction *ip)
2390 uint32_t ext_func_id = ip->ext_func.ext_func_id;
2391 struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
2392 rte_swx_extern_func_t func = ext_func->func;
2395 TRACE("[Thread %2u] extern func %u\n",
2399 done = func(ext_func->mailbox);
2408 __instr_hash_func_exec(struct rte_swx_pipeline *p,
2410 const struct instruction *ip)
2412 uint32_t hash_func_id = ip->hash_func.hash_func_id;
2413 uint32_t dst_offset = ip->hash_func.dst.offset;
2414 uint32_t n_dst_bits = ip->hash_func.dst.n_bits;
2415 uint32_t src_struct_id = ip->hash_func.src.struct_id;
2416 uint32_t src_offset = ip->hash_func.src.offset;
2417 uint32_t n_src_bytes = ip->hash_func.src.n_bytes;
2419 struct hash_func_runtime *func = &p->hash_func_runtime[hash_func_id];
2420 uint8_t *src_ptr = t->structs[src_struct_id];
2423 TRACE("[Thread %2u] hash %u\n",
2427 result = func->func(&src_ptr[src_offset], n_src_bytes, 0);
2428 METADATA_WRITE(t, dst_offset, n_dst_bits, result);
2435 __instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
2437 const struct instruction *ip)
2439 TRACE("[Thread %2u] mov\n", p->thread_id);
2445 __instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2447 const struct instruction *ip)
2449 TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
2455 __instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2457 const struct instruction *ip)
2459 TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
2465 __instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2467 const struct instruction *ip)
2469 TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
2475 __instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
2477 const struct instruction *ip)
2479 TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
2488 __instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
2490 const struct instruction *ip,
2493 uint8_t *action_data = t->structs[0];
2494 uint64_t valid_headers = t->valid_headers;
2497 for (i = 0; i < n_dma; i++) {
2498 uint32_t header_id = ip->dma.dst.header_id[i];
2499 uint32_t struct_id = ip->dma.dst.struct_id[i];
2500 uint32_t offset = ip->dma.src.offset[i];
2501 uint32_t n_bytes = ip->dma.n_bytes[i];
2503 struct header_runtime *h = &t->headers[header_id];
2504 uint8_t *h_ptr0 = h->ptr0;
2505 uint8_t *h_ptr = t->structs[struct_id];
2507 void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
2509 void *src = &action_data[offset];
2511 TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
2514 memcpy(dst, src, n_bytes);
2515 t->structs[struct_id] = dst;
2516 valid_headers = MASK64_BIT_SET(valid_headers, header_id);
2519 t->valid_headers = valid_headers;
2523 __instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2525 __instr_dma_ht_many_exec(p, t, ip, 1);
2529 __instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2531 TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
2533 __instr_dma_ht_many_exec(p, t, ip, 2);
2537 __instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2539 TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
2541 __instr_dma_ht_many_exec(p, t, ip, 3);
2545 __instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2547 TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
2549 __instr_dma_ht_many_exec(p, t, ip, 4);
2553 __instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2555 TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
2557 __instr_dma_ht_many_exec(p, t, ip, 5);
2561 __instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2563 TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
2565 __instr_dma_ht_many_exec(p, t, ip, 6);
2569 __instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2571 TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
2573 __instr_dma_ht_many_exec(p, t, ip, 7);
2577 __instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
2579 TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
2581 __instr_dma_ht_many_exec(p, t, ip, 8);
2588 __instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
2590 const struct instruction *ip)
2592 TRACE("[Thread %2u] add\n", p->thread_id);
2598 __instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2600 const struct instruction *ip)
2602 TRACE("[Thread %2u] add (mh)\n", p->thread_id);
2608 __instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2610 const struct instruction *ip)
2612 TRACE("[Thread %2u] add (hm)\n", p->thread_id);
2618 __instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2620 const struct instruction *ip)
2622 TRACE("[Thread %2u] add (hh)\n", p->thread_id);
2628 __instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2630 const struct instruction *ip)
2632 TRACE("[Thread %2u] add (mi)\n", p->thread_id);
2638 __instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2640 const struct instruction *ip)
2642 TRACE("[Thread %2u] add (hi)\n", p->thread_id);
2648 __instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
2650 const struct instruction *ip)
2652 TRACE("[Thread %2u] sub\n", p->thread_id);
2658 __instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2660 const struct instruction *ip)
2662 TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
2668 __instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2670 const struct instruction *ip)
2672 TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
2678 __instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2680 const struct instruction *ip)
2682 TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
2688 __instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2690 const struct instruction *ip)
2692 TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
2698 __instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2700 const struct instruction *ip)
2702 TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
2708 __instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
2710 const struct instruction *ip)
2712 TRACE("[Thread %2u] shl\n", p->thread_id);
2718 __instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2720 const struct instruction *ip)
2722 TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
2728 __instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2730 const struct instruction *ip)
2732 TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
2738 __instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2740 const struct instruction *ip)
2742 TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
2748 __instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2750 const struct instruction *ip)
2752 TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
2758 __instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2760 const struct instruction *ip)
2762 TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
2768 __instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
2770 const struct instruction *ip)
2772 TRACE("[Thread %2u] shr\n", p->thread_id);
2778 __instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2780 const struct instruction *ip)
2782 TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
2788 __instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2790 const struct instruction *ip)
2792 TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
2798 __instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2800 const struct instruction *ip)
2802 TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
2808 __instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
2810 const struct instruction *ip)
2812 TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
2819 __instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
2821 const struct instruction *ip)
2823 TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
2829 __instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
2831 const struct instruction *ip)
2833 TRACE("[Thread %2u] and\n", p->thread_id);
2839 __instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2841 const struct instruction *ip)
2843 TRACE("[Thread %2u] and (mh)\n", p->thread_id);
2849 __instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2851 const struct instruction *ip)
2853 TRACE("[Thread %2u] and (hm)\n", p->thread_id);
2855 ALU_HM_FAST(t, ip, &);
2859 __instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2861 const struct instruction *ip)
2863 TRACE("[Thread %2u] and (hh)\n", p->thread_id);
2865 ALU_HH_FAST(t, ip, &);
2869 __instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
2871 const struct instruction *ip)
2873 TRACE("[Thread %2u] and (i)\n", p->thread_id);
2879 __instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
2881 const struct instruction *ip)
2883 TRACE("[Thread %2u] or\n", p->thread_id);
2889 __instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2891 const struct instruction *ip)
2893 TRACE("[Thread %2u] or (mh)\n", p->thread_id);
2899 __instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2901 const struct instruction *ip)
2903 TRACE("[Thread %2u] or (hm)\n", p->thread_id);
2905 ALU_HM_FAST(t, ip, |);
2909 __instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2911 const struct instruction *ip)
2913 TRACE("[Thread %2u] or (hh)\n", p->thread_id);
2915 ALU_HH_FAST(t, ip, |);
2919 __instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
2921 const struct instruction *ip)
2923 TRACE("[Thread %2u] or (i)\n", p->thread_id);
2929 __instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
2931 const struct instruction *ip)
2933 TRACE("[Thread %2u] xor\n", p->thread_id);
2939 __instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
2941 const struct instruction *ip)
2943 TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
2949 __instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
2951 const struct instruction *ip)
2953 TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
2955 ALU_HM_FAST(t, ip, ^);
2959 __instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
2961 const struct instruction *ip)
2963 TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
2965 ALU_HH_FAST(t, ip, ^);
2969 __instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
2971 const struct instruction *ip)
2973 TRACE("[Thread %2u] xor (i)\n", p->thread_id);
2979 __instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
2981 const struct instruction *ip)
2983 uint8_t *dst_struct, *src_struct;
2984 uint16_t *dst16_ptr, dst;
2985 uint64_t *src64_ptr, src64, src64_mask, src;
2988 TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
2991 dst_struct = t->structs[ip->alu.dst.struct_id];
2992 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
2995 src_struct = t->structs[ip->alu.src.struct_id];
2996 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
2998 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
2999 src = src64 & src64_mask;
3001 /* Initialize the result with destination 1's complement. */
3005 /* The first input (r) is a 16-bit number. The second and the third
3006 * inputs are 32-bit numbers. In the worst case scenario, the sum of the
3007 * three numbers (output r) is a 34-bit number.
3009 r += (src >> 32) + (src & 0xFFFFFFFF);
3011 /* The first input is a 16-bit number. The second input is an 18-bit
3012 * number. In the worst case scenario, the sum of the two numbers is a
3015 r = (r & 0xFFFF) + (r >> 16);
3017 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3018 * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
3020 r = (r & 0xFFFF) + (r >> 16);
3022 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3023 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3024 * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
3025 * therefore the output r is always a 16-bit number.
3027 r = (r & 0xFFFF) + (r >> 16);
3029 /* Apply 1's complement to the result. */
3033 *dst16_ptr = (uint16_t)r;
3037 __instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
3039 const struct instruction *ip)
3041 uint8_t *dst_struct, *src_struct;
3042 uint16_t *dst16_ptr, dst;
3043 uint64_t *src64_ptr, src64, src64_mask, src;
3046 TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
3049 dst_struct = t->structs[ip->alu.dst.struct_id];
3050 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3053 src_struct = t->structs[ip->alu.src.struct_id];
3054 src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
3056 src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
3057 src = src64 & src64_mask;
3059 /* Initialize the result with destination 1's complement. */
3063 /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
3064 * the following sequence of operations in 2's complement arithmetic:
3065 * a '- b = (a - b) % 0xFFFF.
3067 * In order to prevent an underflow for the below subtraction, in which
3068 * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
3069 * minuend), we first add a multiple of the 0xFFFF modulus to the
3070 * minuend. The number we add to the minuend needs to be a 34-bit number
3071 * or higher, so for readability reasons we picked the 36-bit multiple.
3072 * We are effectively turning the 16-bit minuend into a 36-bit number:
3073 * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
3075 r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
3077 /* A 33-bit number is subtracted from a 36-bit number (the input r). The
3078 * result (the output r) is a 36-bit number.
3080 r -= (src >> 32) + (src & 0xFFFFFFFF);
3082 /* The first input is a 16-bit number. The second input is a 20-bit
3083 * number. Their sum is a 21-bit number.
3085 r = (r & 0xFFFF) + (r >> 16);
3087 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3088 * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
3090 r = (r & 0xFFFF) + (r >> 16);
3092 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3093 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3094 * 0x1001E), the output r is (0 .. 31). So no carry bit can be
3095 * generated, therefore the output r is always a 16-bit number.
3097 r = (r & 0xFFFF) + (r >> 16);
3099 /* Apply 1's complement to the result. */
3103 *dst16_ptr = (uint16_t)r;
3107 __instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
3109 const struct instruction *ip)
3111 uint8_t *dst_struct, *src_struct;
3112 uint16_t *dst16_ptr, dst;
3113 uint32_t *src32_ptr;
3116 TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
3119 dst_struct = t->structs[ip->alu.dst.struct_id];
3120 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3123 src_struct = t->structs[ip->alu.src.struct_id];
3124 src32_ptr = (uint32_t *)&src_struct[0];
3126 /* Initialize the result with destination 1's complement. */
3130 r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
3131 r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
3132 r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
3133 r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
3134 r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
3136 /* The first input is a 16-bit number. The second input is a 19-bit
3137 * number. Their sum is a 20-bit number.
3139 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3141 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3142 * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
3144 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3146 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3147 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3148 * 0x1000E), the output r is (0 .. 15). So no carry bit can be
3149 * generated, therefore the output r is always a 16-bit number.
3151 r0 = (r0 & 0xFFFF) + (r0 >> 16);
3153 /* Apply 1's complement to the result. */
3155 r0 = r0 ? r0 : 0xFFFF;
3157 *dst16_ptr = (uint16_t)r0;
3161 __instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
3163 const struct instruction *ip)
3165 uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
3166 uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
3167 uint8_t *dst_struct, *src_struct;
3168 uint16_t *dst16_ptr, dst;
3169 uint32_t *src32_ptr;
3173 if (n_src_header_bytes == 20) {
3174 __instr_alu_ckadd_struct20_exec(p, t, ip);
3178 TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
3181 dst_struct = t->structs[ip->alu.dst.struct_id];
3182 dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
3185 src_struct = t->structs[ip->alu.src.struct_id];
3186 src32_ptr = (uint32_t *)&src_struct[0];
3188 /* Initialize the result with destination 1's complement. */
3192 /* The max number of 32-bit words in a 32K-byte header is 2^13.
3193 * Therefore, in the worst case scenario, a 45-bit number is added to a
3194 * 16-bit number (the input r), so the output r is 46-bit number.
3196 for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
3199 /* The first input is a 16-bit number. The second input is a 30-bit
3200 * number. Their sum is a 31-bit number.
3202 r = (r & 0xFFFF) + (r >> 16);
3204 /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
3205 * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
3207 r = (r & 0xFFFF) + (r >> 16);
3209 /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
3210 * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
3211 * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
3212 * generated, therefore the output r is always a 16-bit number.
3214 r = (r & 0xFFFF) + (r >> 16);
3216 /* Apply 1's complement to the result. */
3220 *dst16_ptr = (uint16_t)r;
3226 static inline uint64_t *
3227 instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
3229 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3233 static inline uint64_t
3234 instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3236 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3238 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3239 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3240 uint64_t idx64 = *idx64_ptr;
3241 uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
3242 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3247 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3249 static inline uint64_t
3250 instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3252 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3254 uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
3255 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
3256 uint64_t idx64 = *idx64_ptr;
3257 uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
3264 #define instr_regarray_idx_nbo instr_regarray_idx_hbo
3268 static inline uint64_t
3269 instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3271 struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
3273 uint64_t idx = ip->regarray.idx_val & r->size_mask;
3278 static inline uint64_t
3279 instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
3281 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3282 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3283 uint64_t src64 = *src64_ptr;
3284 uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3285 uint64_t src = src64 & src64_mask;
3290 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3292 static inline uint64_t
3293 instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
3295 uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
3296 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
3297 uint64_t src64 = *src64_ptr;
3298 uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
3305 #define instr_regarray_src_nbo instr_regarray_src_hbo
3310 instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3312 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3313 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3314 uint64_t dst64 = *dst64_ptr;
3315 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3317 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3321 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3324 instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
3326 uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
3327 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
3328 uint64_t dst64 = *dst64_ptr;
3329 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
3331 src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
3332 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3337 #define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
3342 __instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
3344 const struct instruction *ip)
3346 uint64_t *regarray, idx;
3348 TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
3350 regarray = instr_regarray_regarray(p, ip);
3351 idx = instr_regarray_idx_nbo(p, t, ip);
3352 rte_prefetch0(®array[idx]);
3356 __instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
3358 const struct instruction *ip)
3360 uint64_t *regarray, idx;
3362 TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
3364 regarray = instr_regarray_regarray(p, ip);
3365 idx = instr_regarray_idx_hbo(p, t, ip);
3366 rte_prefetch0(®array[idx]);
3370 __instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
3371 struct thread *t __rte_unused,
3372 const struct instruction *ip)
3374 uint64_t *regarray, idx;
3376 TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
3378 regarray = instr_regarray_regarray(p, ip);
3379 idx = instr_regarray_idx_imm(p, ip);
3380 rte_prefetch0(®array[idx]);
3384 __instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
3386 const struct instruction *ip)
3388 uint64_t *regarray, idx;
3390 TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
3392 regarray = instr_regarray_regarray(p, ip);
3393 idx = instr_regarray_idx_nbo(p, t, ip);
3394 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3398 __instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
3400 const struct instruction *ip)
3402 uint64_t *regarray, idx;
3404 TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
3407 regarray = instr_regarray_regarray(p, ip);
3408 idx = instr_regarray_idx_hbo(p, t, ip);
3409 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3413 __instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3415 uint64_t *regarray, idx;
3417 TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
3419 regarray = instr_regarray_regarray(p, ip);
3420 idx = instr_regarray_idx_nbo(p, t, ip);
3421 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3425 __instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3427 uint64_t *regarray, idx;
3429 TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
3431 regarray = instr_regarray_regarray(p, ip);
3432 idx = instr_regarray_idx_hbo(p, t, ip);
3433 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3437 __instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3439 uint64_t *regarray, idx;
3441 TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
3443 regarray = instr_regarray_regarray(p, ip);
3444 idx = instr_regarray_idx_imm(p, ip);
3445 instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
3449 __instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3451 uint64_t *regarray, idx;
3453 TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
3455 regarray = instr_regarray_regarray(p, ip);
3456 idx = instr_regarray_idx_imm(p, ip);
3457 instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
3461 __instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3463 uint64_t *regarray, idx, src;
3465 TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
3467 regarray = instr_regarray_regarray(p, ip);
3468 idx = instr_regarray_idx_nbo(p, t, ip);
3469 src = instr_regarray_src_nbo(t, ip);
3470 regarray[idx] = src;
3474 __instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3476 uint64_t *regarray, idx, src;
3478 TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
3480 regarray = instr_regarray_regarray(p, ip);
3481 idx = instr_regarray_idx_nbo(p, t, ip);
3482 src = instr_regarray_src_hbo(t, ip);
3483 regarray[idx] = src;
3487 __instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3489 uint64_t *regarray, idx, src;
3491 TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
3493 regarray = instr_regarray_regarray(p, ip);
3494 idx = instr_regarray_idx_hbo(p, t, ip);
3495 src = instr_regarray_src_nbo(t, ip);
3496 regarray[idx] = src;
3500 __instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3502 uint64_t *regarray, idx, src;
3504 TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
3506 regarray = instr_regarray_regarray(p, ip);
3507 idx = instr_regarray_idx_hbo(p, t, ip);
3508 src = instr_regarray_src_hbo(t, ip);
3509 regarray[idx] = src;
3513 __instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3515 uint64_t *regarray, idx, src;
3517 TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
3519 regarray = instr_regarray_regarray(p, ip);
3520 idx = instr_regarray_idx_nbo(p, t, ip);
3521 src = ip->regarray.dstsrc_val;
3522 regarray[idx] = src;
3526 __instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3528 uint64_t *regarray, idx, src;
3530 TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
3532 regarray = instr_regarray_regarray(p, ip);
3533 idx = instr_regarray_idx_hbo(p, t, ip);
3534 src = ip->regarray.dstsrc_val;
3535 regarray[idx] = src;
3539 __instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3541 uint64_t *regarray, idx, src;
3543 TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
3545 regarray = instr_regarray_regarray(p, ip);
3546 idx = instr_regarray_idx_imm(p, ip);
3547 src = instr_regarray_src_nbo(t, ip);
3548 regarray[idx] = src;
3552 __instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3554 uint64_t *regarray, idx, src;
3556 TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
3558 regarray = instr_regarray_regarray(p, ip);
3559 idx = instr_regarray_idx_imm(p, ip);
3560 src = instr_regarray_src_hbo(t, ip);
3561 regarray[idx] = src;
3565 __instr_regwr_rii_exec(struct rte_swx_pipeline *p,
3566 struct thread *t __rte_unused,
3567 const struct instruction *ip)
3569 uint64_t *regarray, idx, src;
3571 TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
3573 regarray = instr_regarray_regarray(p, ip);
3574 idx = instr_regarray_idx_imm(p, ip);
3575 src = ip->regarray.dstsrc_val;
3576 regarray[idx] = src;
3580 __instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3582 uint64_t *regarray, idx, src;
3584 TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
3586 regarray = instr_regarray_regarray(p, ip);
3587 idx = instr_regarray_idx_nbo(p, t, ip);
3588 src = instr_regarray_src_nbo(t, ip);
3589 regarray[idx] += src;
3593 __instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3595 uint64_t *regarray, idx, src;
3597 TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
3599 regarray = instr_regarray_regarray(p, ip);
3600 idx = instr_regarray_idx_nbo(p, t, ip);
3601 src = instr_regarray_src_hbo(t, ip);
3602 regarray[idx] += src;
3606 __instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3608 uint64_t *regarray, idx, src;
3610 TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
3612 regarray = instr_regarray_regarray(p, ip);
3613 idx = instr_regarray_idx_hbo(p, t, ip);
3614 src = instr_regarray_src_nbo(t, ip);
3615 regarray[idx] += src;
3619 __instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3621 uint64_t *regarray, idx, src;
3623 TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
3625 regarray = instr_regarray_regarray(p, ip);
3626 idx = instr_regarray_idx_hbo(p, t, ip);
3627 src = instr_regarray_src_hbo(t, ip);
3628 regarray[idx] += src;
3632 __instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3634 uint64_t *regarray, idx, src;
3636 TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
3638 regarray = instr_regarray_regarray(p, ip);
3639 idx = instr_regarray_idx_nbo(p, t, ip);
3640 src = ip->regarray.dstsrc_val;
3641 regarray[idx] += src;
3645 __instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3647 uint64_t *regarray, idx, src;
3649 TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
3651 regarray = instr_regarray_regarray(p, ip);
3652 idx = instr_regarray_idx_hbo(p, t, ip);
3653 src = ip->regarray.dstsrc_val;
3654 regarray[idx] += src;
3658 __instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3660 uint64_t *regarray, idx, src;
3662 TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
3664 regarray = instr_regarray_regarray(p, ip);
3665 idx = instr_regarray_idx_imm(p, ip);
3666 src = instr_regarray_src_nbo(t, ip);
3667 regarray[idx] += src;
3671 __instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3673 uint64_t *regarray, idx, src;
3675 TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
3677 regarray = instr_regarray_regarray(p, ip);
3678 idx = instr_regarray_idx_imm(p, ip);
3679 src = instr_regarray_src_hbo(t, ip);
3680 regarray[idx] += src;
3684 __instr_regadd_rii_exec(struct rte_swx_pipeline *p,
3685 struct thread *t __rte_unused,
3686 const struct instruction *ip)
3688 uint64_t *regarray, idx, src;
3690 TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
3692 regarray = instr_regarray_regarray(p, ip);
3693 idx = instr_regarray_idx_imm(p, ip);
3694 src = ip->regarray.dstsrc_val;
3695 regarray[idx] += src;
3701 static inline struct meter *
3702 instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3704 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3706 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3707 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3708 uint64_t idx64 = *idx64_ptr;
3709 uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
3710 uint64_t idx = idx64 & idx64_mask & r->size_mask;
3712 return &r->metarray[idx];
3715 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3717 static inline struct meter *
3718 instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3720 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3722 uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
3723 uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
3724 uint64_t idx64 = *idx64_ptr;
3725 uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
3727 return &r->metarray[idx];
3732 #define instr_meter_idx_nbo instr_meter_idx_hbo
3736 static inline struct meter *
3737 instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
3739 struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
3741 uint64_t idx = ip->meter.idx_val & r->size_mask;
3743 return &r->metarray[idx];
3746 static inline uint32_t
3747 instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
3749 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3750 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3751 uint64_t src64 = *src64_ptr;
3752 uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
3753 uint64_t src = src64 & src64_mask;
3755 return (uint32_t)src;
3758 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
3760 static inline uint32_t
3761 instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
3763 uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
3764 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
3765 uint64_t src64 = *src64_ptr;
3766 uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
3768 return (uint32_t)src;
3773 #define instr_meter_length_nbo instr_meter_length_hbo
3777 static inline enum rte_color
3778 instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
3780 uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
3781 uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
3782 uint64_t src64 = *src64_ptr;
3783 uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
3784 uint64_t src = src64 & src64_mask;
3786 return (enum rte_color)src;
3790 instr_meter_color_out_hbo_set(struct thread *t,
3791 const struct instruction *ip,
3792 enum rte_color color_out)
3794 uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
3795 uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
3796 uint64_t dst64 = *dst64_ptr;
3797 uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
3799 uint64_t src = (uint64_t)color_out;
3801 *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
3805 __instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
3807 const struct instruction *ip)
3811 TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
3813 m = instr_meter_idx_nbo(p, t, ip);
3818 __instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
3820 const struct instruction *ip)
3824 TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
3826 m = instr_meter_idx_hbo(p, t, ip);
3831 __instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
3832 struct thread *t __rte_unused,
3833 const struct instruction *ip)
3837 TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
3839 m = instr_meter_idx_imm(p, ip);
3844 __instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3847 uint64_t time, n_pkts, n_bytes;
3849 enum rte_color color_in, color_out;
3851 TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
3853 m = instr_meter_idx_nbo(p, t, ip);
3854 rte_prefetch0(m->n_pkts);
3855 time = rte_get_tsc_cycles();
3856 length = instr_meter_length_nbo(t, ip);
3857 color_in = instr_meter_color_in_hbo(t, ip);
3859 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3860 &m->profile->profile,
3865 color_out &= m->color_mask;
3867 n_pkts = m->n_pkts[color_out];
3868 n_bytes = m->n_bytes[color_out];
3870 instr_meter_color_out_hbo_set(t, ip, color_out);
3872 m->n_pkts[color_out] = n_pkts + 1;
3873 m->n_bytes[color_out] = n_bytes + length;
3877 __instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3880 uint64_t time, n_pkts, n_bytes;
3882 enum rte_color color_in, color_out;
3884 TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
3886 m = instr_meter_idx_nbo(p, t, ip);
3887 rte_prefetch0(m->n_pkts);
3888 time = rte_get_tsc_cycles();
3889 length = instr_meter_length_nbo(t, ip);
3890 color_in = (enum rte_color)ip->meter.color_in_val;
3892 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3893 &m->profile->profile,
3898 color_out &= m->color_mask;
3900 n_pkts = m->n_pkts[color_out];
3901 n_bytes = m->n_bytes[color_out];
3903 instr_meter_color_out_hbo_set(t, ip, color_out);
3905 m->n_pkts[color_out] = n_pkts + 1;
3906 m->n_bytes[color_out] = n_bytes + length;
3910 __instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3913 uint64_t time, n_pkts, n_bytes;
3915 enum rte_color color_in, color_out;
3917 TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
3919 m = instr_meter_idx_nbo(p, t, ip);
3920 rte_prefetch0(m->n_pkts);
3921 time = rte_get_tsc_cycles();
3922 length = instr_meter_length_hbo(t, ip);
3923 color_in = instr_meter_color_in_hbo(t, ip);
3925 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3926 &m->profile->profile,
3931 color_out &= m->color_mask;
3933 n_pkts = m->n_pkts[color_out];
3934 n_bytes = m->n_bytes[color_out];
3936 instr_meter_color_out_hbo_set(t, ip, color_out);
3938 m->n_pkts[color_out] = n_pkts + 1;
3939 m->n_bytes[color_out] = n_bytes + length;
3943 __instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3946 uint64_t time, n_pkts, n_bytes;
3948 enum rte_color color_in, color_out;
3950 TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
3952 m = instr_meter_idx_nbo(p, t, ip);
3953 rte_prefetch0(m->n_pkts);
3954 time = rte_get_tsc_cycles();
3955 length = instr_meter_length_hbo(t, ip);
3956 color_in = (enum rte_color)ip->meter.color_in_val;
3958 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3959 &m->profile->profile,
3964 color_out &= m->color_mask;
3966 n_pkts = m->n_pkts[color_out];
3967 n_bytes = m->n_bytes[color_out];
3969 instr_meter_color_out_hbo_set(t, ip, color_out);
3971 m->n_pkts[color_out] = n_pkts + 1;
3972 m->n_bytes[color_out] = n_bytes + length;
3976 __instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
3979 uint64_t time, n_pkts, n_bytes;
3981 enum rte_color color_in, color_out;
3983 TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
3985 m = instr_meter_idx_hbo(p, t, ip);
3986 rte_prefetch0(m->n_pkts);
3987 time = rte_get_tsc_cycles();
3988 length = instr_meter_length_nbo(t, ip);
3989 color_in = instr_meter_color_in_hbo(t, ip);
3991 color_out = rte_meter_trtcm_color_aware_check(&m->m,
3992 &m->profile->profile,
3997 color_out &= m->color_mask;
3999 n_pkts = m->n_pkts[color_out];
4000 n_bytes = m->n_bytes[color_out];
4002 instr_meter_color_out_hbo_set(t, ip, color_out);
4004 m->n_pkts[color_out] = n_pkts + 1;
4005 m->n_bytes[color_out] = n_bytes + length;
4009 __instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4012 uint64_t time, n_pkts, n_bytes;
4014 enum rte_color color_in, color_out;
4016 TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
4018 m = instr_meter_idx_hbo(p, t, ip);
4019 rte_prefetch0(m->n_pkts);
4020 time = rte_get_tsc_cycles();
4021 length = instr_meter_length_nbo(t, ip);
4022 color_in = (enum rte_color)ip->meter.color_in_val;
4024 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4025 &m->profile->profile,
4030 color_out &= m->color_mask;
4032 n_pkts = m->n_pkts[color_out];
4033 n_bytes = m->n_bytes[color_out];
4035 instr_meter_color_out_hbo_set(t, ip, color_out);
4037 m->n_pkts[color_out] = n_pkts + 1;
4038 m->n_bytes[color_out] = n_bytes + length;
4042 __instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4045 uint64_t time, n_pkts, n_bytes;
4047 enum rte_color color_in, color_out;
4049 TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
4051 m = instr_meter_idx_hbo(p, t, ip);
4052 rte_prefetch0(m->n_pkts);
4053 time = rte_get_tsc_cycles();
4054 length = instr_meter_length_hbo(t, ip);
4055 color_in = instr_meter_color_in_hbo(t, ip);
4057 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4058 &m->profile->profile,
4063 color_out &= m->color_mask;
4065 n_pkts = m->n_pkts[color_out];
4066 n_bytes = m->n_bytes[color_out];
4068 instr_meter_color_out_hbo_set(t, ip, color_out);
4070 m->n_pkts[color_out] = n_pkts + 1;
4071 m->n_bytes[color_out] = n_bytes + length;
4075 __instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4078 uint64_t time, n_pkts, n_bytes;
4080 enum rte_color color_in, color_out;
4082 TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
4084 m = instr_meter_idx_hbo(p, t, ip);
4085 rte_prefetch0(m->n_pkts);
4086 time = rte_get_tsc_cycles();
4087 length = instr_meter_length_hbo(t, ip);
4088 color_in = (enum rte_color)ip->meter.color_in_val;
4090 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4091 &m->profile->profile,
4096 color_out &= m->color_mask;
4098 n_pkts = m->n_pkts[color_out];
4099 n_bytes = m->n_bytes[color_out];
4101 instr_meter_color_out_hbo_set(t, ip, color_out);
4103 m->n_pkts[color_out] = n_pkts + 1;
4104 m->n_bytes[color_out] = n_bytes + length;
4108 __instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4111 uint64_t time, n_pkts, n_bytes;
4113 enum rte_color color_in, color_out;
4115 TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
4117 m = instr_meter_idx_imm(p, ip);
4118 rte_prefetch0(m->n_pkts);
4119 time = rte_get_tsc_cycles();
4120 length = instr_meter_length_nbo(t, ip);
4121 color_in = instr_meter_color_in_hbo(t, ip);
4123 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4124 &m->profile->profile,
4129 color_out &= m->color_mask;
4131 n_pkts = m->n_pkts[color_out];
4132 n_bytes = m->n_bytes[color_out];
4134 instr_meter_color_out_hbo_set(t, ip, color_out);
4136 m->n_pkts[color_out] = n_pkts + 1;
4137 m->n_bytes[color_out] = n_bytes + length;
4141 __instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4144 uint64_t time, n_pkts, n_bytes;
4146 enum rte_color color_in, color_out;
4148 TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
4150 m = instr_meter_idx_imm(p, ip);
4151 rte_prefetch0(m->n_pkts);
4152 time = rte_get_tsc_cycles();
4153 length = instr_meter_length_nbo(t, ip);
4154 color_in = (enum rte_color)ip->meter.color_in_val;
4156 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4157 &m->profile->profile,
4162 color_out &= m->color_mask;
4164 n_pkts = m->n_pkts[color_out];
4165 n_bytes = m->n_bytes[color_out];
4167 instr_meter_color_out_hbo_set(t, ip, color_out);
4169 m->n_pkts[color_out] = n_pkts + 1;
4170 m->n_bytes[color_out] = n_bytes + length;
4174 __instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4177 uint64_t time, n_pkts, n_bytes;
4179 enum rte_color color_in, color_out;
4181 TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
4183 m = instr_meter_idx_imm(p, ip);
4184 rte_prefetch0(m->n_pkts);
4185 time = rte_get_tsc_cycles();
4186 length = instr_meter_length_hbo(t, ip);
4187 color_in = instr_meter_color_in_hbo(t, ip);
4189 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4190 &m->profile->profile,
4195 color_out &= m->color_mask;
4197 n_pkts = m->n_pkts[color_out];
4198 n_bytes = m->n_bytes[color_out];
4200 instr_meter_color_out_hbo_set(t, ip, color_out);
4202 m->n_pkts[color_out] = n_pkts + 1;
4203 m->n_bytes[color_out] = n_bytes + length;
4207 __instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
4210 uint64_t time, n_pkts, n_bytes;
4212 enum rte_color color_in, color_out;
4214 TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
4216 m = instr_meter_idx_imm(p, ip);
4217 rte_prefetch0(m->n_pkts);
4218 time = rte_get_tsc_cycles();
4219 length = instr_meter_length_hbo(t, ip);
4220 color_in = (enum rte_color)ip->meter.color_in_val;
4222 color_out = rte_meter_trtcm_color_aware_check(&m->m,
4223 &m->profile->profile,
4228 color_out &= m->color_mask;
4230 n_pkts = m->n_pkts[color_out];
4231 n_bytes = m->n_bytes[color_out];
4233 instr_meter_color_out_hbo_set(t, ip, color_out);
4235 m->n_pkts[color_out] = n_pkts + 1;
4236 m->n_bytes[color_out] = n_bytes + length;