X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Fpipeline%2Frte_swx_pipeline_internal.h;h=fa944c95f2458afce6bf5a46e5ce9e52e2d9a292;hb=f3ca33bb20925cca90184bd3f05ebedc78a929b2;hp=24096a23b6d2a79d567abf58d8fcb529c153b1f9;hpb=d1a58ada1a43691ffba6813ba23b15a1cd3b2012;p=dpdk.git diff --git a/lib/pipeline/rte_swx_pipeline_internal.h b/lib/pipeline/rte_swx_pipeline_internal.h index 24096a23b6..fa944c95f2 100644 --- a/lib/pipeline/rte_swx_pipeline_internal.h +++ b/lib/pipeline/rte_swx_pipeline_internal.h @@ -309,7 +309,7 @@ enum instruction_type { */ INSTR_ALU_CKADD_FIELD, /* src = H */ INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */ - INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */ + INSTR_ALU_CKADD_STRUCT, /* src = h.header, with any sizeof(header) */ /* cksub dst src * dst = dst '- src @@ -443,10 +443,12 @@ enum instruction_type { /* table TABLE */ INSTR_TABLE, + INSTR_TABLE_AF, INSTR_SELECTOR, INSTR_LEARNER, + INSTR_LEARNER_AF, - /* learn LEARNER ACTION_NAME */ + /* learn LEARNER ACTION_NAME [ m.action_first_arg ] */ INSTR_LEARNER_LEARN, /* forget */ @@ -539,6 +541,9 @@ enum instruction_type { * Return from action */ INSTR_RETURN, + + /* Start of custom instructions. */ + INSTR_CUSTOM_0, }; struct instr_operand { @@ -578,6 +583,7 @@ struct instr_table { struct instr_learn { uint8_t action_id; + uint8_t mf_offset; }; struct instr_extern_obj { @@ -684,15 +690,21 @@ struct instruction_data { int invalid; }; +typedef void (*instr_exec_t)(struct rte_swx_pipeline *); + /* * Action. */ +typedef void +(*action_func_t)(struct rte_swx_pipeline *p); + struct action { TAILQ_ENTRY(action) node; char name[RTE_SWX_NAME_SIZE]; struct struct_type *st; int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */ struct instruction *instructions; + struct instruction_data *instruction_data; uint32_t n_instructions; uint32_t id; }; @@ -734,6 +746,8 @@ struct table { uint32_t n_actions; int default_action_is_const; uint32_t action_data_size_max; + int *action_is_for_table_entries; + int *action_is_for_default_entry; uint32_t size; uint32_t id; @@ -798,12 +812,13 @@ struct learner { /* Action. */ struct action **actions; - struct field **action_arg; struct action *default_action; uint8_t *default_action_data; uint32_t n_actions; int default_action_is_const; uint32_t action_data_size_max; + int *action_is_for_table_entries; + int *action_is_for_default_entry; uint32_t size; uint32_t timeout; @@ -815,7 +830,6 @@ TAILQ_HEAD(learner_tailq, learner); struct learner_runtime { void *mailbox; uint8_t **key; - uint8_t **action_data; }; struct learner_statistics { @@ -928,6 +942,35 @@ struct thread { #define HEADER_VALID(thread, header_id) \ MASK64_BIT_GET((thread)->valid_headers, header_id) +static inline uint64_t +instr_operand_hbo(struct thread *t, const struct instr_operand *x) +{ + uint8_t *x_struct = t->structs[x->struct_id]; + uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; + uint64_t x64 = *x64_ptr; + uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits); + + return x64 & x64_mask; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_operand_nbo(struct thread *t, const struct instr_operand *x) +{ + uint8_t *x_struct = t->structs[x->struct_id]; + uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; + uint64_t x64 = *x64_ptr; + + return ntoh64(x64) >> (64 - x->n_bits); +} + +#else + +#define instr_operand_nbo instr_operand_hbo + +#endif + #define ALU(thread, ip, operator) \ { \ uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ @@ -1328,6 +1371,10 @@ struct thread { #define RTE_SWX_PIPELINE_THREADS_MAX 16 #endif +#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX +#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256 +#endif + struct rte_swx_pipeline { struct struct_type_tailq struct_types; struct port_in_type_tailq port_in_types; @@ -1352,6 +1399,7 @@ struct rte_swx_pipeline { struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; + action_func_t *action_funcs; struct rte_swx_table_state *table_state; struct table_statistics *table_stats; struct selector_statistics *selector_stats; @@ -1359,7 +1407,10 @@ struct rte_swx_pipeline { struct regarray_runtime *regarray_runtime; struct metarray_runtime *metarray_runtime; struct instruction *instructions; + struct instruction_data *instruction_data; + instr_exec_t *instruction_table; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; + void *lib; uint32_t n_structs; uint32_t n_ports_in; @@ -1511,7 +1562,7 @@ emit_handler(struct thread *t) return; } - /* Header encapsulation (optionally, with prior header decasulation). */ + /* Header encapsulation (optionally, with prior header decapsulation). */ if ((t->n_headers_out == 2) && (h1->ptr + h1->n_bytes == t->ptr) && (h0->ptr == h0->ptr0)) { @@ -1969,6 +2020,7 @@ __instr_learn_exec(struct rte_swx_pipeline *p, const struct instruction *ip) { uint64_t action_id = ip->learn.action_id; + uint32_t mf_offset = ip->learn.mf_offset; uint32_t learner_id = t->learner_id; struct rte_swx_table_state *ts = &t->table_state[p->n_tables + p->n_selectors + learner_id]; @@ -1981,7 +2033,7 @@ __instr_learn_exec(struct rte_swx_pipeline *p, l->mailbox, t->time, action_id, - l->action_data[action_id]); + &t->metadata[mf_offset]); TRACE("[Thread %2u] learner %u learn %s\n", p->thread_id, @@ -2015,4 +2067,1832 @@ __instr_forget_exec(struct rte_swx_pipeline *p, stats->n_pkts_forget += 1; } +/* + * extern. + */ +static inline uint32_t +__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t obj_id = ip->ext_obj.ext_obj_id; + uint32_t func_id = ip->ext_obj.func_id; + struct extern_obj_runtime *obj = &t->extern_objs[obj_id]; + rte_swx_extern_type_member_func_t func = obj->funcs[func_id]; + uint32_t done; + + TRACE("[Thread %2u] extern obj %u member func %u\n", + p->thread_id, + obj_id, + func_id); + + done = func(obj->obj, obj->mailbox); + + return done; +} + +static inline uint32_t +__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t ext_func_id = ip->ext_func.ext_func_id; + struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id]; + rte_swx_extern_func_t func = ext_func->func; + uint32_t done; + + TRACE("[Thread %2u] extern func %u\n", + p->thread_id, + ext_func_id); + + done = func(ext_func->mailbox); + + return done; +} + +/* + * mov. + */ +static inline void +__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov\n", p->thread_id); + + MOV(t, ip); +} + +static inline void +__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov (mh)\n", p->thread_id); + + MOV_MH(t, ip); +} + +static inline void +__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov (hm)\n", p->thread_id); + + MOV_HM(t, ip); +} + +static inline void +__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov (hh)\n", p->thread_id); + + MOV_HH(t, ip); +} + +static inline void +__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val); + + MOV_I(t, ip); +} + +/* + * dma. + */ +static inline void +__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip, + uint32_t n_dma) +{ + uint8_t *action_data = t->structs[0]; + uint64_t valid_headers = t->valid_headers; + uint32_t i; + + for (i = 0; i < n_dma; i++) { + uint32_t header_id = ip->dma.dst.header_id[i]; + uint32_t struct_id = ip->dma.dst.struct_id[i]; + uint32_t offset = ip->dma.src.offset[i]; + uint32_t n_bytes = ip->dma.n_bytes[i]; + + struct header_runtime *h = &t->headers[header_id]; + uint8_t *h_ptr0 = h->ptr0; + uint8_t *h_ptr = t->structs[struct_id]; + + void *dst = MASK64_BIT_GET(valid_headers, header_id) ? + h_ptr : h_ptr0; + void *src = &action_data[offset]; + + TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id); + + /* Headers. */ + memcpy(dst, src, n_bytes); + t->structs[struct_id] = dst; + valid_headers = MASK64_BIT_SET(valid_headers, header_id); + } + + t->valid_headers = valid_headers; +} + +static inline void +__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + __instr_dma_ht_many_exec(p, t, ip, 1); +} + +static inline void +__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 2); +} + +static inline void +__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 3); +} + +static inline void +__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 4); +} + +static inline void +__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 5); +} + +static inline void +__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 6); +} + +static inline void +__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 7); +} + +static inline void +__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 8); +} + +/* + * alu. + */ +static inline void +__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add\n", p->thread_id); + + ALU(t, ip, +); +} + +static inline void +__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (mh)\n", p->thread_id); + + ALU_MH(t, ip, +); +} + +static inline void +__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (hm)\n", p->thread_id); + + ALU_HM(t, ip, +); +} + +static inline void +__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (hh)\n", p->thread_id); + + ALU_HH(t, ip, +); +} + +static inline void +__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (mi)\n", p->thread_id); + + ALU_MI(t, ip, +); +} + +static inline void +__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (hi)\n", p->thread_id); + + ALU_HI(t, ip, +); +} + +static inline void +__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub\n", p->thread_id); + + ALU(t, ip, -); +} + +static inline void +__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (mh)\n", p->thread_id); + + ALU_MH(t, ip, -); +} + +static inline void +__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (hm)\n", p->thread_id); + + ALU_HM(t, ip, -); +} + +static inline void +__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (hh)\n", p->thread_id); + + ALU_HH(t, ip, -); +} + +static inline void +__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (mi)\n", p->thread_id); + + ALU_MI(t, ip, -); +} + +static inline void +__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (hi)\n", p->thread_id); + + ALU_HI(t, ip, -); +} + +static inline void +__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl\n", p->thread_id); + + ALU(t, ip, <<); +} + +static inline void +__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (mh)\n", p->thread_id); + + ALU_MH(t, ip, <<); +} + +static inline void +__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (hm)\n", p->thread_id); + + ALU_HM(t, ip, <<); +} + +static inline void +__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (hh)\n", p->thread_id); + + ALU_HH(t, ip, <<); +} + +static inline void +__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (mi)\n", p->thread_id); + + ALU_MI(t, ip, <<); +} + +static inline void +__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (hi)\n", p->thread_id); + + ALU_HI(t, ip, <<); +} + +static inline void +__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr\n", p->thread_id); + + ALU(t, ip, >>); +} + +static inline void +__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (mh)\n", p->thread_id); + + ALU_MH(t, ip, >>); +} + +static inline void +__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (hm)\n", p->thread_id); + + ALU_HM(t, ip, >>); +} + +static inline void +__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (hh)\n", p->thread_id); + + ALU_HH(t, ip, >>); +} + +static inline void +__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (mi)\n", p->thread_id); + + /* Structs. */ + ALU_MI(t, ip, >>); +} + +static inline void +__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (hi)\n", p->thread_id); + + ALU_HI(t, ip, >>); +} + +static inline void +__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and\n", p->thread_id); + + ALU(t, ip, &); +} + +static inline void +__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (mh)\n", p->thread_id); + + ALU_MH(t, ip, &); +} + +static inline void +__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (hm)\n", p->thread_id); + + ALU_HM_FAST(t, ip, &); +} + +static inline void +__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (hh)\n", p->thread_id); + + ALU_HH_FAST(t, ip, &); +} + +static inline void +__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (i)\n", p->thread_id); + + ALU_I(t, ip, &); +} + +static inline void +__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or\n", p->thread_id); + + ALU(t, ip, |); +} + +static inline void +__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (mh)\n", p->thread_id); + + ALU_MH(t, ip, |); +} + +static inline void +__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (hm)\n", p->thread_id); + + ALU_HM_FAST(t, ip, |); +} + +static inline void +__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (hh)\n", p->thread_id); + + ALU_HH_FAST(t, ip, |); +} + +static inline void +__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (i)\n", p->thread_id); + + ALU_I(t, ip, |); +} + +static inline void +__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor\n", p->thread_id); + + ALU(t, ip, ^); +} + +static inline void +__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (mh)\n", p->thread_id); + + ALU_MH(t, ip, ^); +} + +static inline void +__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (hm)\n", p->thread_id); + + ALU_HM_FAST(t, ip, ^); +} + +static inline void +__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (hh)\n", p->thread_id); + + ALU_HH_FAST(t, ip, ^); +} + +static inline void +__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (i)\n", p->thread_id); + + ALU_I(t, ip, ^); +} + +static inline void +__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint64_t *src64_ptr, src64, src64_mask, src; + uint64_t r; + + TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; + src64 = *src64_ptr; + src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); + src = src64 & src64_mask; + + r = dst; + r = ~r & 0xFFFF; + + /* The first input (r) is a 16-bit number. The second and the third + * inputs are 32-bit numbers. In the worst case scenario, the sum of the + * three numbers (output r) is a 34-bit number. + */ + r += (src >> 32) + (src & 0xFFFFFFFF); + + /* The first input is a 16-bit number. The second input is an 18-bit + * number. In the worst case scenario, the sum of the two numbers is a + * 19-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, + * therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; +} + +static inline void +__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint64_t *src64_ptr, src64, src64_mask, src; + uint64_t r; + + TRACE("[Thread %2u] cksub (field)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; + src64 = *src64_ptr; + src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); + src = src64 & src64_mask; + + r = dst; + r = ~r & 0xFFFF; + + /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as + * the following sequence of operations in 2's complement arithmetic: + * a '- b = (a - b) % 0xFFFF. + * + * In order to prevent an underflow for the below subtraction, in which + * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the + * minuend), we first add a multiple of the 0xFFFF modulus to the + * minuend. The number we add to the minuend needs to be a 34-bit number + * or higher, so for readability reasons we picked the 36-bit multiple. + * We are effectively turning the 16-bit minuend into a 36-bit number: + * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. + */ + r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */ + + /* A 33-bit number is subtracted from a 36-bit number (the input r). The + * result (the output r) is a 36-bit number. + */ + r -= (src >> 32) + (src & 0xFFFFFFFF); + + /* The first input is a 16-bit number. The second input is a 20-bit + * number. Their sum is a 21-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1001E), the output r is (0 .. 31). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; +} + +static inline void +__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr; + uint32_t *src32_ptr; + uint64_t r0, r1; + + TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + + src_struct = t->structs[ip->alu.src.struct_id]; + src32_ptr = (uint32_t *)&src_struct[0]; + + r0 = src32_ptr[0]; /* r0 is a 32-bit number. */ + r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ + r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */ + r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ + r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ + + /* The first input is a 16-bit number. The second input is a 19-bit + * number. Their sum is a 20-bit number. + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1000E), the output r is (0 .. 15). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + r0 = ~r0 & 0xFFFF; + r0 = r0 ? r0 : 0xFFFF; + + *dst16_ptr = (uint16_t)r0; +} + +static inline void +__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr; + uint32_t *src32_ptr; + uint64_t r = 0; + uint32_t i; + + TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + + src_struct = t->structs[ip->alu.src.struct_id]; + src32_ptr = (uint32_t *)&src_struct[0]; + + /* The max number of 32-bit words in a 256-byte header is 8 = 2^3. + * Therefore, in the worst case scenario, a 35-bit number is added to a + * 16-bit number (the input r), so the output r is 36-bit number. + */ + for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++) + r += *src32_ptr; + + /* The first input is a 16-bit number. The second input is a 20-bit + * number. Their sum is a 21-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1001E), the output r is (0 .. 31). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; +} + +/* + * Register array. + */ +static inline uint64_t * +instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + return r->regarray; +} + +static inline uint64_t +instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return idx; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; + + return idx; +} + +#else + +#define instr_regarray_idx_nbo instr_regarray_idx_hbo + +#endif + +static inline uint64_t +instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint64_t idx = ip->regarray.idx_val & r->size_mask; + + return idx; +} + +static inline uint64_t +instr_regarray_src_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + uint64_t src = src64 & src64_mask; + + return src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_src_nbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); + + return src; +} + +#else + +#define instr_regarray_src_nbo instr_regarray_src_hbo + +#endif + +static inline void +instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); + +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline void +instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +#else + +#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set + +#endif + +static inline void +__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regrd_hrh_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_hrm_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rii_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +static inline void +__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rii_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +/* + * metarray. + */ +static inline struct meter * +instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return &r->metarray[idx]; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline struct meter * +instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; + + return &r->metarray[idx]; +} + +#else + +#define instr_meter_idx_nbo instr_meter_idx_hbo + +#endif + +static inline struct meter * +instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint64_t idx = ip->meter.idx_val & r->size_mask; + + return &r->metarray[idx]; +} + +static inline uint32_t +instr_meter_length_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); + uint64_t src = src64 & src64_mask; + + return (uint32_t)src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint32_t +instr_meter_length_nbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); + + return (uint32_t)src; +} + +#else + +#define instr_meter_length_nbo instr_meter_length_hbo + +#endif + +static inline enum rte_color +instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); + uint64_t src = src64 & src64_mask; + + return (enum rte_color)src; +} + +static inline void +instr_meter_color_out_hbo_set(struct thread *t, + const struct instruction *ip, + enum rte_color color_out) +{ + uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); + + uint64_t src = (uint64_t)color_out; + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +static inline void +__instr_metprefetch_h_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m); +} + +static inline void +__instr_metprefetch_m_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m); +} + +static inline void +__instr_metprefetch_i_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m); +} + +static inline void +__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imm)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imi)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + #endif