X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=lib%2Fpipeline%2Frte_swx_pipeline_internal.h;h=8d932993febb17041ae2669696c03f2ef5cc6d44;hb=775be5b5399da981100f77988b100613ce44a9d5;hp=e9fe6632b6effc3cfe95671d628b1b965f123d0f;hpb=fcb03ae09e5ca082834a424d043024447b54f525;p=dpdk.git diff --git a/lib/pipeline/rte_swx_pipeline_internal.h b/lib/pipeline/rte_swx_pipeline_internal.h index e9fe6632b6..8d932993fe 100644 --- a/lib/pipeline/rte_swx_pipeline_internal.h +++ b/lib/pipeline/rte_swx_pipeline_internal.h @@ -225,6 +225,7 @@ enum instruction_type { */ INSTR_TX, /* port_out = M */ INSTR_TX_I, /* port_out = I */ + INSTR_DROP, /* extract h.header */ INSTR_HDR_EXTRACT, @@ -304,16 +305,16 @@ enum instruction_type { INSTR_ALU_SUB_HI, /* dst = H, src = I */ /* ckadd dst src - * dst = dst '+ src[0:1] '+ src[2:3] + ... - * dst = H, src = {H, h.header} + * dst = dst '+ src[0:1] '+ src[2:3] '+ ... + * dst = H, src = {H, h.header}, '+ = 1's complement addition operator */ INSTR_ALU_CKADD_FIELD, /* src = H */ - INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */ - INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */ + INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */ + INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */ /* cksub dst src * dst = dst '- src - * dst = H, src = H + * dst = H, src = H, '- = 1's complement subtraction operator */ INSTR_ALU_CKSUB_FIELD, @@ -443,10 +444,12 @@ enum instruction_type { /* table TABLE */ INSTR_TABLE, + INSTR_TABLE_AF, INSTR_SELECTOR, INSTR_LEARNER, + INSTR_LEARNER_AF, - /* learn LEARNER ACTION_NAME */ + /* learn LEARNER ACTION_NAME [ m.action_first_arg ] */ INSTR_LEARNER_LEARN, /* forget */ @@ -539,6 +542,9 @@ enum instruction_type { * Return from action */ INSTR_RETURN, + + /* Start of custom instructions. */ + INSTR_CUSTOM_0, }; struct instr_operand { @@ -578,6 +584,7 @@ struct instr_table { struct instr_learn { uint8_t action_id; + uint8_t mf_offset; }; struct instr_extern_obj { @@ -684,15 +691,21 @@ struct instruction_data { int invalid; }; +typedef void (*instr_exec_t)(struct rte_swx_pipeline *); + /* * Action. */ +typedef void +(*action_func_t)(struct rte_swx_pipeline *p); + struct action { TAILQ_ENTRY(action) node; char name[RTE_SWX_NAME_SIZE]; struct struct_type *st; int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */ struct instruction *instructions; + struct instruction_data *instruction_data; uint32_t n_instructions; uint32_t id; }; @@ -734,6 +747,8 @@ struct table { uint32_t n_actions; int default_action_is_const; uint32_t action_data_size_max; + int *action_is_for_table_entries; + int *action_is_for_default_entry; uint32_t size; uint32_t id; @@ -798,12 +813,13 @@ struct learner { /* Action. */ struct action **actions; - struct field **action_arg; struct action *default_action; uint8_t *default_action_data; uint32_t n_actions; int default_action_is_const; uint32_t action_data_size_max; + int *action_is_for_table_entries; + int *action_is_for_default_entry; uint32_t size; uint32_t timeout; @@ -815,7 +831,6 @@ TAILQ_HEAD(learner_tailq, learner); struct learner_runtime { void *mailbox; uint8_t **key; - uint8_t **action_data; }; struct learner_statistics { @@ -928,6 +943,35 @@ struct thread { #define HEADER_VALID(thread, header_id) \ MASK64_BIT_GET((thread)->valid_headers, header_id) +static inline uint64_t +instr_operand_hbo(struct thread *t, const struct instr_operand *x) +{ + uint8_t *x_struct = t->structs[x->struct_id]; + uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; + uint64_t x64 = *x64_ptr; + uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits); + + return x64 & x64_mask; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_operand_nbo(struct thread *t, const struct instr_operand *x) +{ + uint8_t *x_struct = t->structs[x->struct_id]; + uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset]; + uint64_t x64 = *x64_ptr; + + return ntoh64(x64) >> (64 - x->n_bits); +} + +#else + +#define instr_operand_nbo instr_operand_hbo + +#endif + #define ALU(thread, ip, operator) \ { \ uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ @@ -1328,6 +1372,10 @@ struct thread { #define RTE_SWX_PIPELINE_THREADS_MAX 16 #endif +#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX +#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256 +#endif + struct rte_swx_pipeline { struct struct_type_tailq struct_types; struct port_in_type_tailq port_in_types; @@ -1352,6 +1400,7 @@ struct rte_swx_pipeline { struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; + action_func_t *action_funcs; struct rte_swx_table_state *table_state; struct table_statistics *table_stats; struct selector_statistics *selector_stats; @@ -1359,7 +1408,10 @@ struct rte_swx_pipeline { struct regarray_runtime *regarray_runtime; struct metarray_runtime *metarray_runtime; struct instruction *instructions; + struct instruction_data *instruction_data; + instr_exec_t *instruction_table; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; + void *lib; uint32_t n_structs; uint32_t n_ports_in; @@ -1511,7 +1563,7 @@ emit_handler(struct thread *t) return; } - /* Header encapsulation (optionally, with prior header decasulation). */ + /* Header encapsulation (optionally, with prior header decapsulation). */ if ((t->n_headers_out == 2) && (h1->ptr + h1->n_bytes == t->ptr) && (h0->ptr == h0->ptr0)) { @@ -1580,4 +1632,2316 @@ __instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct ins port->pkt_tx(port->obj, pkt); } +static inline void +__instr_drop_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip __rte_unused) +{ + uint64_t port_id = p->n_ports_out - 1; + struct port_out_runtime *port = &p->out[port_id]; + struct rte_swx_pkt *pkt = &t->pkt; + + TRACE("[Thread %2u]: drop 1 pkt\n", + p->thread_id); + + /* Headers. */ + emit_handler(t); + + /* Packet. */ + port->pkt_tx(port->obj, pkt); +} + +/* + * extract. + */ +static inline void +__instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip, + uint32_t n_extract) +{ + uint64_t valid_headers = t->valid_headers; + uint8_t *ptr = t->ptr; + uint32_t offset = t->pkt.offset; + uint32_t length = t->pkt.length; + uint32_t i; + + for (i = 0; i < n_extract; i++) { + uint32_t header_id = ip->io.hdr.header_id[i]; + uint32_t struct_id = ip->io.hdr.struct_id[i]; + uint32_t n_bytes = ip->io.hdr.n_bytes[i]; + + TRACE("[Thread %2u]: extract header %u (%u bytes)\n", + p->thread_id, + header_id, + n_bytes); + + /* Headers. */ + t->structs[struct_id] = ptr; + valid_headers = MASK64_BIT_SET(valid_headers, header_id); + + /* Packet. */ + offset += n_bytes; + length -= n_bytes; + ptr += n_bytes; + } + + /* Headers. */ + t->valid_headers = valid_headers; + + /* Packet. */ + t->pkt.offset = offset; + t->pkt.length = length; + t->ptr = ptr; +} + +static inline void +__instr_hdr_extract_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + __instr_hdr_extract_many_exec(p, t, ip, 1); +} + +static inline void +__instr_hdr_extract2_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 2); +} + +static inline void +__instr_hdr_extract3_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 3); +} + +static inline void +__instr_hdr_extract4_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 4); +} + +static inline void +__instr_hdr_extract5_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 5); +} + +static inline void +__instr_hdr_extract6_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 6); +} + +static inline void +__instr_hdr_extract7_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 7); +} + +static inline void +__instr_hdr_extract8_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_extract_many_exec(p, t, ip, 8); +} + +static inline void +__instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint64_t valid_headers = t->valid_headers; + uint8_t *ptr = t->ptr; + uint32_t offset = t->pkt.offset; + uint32_t length = t->pkt.length; + + uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits); + uint32_t header_id = ip->io.hdr.header_id[0]; + uint32_t struct_id = ip->io.hdr.struct_id[0]; + uint32_t n_bytes = ip->io.hdr.n_bytes[0]; + + struct header_runtime *h = &t->headers[header_id]; + + TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n", + p->thread_id, + header_id, + n_bytes, + n_bytes_last); + + n_bytes += n_bytes_last; + + /* Headers. */ + t->structs[struct_id] = ptr; + t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); + h->n_bytes = n_bytes; + + /* Packet. */ + t->pkt.offset = offset + n_bytes; + t->pkt.length = length - n_bytes; + t->ptr = ptr + n_bytes; +} + +static inline void +__instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint64_t valid_headers = t->valid_headers; + uint8_t *ptr = t->ptr; + + uint32_t header_id = ip->io.hdr.header_id[0]; + uint32_t struct_id = ip->io.hdr.struct_id[0]; + + TRACE("[Thread %2u]: lookahead header %u\n", + p->thread_id, + header_id); + + /* Headers. */ + t->structs[struct_id] = ptr; + t->valid_headers = MASK64_BIT_SET(valid_headers, header_id); +} + +/* + * emit. + */ +static inline void +__instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip, + uint32_t n_emit) +{ + uint64_t valid_headers = t->valid_headers; + uint32_t n_headers_out = t->n_headers_out; + struct header_out_runtime *ho = NULL; + uint8_t *ho_ptr = NULL; + uint32_t ho_nbytes = 0, i; + + for (i = 0; i < n_emit; i++) { + uint32_t header_id = ip->io.hdr.header_id[i]; + uint32_t struct_id = ip->io.hdr.struct_id[i]; + + struct header_runtime *hi = &t->headers[header_id]; + uint8_t *hi_ptr0 = hi->ptr0; + uint32_t n_bytes = hi->n_bytes; + + uint8_t *hi_ptr = t->structs[struct_id]; + + if (!MASK64_BIT_GET(valid_headers, header_id)) { + TRACE("[Thread %2u]: emit header %u (invalid)\n", + p->thread_id, + header_id); + + continue; + } + + TRACE("[Thread %2u]: emit header %u (valid)\n", + p->thread_id, + header_id); + + /* Headers. */ + if (!ho) { + if (!n_headers_out) { + ho = &t->headers_out[0]; + + ho->ptr0 = hi_ptr0; + ho->ptr = hi_ptr; + + ho_ptr = hi_ptr; + ho_nbytes = n_bytes; + + n_headers_out = 1; + + continue; + } else { + ho = &t->headers_out[n_headers_out - 1]; + + ho_ptr = ho->ptr; + ho_nbytes = ho->n_bytes; + } + } + + if (ho_ptr + ho_nbytes == hi_ptr) { + ho_nbytes += n_bytes; + } else { + ho->n_bytes = ho_nbytes; + + ho++; + ho->ptr0 = hi_ptr0; + ho->ptr = hi_ptr; + + ho_ptr = hi_ptr; + ho_nbytes = n_bytes; + + n_headers_out++; + } + } + + if (ho) + ho->n_bytes = ho_nbytes; + t->n_headers_out = n_headers_out; +} + +static inline void +__instr_hdr_emit_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + __instr_hdr_emit_many_exec(p, t, ip, 1); +} + +static inline void +__instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 1); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 2); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 3); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 4); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 5); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 6); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 7); + __instr_tx_exec(p, t, ip); +} + +static inline void +__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id); + + __instr_hdr_emit_many_exec(p, t, ip, 8); + __instr_tx_exec(p, t, ip); +} + +/* + * validate. + */ +static inline void +__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t header_id = ip->valid.header_id; + + TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id); + + /* Headers. */ + t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id); +} + +/* + * invalidate. + */ +static inline void +__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t header_id = ip->valid.header_id; + + TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id); + + /* Headers. */ + t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id); +} + +/* + * learn. + */ +static inline void +__instr_learn_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t action_id = ip->learn.action_id; + uint32_t mf_offset = ip->learn.mf_offset; + uint32_t learner_id = t->learner_id; + struct rte_swx_table_state *ts = &t->table_state[p->n_tables + + p->n_selectors + learner_id]; + struct learner_runtime *l = &t->learners[learner_id]; + struct learner_statistics *stats = &p->learner_stats[learner_id]; + uint32_t status; + + /* Table. */ + status = rte_swx_table_learner_add(ts->obj, + l->mailbox, + t->time, + action_id, + &t->metadata[mf_offset]); + + TRACE("[Thread %2u] learner %u learn %s\n", + p->thread_id, + learner_id, + status ? "ok" : "error"); + + stats->n_pkts_learn[status] += 1; +} + +/* + * forget. + */ +static inline void +__instr_forget_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip __rte_unused) +{ + uint32_t learner_id = t->learner_id; + struct rte_swx_table_state *ts = &t->table_state[p->n_tables + + p->n_selectors + learner_id]; + struct learner_runtime *l = &t->learners[learner_id]; + struct learner_statistics *stats = &p->learner_stats[learner_id]; + + /* Table. */ + rte_swx_table_learner_delete(ts->obj, l->mailbox); + + TRACE("[Thread %2u] learner %u forget\n", + p->thread_id, + learner_id); + + stats->n_pkts_forget += 1; +} + +/* + * extern. + */ +static inline uint32_t +__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t obj_id = ip->ext_obj.ext_obj_id; + uint32_t func_id = ip->ext_obj.func_id; + struct extern_obj_runtime *obj = &t->extern_objs[obj_id]; + rte_swx_extern_type_member_func_t func = obj->funcs[func_id]; + uint32_t done; + + TRACE("[Thread %2u] extern obj %u member func %u\n", + p->thread_id, + obj_id, + func_id); + + done = func(obj->obj, obj->mailbox); + + return done; +} + +static inline uint32_t +__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t ext_func_id = ip->ext_func.ext_func_id; + struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id]; + rte_swx_extern_func_t func = ext_func->func; + uint32_t done; + + TRACE("[Thread %2u] extern func %u\n", + p->thread_id, + ext_func_id); + + done = func(ext_func->mailbox); + + return done; +} + +/* + * mov. + */ +static inline void +__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov\n", p->thread_id); + + MOV(t, ip); +} + +static inline void +__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov (mh)\n", p->thread_id); + + MOV_MH(t, ip); +} + +static inline void +__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov (hm)\n", p->thread_id); + + MOV_HM(t, ip); +} + +static inline void +__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov (hh)\n", p->thread_id); + + MOV_HH(t, ip); +} + +static inline void +__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val); + + MOV_I(t, ip); +} + +/* + * dma. + */ +static inline void +__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip, + uint32_t n_dma) +{ + uint8_t *action_data = t->structs[0]; + uint64_t valid_headers = t->valid_headers; + uint32_t i; + + for (i = 0; i < n_dma; i++) { + uint32_t header_id = ip->dma.dst.header_id[i]; + uint32_t struct_id = ip->dma.dst.struct_id[i]; + uint32_t offset = ip->dma.src.offset[i]; + uint32_t n_bytes = ip->dma.n_bytes[i]; + + struct header_runtime *h = &t->headers[header_id]; + uint8_t *h_ptr0 = h->ptr0; + uint8_t *h_ptr = t->structs[struct_id]; + + void *dst = MASK64_BIT_GET(valid_headers, header_id) ? + h_ptr : h_ptr0; + void *src = &action_data[offset]; + + TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id); + + /* Headers. */ + memcpy(dst, src, n_bytes); + t->structs[struct_id] = dst; + valid_headers = MASK64_BIT_SET(valid_headers, header_id); + } + + t->valid_headers = valid_headers; +} + +static inline void +__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + __instr_dma_ht_many_exec(p, t, ip, 1); +} + +static inline void +__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 2); +} + +static inline void +__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 3); +} + +static inline void +__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 4); +} + +static inline void +__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 5); +} + +static inline void +__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 6); +} + +static inline void +__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 7); +} + +static inline void +__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id); + + __instr_dma_ht_many_exec(p, t, ip, 8); +} + +/* + * alu. + */ +static inline void +__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add\n", p->thread_id); + + ALU(t, ip, +); +} + +static inline void +__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (mh)\n", p->thread_id); + + ALU_MH(t, ip, +); +} + +static inline void +__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (hm)\n", p->thread_id); + + ALU_HM(t, ip, +); +} + +static inline void +__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (hh)\n", p->thread_id); + + ALU_HH(t, ip, +); +} + +static inline void +__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (mi)\n", p->thread_id); + + ALU_MI(t, ip, +); +} + +static inline void +__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] add (hi)\n", p->thread_id); + + ALU_HI(t, ip, +); +} + +static inline void +__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub\n", p->thread_id); + + ALU(t, ip, -); +} + +static inline void +__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (mh)\n", p->thread_id); + + ALU_MH(t, ip, -); +} + +static inline void +__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (hm)\n", p->thread_id); + + ALU_HM(t, ip, -); +} + +static inline void +__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (hh)\n", p->thread_id); + + ALU_HH(t, ip, -); +} + +static inline void +__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (mi)\n", p->thread_id); + + ALU_MI(t, ip, -); +} + +static inline void +__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] sub (hi)\n", p->thread_id); + + ALU_HI(t, ip, -); +} + +static inline void +__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl\n", p->thread_id); + + ALU(t, ip, <<); +} + +static inline void +__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (mh)\n", p->thread_id); + + ALU_MH(t, ip, <<); +} + +static inline void +__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (hm)\n", p->thread_id); + + ALU_HM(t, ip, <<); +} + +static inline void +__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (hh)\n", p->thread_id); + + ALU_HH(t, ip, <<); +} + +static inline void +__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (mi)\n", p->thread_id); + + ALU_MI(t, ip, <<); +} + +static inline void +__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shl (hi)\n", p->thread_id); + + ALU_HI(t, ip, <<); +} + +static inline void +__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr\n", p->thread_id); + + ALU(t, ip, >>); +} + +static inline void +__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (mh)\n", p->thread_id); + + ALU_MH(t, ip, >>); +} + +static inline void +__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (hm)\n", p->thread_id); + + ALU_HM(t, ip, >>); +} + +static inline void +__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (hh)\n", p->thread_id); + + ALU_HH(t, ip, >>); +} + +static inline void +__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (mi)\n", p->thread_id); + + /* Structs. */ + ALU_MI(t, ip, >>); +} + +static inline void +__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] shr (hi)\n", p->thread_id); + + ALU_HI(t, ip, >>); +} + +static inline void +__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and\n", p->thread_id); + + ALU(t, ip, &); +} + +static inline void +__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (mh)\n", p->thread_id); + + ALU_MH(t, ip, &); +} + +static inline void +__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (hm)\n", p->thread_id); + + ALU_HM_FAST(t, ip, &); +} + +static inline void +__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (hh)\n", p->thread_id); + + ALU_HH_FAST(t, ip, &); +} + +static inline void +__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] and (i)\n", p->thread_id); + + ALU_I(t, ip, &); +} + +static inline void +__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or\n", p->thread_id); + + ALU(t, ip, |); +} + +static inline void +__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (mh)\n", p->thread_id); + + ALU_MH(t, ip, |); +} + +static inline void +__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (hm)\n", p->thread_id); + + ALU_HM_FAST(t, ip, |); +} + +static inline void +__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (hh)\n", p->thread_id); + + ALU_HH_FAST(t, ip, |); +} + +static inline void +__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] or (i)\n", p->thread_id); + + ALU_I(t, ip, |); +} + +static inline void +__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor\n", p->thread_id); + + ALU(t, ip, ^); +} + +static inline void +__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (mh)\n", p->thread_id); + + ALU_MH(t, ip, ^); +} + +static inline void +__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (hm)\n", p->thread_id); + + ALU_HM_FAST(t, ip, ^); +} + +static inline void +__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (hh)\n", p->thread_id); + + ALU_HH_FAST(t, ip, ^); +} + +static inline void +__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + TRACE("[Thread %2u] xor (i)\n", p->thread_id); + + ALU_I(t, ip, ^); +} + +static inline void +__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint64_t *src64_ptr, src64, src64_mask, src; + uint64_t r; + + TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; + src64 = *src64_ptr; + src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); + src = src64 & src64_mask; + + /* Initialize the result with destination 1's complement. */ + r = dst; + r = ~r & 0xFFFF; + + /* The first input (r) is a 16-bit number. The second and the third + * inputs are 32-bit numbers. In the worst case scenario, the sum of the + * three numbers (output r) is a 34-bit number. + */ + r += (src >> 32) + (src & 0xFFFFFFFF); + + /* The first input is a 16-bit number. The second input is an 18-bit + * number. In the worst case scenario, the sum of the two numbers is a + * 19-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, + * therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* Apply 1's complement to the result. */ + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; +} + +static inline void +__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint64_t *src64_ptr, src64, src64_mask, src; + uint64_t r; + + TRACE("[Thread %2u] cksub (field)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; + src64 = *src64_ptr; + src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); + src = src64 & src64_mask; + + /* Initialize the result with destination 1's complement. */ + r = dst; + r = ~r & 0xFFFF; + + /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as + * the following sequence of operations in 2's complement arithmetic: + * a '- b = (a - b) % 0xFFFF. + * + * In order to prevent an underflow for the below subtraction, in which + * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the + * minuend), we first add a multiple of the 0xFFFF modulus to the + * minuend. The number we add to the minuend needs to be a 34-bit number + * or higher, so for readability reasons we picked the 36-bit multiple. + * We are effectively turning the 16-bit minuend into a 36-bit number: + * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. + */ + r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */ + + /* A 33-bit number is subtracted from a 36-bit number (the input r). The + * result (the output r) is a 36-bit number. + */ + r -= (src >> 32) + (src & 0xFFFFFFFF); + + /* The first input is a 16-bit number. The second input is a 20-bit + * number. Their sum is a 21-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1001E), the output r is (0 .. 31). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* Apply 1's complement to the result. */ + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; +} + +static inline void +__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint32_t *src32_ptr; + uint64_t r0, r1; + + TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src32_ptr = (uint32_t *)&src_struct[0]; + + /* Initialize the result with destination 1's complement. */ + r0 = dst; + r0 = ~r0 & 0xFFFF; + + r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */ + r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ + r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */ + r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ + r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ + + /* The first input is a 16-bit number. The second input is a 19-bit + * number. Their sum is a 20-bit number. + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1000E), the output r is (0 .. 15). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* Apply 1's complement to the result. */ + r0 = ~r0 & 0xFFFF; + r0 = r0 ? r0 : 0xFFFF; + + *dst16_ptr = (uint16_t)r0; +} + +static inline void +__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused, + struct thread *t, + const struct instruction *ip) +{ + uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */ + uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes; + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint32_t *src32_ptr; + uint64_t r; + uint32_t i; + + if (n_src_header_bytes == 20) { + __instr_alu_ckadd_struct20_exec(p, t, ip); + return; + } + + TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src32_ptr = (uint32_t *)&src_struct[0]; + + /* Initialize the result with destination 1's complement. */ + r = dst; + r = ~r & 0xFFFF; + + /* The max number of 32-bit words in a 32K-byte header is 2^13. + * Therefore, in the worst case scenario, a 45-bit number is added to a + * 16-bit number (the input r), so the output r is 46-bit number. + */ + for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++) + r += *src32_ptr; + + /* The first input is a 16-bit number. The second input is a 30-bit + * number. Their sum is a 31-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* Apply 1's complement to the result. */ + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; +} + +/* + * Register array. + */ +static inline uint64_t * +instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + return r->regarray; +} + +static inline uint64_t +instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return idx; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask; + + return idx; +} + +#else + +#define instr_regarray_idx_nbo instr_regarray_idx_hbo + +#endif + +static inline uint64_t +instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id]; + + uint64_t idx = ip->regarray.idx_val & r->size_mask; + + return idx; +} + +static inline uint64_t +instr_regarray_src_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + uint64_t src = src64 & src64_mask; + + return src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint64_t +instr_regarray_src_nbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits); + + return src; +} + +#else + +#define instr_regarray_src_nbo instr_regarray_src_hbo + +#endif + +static inline void +instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); + +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline void +instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src) +{ + uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits); + + src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits); + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +#else + +#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set + +#endif + +static inline void +__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + rte_prefetch0(®array[idx]); +} + +static inline void +__instr_regrd_hrh_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_hrm_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]); +} + +static inline void +__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] = src; +} + +static inline void +__instr_regwr_rii_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] = src; +} + +static inline void +__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_nbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +static inline void +__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_hbo(p, t, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +static inline void +__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_nbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = instr_regarray_src_hbo(t, ip); + regarray[idx] += src; +} + +static inline void +__instr_regadd_rii_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + uint64_t *regarray, idx, src; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + regarray = instr_regarray_regarray(p, ip); + idx = instr_regarray_idx_imm(p, ip); + src = ip->regarray.dstsrc_val; + regarray[idx] += src; +} + +/* + * metarray. + */ +static inline struct meter * +instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits); + uint64_t idx = idx64 & idx64_mask & r->size_mask; + + return &r->metarray[idx]; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline struct meter * +instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id]; + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset]; + uint64_t idx64 = *idx64_ptr; + uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask; + + return &r->metarray[idx]; +} + +#else + +#define instr_meter_idx_nbo instr_meter_idx_hbo + +#endif + +static inline struct meter * +instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip) +{ + struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id]; + + uint64_t idx = ip->meter.idx_val & r->size_mask; + + return &r->metarray[idx]; +} + +static inline uint32_t +instr_meter_length_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits); + uint64_t src = src64 & src64_mask; + + return (uint32_t)src; +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +static inline uint32_t +instr_meter_length_nbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.length.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits); + + return (uint32_t)src; +} + +#else + +#define instr_meter_length_nbo instr_meter_length_hbo + +#endif + +static inline enum rte_color +instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip) +{ + uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id]; + uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset]; + uint64_t src64 = *src64_ptr; + uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits); + uint64_t src = src64 & src64_mask; + + return (enum rte_color)src; +} + +static inline void +instr_meter_color_out_hbo_set(struct thread *t, + const struct instruction *ip, + enum rte_color color_out) +{ + uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id]; + uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset]; + uint64_t dst64 = *dst64_ptr; + uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits); + + uint64_t src = (uint64_t)color_out; + + *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); +} + +static inline void +__instr_metprefetch_h_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m); +} + +static inline void +__instr_metprefetch_m_exec(struct rte_swx_pipeline *p, + struct thread *t, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m); +} + +static inline void +__instr_metprefetch_i_exec(struct rte_swx_pipeline *p, + struct thread *t __rte_unused, + const struct instruction *ip) +{ + struct meter *m; + + TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m); +} + +static inline void +__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhm)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hhi)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmm)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (hmi)\n", p->thread_id); + + m = instr_meter_idx_nbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhm)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mhi)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmm)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (mmi)\n", p->thread_id); + + m = instr_meter_idx_hbo(p, t, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihm)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (ihi)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_nbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imm)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = instr_meter_color_in_hbo(t, ip); + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + +static inline void +__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip) +{ + struct meter *m; + uint64_t time, n_pkts, n_bytes; + uint32_t length; + enum rte_color color_in, color_out; + + TRACE("[Thread %2u] meter (imi)\n", p->thread_id); + + m = instr_meter_idx_imm(p, ip); + rte_prefetch0(m->n_pkts); + time = rte_get_tsc_cycles(); + length = instr_meter_length_hbo(t, ip); + color_in = (enum rte_color)ip->meter.color_in_val; + + color_out = rte_meter_trtcm_color_aware_check(&m->m, + &m->profile->profile, + time, + length, + color_in); + + color_out &= m->color_mask; + + n_pkts = m->n_pkts[color_out]; + n_bytes = m->n_bytes[color_out]; + + instr_meter_color_out_hbo_set(t, ip, color_out); + + m->n_pkts[color_out] = n_pkts + 1; + m->n_bytes[color_out] = n_bytes + length; +} + #endif