+/*
+ * validate.
+ */
+static inline void
+__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t header_id = ip->valid.header_id;
+
+ TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
+
+ /* Headers. */
+ t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
+}
+
+/*
+ * invalidate.
+ */
+static inline void
+__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t header_id = ip->valid.header_id;
+
+ TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
+
+ /* Headers. */
+ t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
+}
+
+/*
+ * learn.
+ */
+static inline void
+__instr_learn_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t action_id = ip->learn.action_id;
+ uint32_t mf_offset = ip->learn.mf_offset;
+ uint32_t learner_id = t->learner_id;
+ struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
+ p->n_selectors + learner_id];
+ struct learner_runtime *l = &t->learners[learner_id];
+ struct learner_statistics *stats = &p->learner_stats[learner_id];
+ uint32_t status;
+
+ /* Table. */
+ status = rte_swx_table_learner_add(ts->obj,
+ l->mailbox,
+ t->time,
+ action_id,
+ &t->metadata[mf_offset]);
+
+ TRACE("[Thread %2u] learner %u learn %s\n",
+ p->thread_id,
+ learner_id,
+ status ? "ok" : "error");
+
+ stats->n_pkts_learn[status] += 1;
+}
+
+/*
+ * forget.
+ */
+static inline void
+__instr_forget_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip __rte_unused)
+{
+ uint32_t learner_id = t->learner_id;
+ struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
+ p->n_selectors + learner_id];
+ struct learner_runtime *l = &t->learners[learner_id];
+ struct learner_statistics *stats = &p->learner_stats[learner_id];
+
+ /* Table. */
+ rte_swx_table_learner_delete(ts->obj, l->mailbox);
+
+ TRACE("[Thread %2u] learner %u forget\n",
+ p->thread_id,
+ learner_id);
+
+ stats->n_pkts_forget += 1;
+}
+
+/*
+ * extern.
+ */
+static inline uint32_t
+__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t obj_id = ip->ext_obj.ext_obj_id;
+ uint32_t func_id = ip->ext_obj.func_id;
+ struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
+ rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
+ uint32_t done;
+
+ TRACE("[Thread %2u] extern obj %u member func %u\n",
+ p->thread_id,
+ obj_id,
+ func_id);
+
+ done = func(obj->obj, obj->mailbox);
+
+ return done;
+}
+
+static inline uint32_t
+__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t ext_func_id = ip->ext_func.ext_func_id;
+ struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
+ rte_swx_extern_func_t func = ext_func->func;
+ uint32_t done;
+
+ TRACE("[Thread %2u] extern func %u\n",
+ p->thread_id,
+ ext_func_id);
+
+ done = func(ext_func->mailbox);
+
+ return done;
+}
+
+/*
+ * mov.
+ */
+static inline void
+__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov\n", p->thread_id);
+
+ MOV(t, ip);
+}
+
+static inline void
+__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
+
+ MOV_MH(t, ip);
+}
+
+static inline void
+__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
+
+ MOV_HM(t, ip);
+}
+
+static inline void
+__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
+
+ MOV_HH(t, ip);
+}
+
+static inline void
+__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
+
+ MOV_I(t, ip);
+}
+
+/*
+ * dma.
+ */
+static inline void
+__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip,
+ uint32_t n_dma)
+{
+ uint8_t *action_data = t->structs[0];
+ uint64_t valid_headers = t->valid_headers;
+ uint32_t i;
+
+ for (i = 0; i < n_dma; i++) {
+ uint32_t header_id = ip->dma.dst.header_id[i];
+ uint32_t struct_id = ip->dma.dst.struct_id[i];
+ uint32_t offset = ip->dma.src.offset[i];
+ uint32_t n_bytes = ip->dma.n_bytes[i];
+
+ struct header_runtime *h = &t->headers[header_id];
+ uint8_t *h_ptr0 = h->ptr0;
+ uint8_t *h_ptr = t->structs[struct_id];
+
+ void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
+ h_ptr : h_ptr0;
+ void *src = &action_data[offset];
+
+ TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
+
+ /* Headers. */
+ memcpy(dst, src, n_bytes);
+ t->structs[struct_id] = dst;
+ valid_headers = MASK64_BIT_SET(valid_headers, header_id);
+ }
+
+ t->valid_headers = valid_headers;
+}
+
+static inline void
+__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ __instr_dma_ht_many_exec(p, t, ip, 1);
+}
+
+static inline void
+__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 2);
+}
+
+static inline void
+__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 3);
+}
+
+static inline void
+__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 4);
+}
+
+static inline void
+__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 5);
+}
+
+static inline void
+__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 6);
+}
+
+static inline void
+__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 7);
+}
+
+static inline void
+__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 8);
+}
+
+/*
+ * alu.
+ */
+static inline void
+__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add\n", p->thread_id);
+
+ ALU(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (mi)\n", p->thread_id);
+
+ ALU_MI(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, +);
+}
+
+static inline void
+__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub\n", p->thread_id);
+
+ ALU(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
+
+ ALU_MI(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, -);
+}
+
+static inline void
+__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl\n", p->thread_id);
+
+ ALU(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
+
+ ALU_MI(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr\n", p->thread_id);
+
+ ALU(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MI(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, >>);
+}
+
+static inline void
+__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and\n", p->thread_id);
+
+ ALU(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (hm)\n", p->thread_id);
+
+ ALU_HM_FAST(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (hh)\n", p->thread_id);
+
+ ALU_HH_FAST(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (i)\n", p->thread_id);
+
+ ALU_I(t, ip, &);
+}
+
+static inline void
+__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or\n", p->thread_id);
+
+ ALU(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (hm)\n", p->thread_id);
+
+ ALU_HM_FAST(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (hh)\n", p->thread_id);
+
+ ALU_HH_FAST(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (i)\n", p->thread_id);
+
+ ALU_I(t, ip, |);
+}
+
+static inline void
+__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor\n", p->thread_id);
+
+ ALU(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
+
+ ALU_HM_FAST(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
+
+ ALU_HH_FAST(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (i)\n", p->thread_id);
+
+ ALU_I(t, ip, ^);
+}
+
+static inline void
+__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* The first input (r) is a 16-bit number. The second and the third
+ * inputs are 32-bit numbers. In the worst case scenario, the sum of the
+ * three numbers (output r) is a 34-bit number.
+ */
+ r += (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is an 18-bit
+ * number. In the worst case scenario, the sum of the two numbers is a
+ * 19-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
+ * therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+}
+
+static inline void
+__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
+ * the following sequence of operations in 2's complement arithmetic:
+ * a '- b = (a - b) % 0xFFFF.
+ *
+ * In order to prevent an underflow for the below subtraction, in which
+ * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
+ * minuend), we first add a multiple of the 0xFFFF modulus to the
+ * minuend. The number we add to the minuend needs to be a 34-bit number
+ * or higher, so for readability reasons we picked the 36-bit multiple.
+ * We are effectively turning the 16-bit minuend into a 36-bit number:
+ * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
+ */
+ r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
+
+ /* A 33-bit number is subtracted from a 36-bit number (the input r). The
+ * result (the output r) is a 36-bit number.
+ */
+ r -= (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+}
+
+static inline void
+__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr;
+ uint32_t *src32_ptr;
+ uint64_t r0, r1;
+
+ TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
+ r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
+ r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
+ r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
+ r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
+
+ /* The first input is a 16-bit number. The second input is a 19-bit
+ * number. Their sum is a 20-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1000E), the output r is (0 .. 15). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ r0 = ~r0 & 0xFFFF;
+ r0 = r0 ? r0 : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r0;
+}
+
+static inline void
+__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr;
+ uint32_t *src32_ptr;
+ uint64_t r = 0;
+ uint32_t i;
+
+ TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
+ * Therefore, in the worst case scenario, a 35-bit number is added to a
+ * 16-bit number (the input r), so the output r is 36-bit number.
+ */
+ for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
+ r += *src32_ptr;
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+}
+
+/*
+ * Register array.
+ */
+static inline uint64_t *
+instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+ return r->regarray;
+}
+
+static inline uint64_t
+instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
+ uint64_t idx = idx64 & idx64_mask & r->size_mask;
+
+ return idx;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
+
+ return idx;
+}
+
+#else
+
+#define instr_regarray_idx_nbo instr_regarray_idx_hbo
+
+#endif
+
+static inline uint64_t
+instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint64_t idx = ip->regarray.idx_val & r->size_mask;
+
+ return idx;
+}
+
+static inline uint64_t
+instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return src;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
+
+ return src;
+}
+
+#else
+
+#define instr_regarray_src_nbo instr_regarray_src_hbo
+
+#endif
+
+static inline void
+instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
+{
+ uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline void
+instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
+{
+ uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+
+ src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+}
+
+#else
+
+#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
+
+#endif
+
+static inline void
+__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ rte_prefetch0(®array[idx]);
+}
+
+static inline void
+__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ rte_prefetch0(®array[idx]);
+}
+
+static inline void
+__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ rte_prefetch0(®array[idx]);
+}
+
+static inline void
+__instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rii_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rii_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+}
+
+/*
+ * metarray.
+ */
+static inline struct meter *
+instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
+ uint64_t idx = idx64 & idx64_mask & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline struct meter *
+instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+#else
+
+#define instr_meter_idx_nbo instr_meter_idx_hbo
+
+#endif
+
+static inline struct meter *
+instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint64_t idx = ip->meter.idx_val & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+static inline uint32_t
+instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return (uint32_t)src;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint32_t
+instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
+
+ return (uint32_t)src;
+}
+
+#else
+
+#define instr_meter_length_nbo instr_meter_length_hbo
+
+#endif
+
+static inline enum rte_color
+instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return (enum rte_color)src;
+}
+
+static inline void
+instr_meter_color_out_hbo_set(struct thread *t,
+ const struct instruction *ip,
+ enum rte_color color_out)
+{
+ uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
+
+ uint64_t src = (uint64_t)color_out;
+
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+}
+
+static inline void
+__instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m);
+}
+
+static inline void
+__instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m);
+}
+
+static inline void
+__instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m);
+}
+
+static inline void
+__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+