*/
INSTR_TX, /* port_out = M */
INSTR_TX_I, /* port_out = I */
+ INSTR_DROP,
/* extract h.header */
INSTR_HDR_EXTRACT,
INSTR_ALU_SUB_HI, /* dst = H, src = I */
/* ckadd dst src
- * dst = dst '+ src[0:1] '+ src[2:3] + ...
- * dst = H, src = {H, h.header}
+ * dst = dst '+ src[0:1] '+ src[2:3] '+ ...
+ * dst = H, src = {H, h.header}, '+ = 1's complement addition operator
*/
INSTR_ALU_CKADD_FIELD, /* src = H */
- INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
- INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
+ INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 bytes. */
+ INSTR_ALU_CKADD_STRUCT, /* src = h.header, with sizeof(header) any 4-byte multiple. */
/* cksub dst src
* dst = dst '- src
- * dst = H, src = H
+ * dst = H, src = H, '- = 1's complement subtraction operator
*/
INSTR_ALU_CKSUB_FIELD,
/* table TABLE */
INSTR_TABLE,
+ INSTR_TABLE_AF,
INSTR_SELECTOR,
INSTR_LEARNER,
+ INSTR_LEARNER_AF,
- /* learn LEARNER ACTION_NAME */
+ /* learn LEARNER ACTION_NAME [ m.action_first_arg ] */
INSTR_LEARNER_LEARN,
/* forget */
* Return from action
*/
INSTR_RETURN,
+
+ /* Start of custom instructions. */
+ INSTR_CUSTOM_0,
};
struct instr_operand {
struct instr_learn {
uint8_t action_id;
+ uint8_t mf_offset;
};
struct instr_extern_obj {
int invalid;
};
+typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
+
/*
* Action.
*/
+typedef void
+(*action_func_t)(struct rte_swx_pipeline *p);
+
struct action {
TAILQ_ENTRY(action) node;
char name[RTE_SWX_NAME_SIZE];
struct struct_type *st;
int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
struct instruction *instructions;
+ struct instruction_data *instruction_data;
uint32_t n_instructions;
uint32_t id;
};
uint32_t n_actions;
int default_action_is_const;
uint32_t action_data_size_max;
+ int *action_is_for_table_entries;
+ int *action_is_for_default_entry;
uint32_t size;
uint32_t id;
/* Action. */
struct action **actions;
- struct field **action_arg;
struct action *default_action;
uint8_t *default_action_data;
uint32_t n_actions;
int default_action_is_const;
uint32_t action_data_size_max;
+ int *action_is_for_table_entries;
+ int *action_is_for_default_entry;
uint32_t size;
uint32_t timeout;
struct learner_runtime {
void *mailbox;
uint8_t **key;
- uint8_t **action_data;
};
struct learner_statistics {
#define HEADER_VALID(thread, header_id) \
MASK64_BIT_GET((thread)->valid_headers, header_id)
+static inline uint64_t
+instr_operand_hbo(struct thread *t, const struct instr_operand *x)
+{
+ uint8_t *x_struct = t->structs[x->struct_id];
+ uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
+ uint64_t x64 = *x64_ptr;
+ uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
+
+ return x64 & x64_mask;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_operand_nbo(struct thread *t, const struct instr_operand *x)
+{
+ uint8_t *x_struct = t->structs[x->struct_id];
+ uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
+ uint64_t x64 = *x64_ptr;
+
+ return ntoh64(x64) >> (64 - x->n_bits);
+}
+
+#else
+
+#define instr_operand_nbo instr_operand_hbo
+
+#endif
+
#define ALU(thread, ip, operator) \
{ \
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
#define RTE_SWX_PIPELINE_THREADS_MAX 16
#endif
+#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
+#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
+#endif
+
struct rte_swx_pipeline {
struct struct_type_tailq struct_types;
struct port_in_type_tailq port_in_types;
struct port_in_runtime *in;
struct port_out_runtime *out;
struct instruction **action_instructions;
+ action_func_t *action_funcs;
struct rte_swx_table_state *table_state;
struct table_statistics *table_stats;
struct selector_statistics *selector_stats;
struct regarray_runtime *regarray_runtime;
struct metarray_runtime *metarray_runtime;
struct instruction *instructions;
+ struct instruction_data *instruction_data;
+ instr_exec_t *instruction_table;
struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
+ void *lib;
uint32_t n_structs;
uint32_t n_ports_in;
return;
}
- /* Header encapsulation (optionally, with prior header decasulation). */
+ /* Header encapsulation (optionally, with prior header decapsulation). */
if ((t->n_headers_out == 2) &&
(h1->ptr + h1->n_bytes == t->ptr) &&
(h0->ptr == h0->ptr0)) {
port->pkt_tx(port->obj, pkt);
}
+static inline void
+__instr_drop_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip __rte_unused)
+{
+ uint64_t port_id = p->n_ports_out - 1;
+ struct port_out_runtime *port = &p->out[port_id];
+ struct rte_swx_pkt *pkt = &t->pkt;
+
+ TRACE("[Thread %2u]: drop 1 pkt\n",
+ p->thread_id);
+
+ /* Headers. */
+ emit_handler(t);
+
+ /* Packet. */
+ port->pkt_tx(port->obj, pkt);
+}
+
+/*
+ * extract.
+ */
+static inline void
+__instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip,
+ uint32_t n_extract)
+{
+ uint64_t valid_headers = t->valid_headers;
+ uint8_t *ptr = t->ptr;
+ uint32_t offset = t->pkt.offset;
+ uint32_t length = t->pkt.length;
+ uint32_t i;
+
+ for (i = 0; i < n_extract; i++) {
+ uint32_t header_id = ip->io.hdr.header_id[i];
+ uint32_t struct_id = ip->io.hdr.struct_id[i];
+ uint32_t n_bytes = ip->io.hdr.n_bytes[i];
+
+ TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
+ p->thread_id,
+ header_id,
+ n_bytes);
+
+ /* Headers. */
+ t->structs[struct_id] = ptr;
+ valid_headers = MASK64_BIT_SET(valid_headers, header_id);
+
+ /* Packet. */
+ offset += n_bytes;
+ length -= n_bytes;
+ ptr += n_bytes;
+ }
+
+ /* Headers. */
+ t->valid_headers = valid_headers;
+
+ /* Packet. */
+ t->pkt.offset = offset;
+ t->pkt.length = length;
+ t->ptr = ptr;
+}
+
+static inline void
+__instr_hdr_extract_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ __instr_hdr_extract_many_exec(p, t, ip, 1);
+}
+
+static inline void
+__instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 2);
+}
+
+static inline void
+__instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 3);
+}
+
+static inline void
+__instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 4);
+}
+
+static inline void
+__instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 5);
+}
+
+static inline void
+__instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 6);
+}
+
+static inline void
+__instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 7);
+}
+
+static inline void
+__instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_extract_many_exec(p, t, ip, 8);
+}
+
+static inline void
+__instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t valid_headers = t->valid_headers;
+ uint8_t *ptr = t->ptr;
+ uint32_t offset = t->pkt.offset;
+ uint32_t length = t->pkt.length;
+
+ uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
+ uint32_t header_id = ip->io.hdr.header_id[0];
+ uint32_t struct_id = ip->io.hdr.struct_id[0];
+ uint32_t n_bytes = ip->io.hdr.n_bytes[0];
+
+ struct header_runtime *h = &t->headers[header_id];
+
+ TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
+ p->thread_id,
+ header_id,
+ n_bytes,
+ n_bytes_last);
+
+ n_bytes += n_bytes_last;
+
+ /* Headers. */
+ t->structs[struct_id] = ptr;
+ t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
+ h->n_bytes = n_bytes;
+
+ /* Packet. */
+ t->pkt.offset = offset + n_bytes;
+ t->pkt.length = length - n_bytes;
+ t->ptr = ptr + n_bytes;
+}
+
+static inline void
+__instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t valid_headers = t->valid_headers;
+ uint8_t *ptr = t->ptr;
+
+ uint32_t header_id = ip->io.hdr.header_id[0];
+ uint32_t struct_id = ip->io.hdr.struct_id[0];
+
+ TRACE("[Thread %2u]: lookahead header %u\n",
+ p->thread_id,
+ header_id);
+
+ /* Headers. */
+ t->structs[struct_id] = ptr;
+ t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
+}
+
+/*
+ * emit.
+ */
+static inline void
+__instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip,
+ uint32_t n_emit)
+{
+ uint64_t valid_headers = t->valid_headers;
+ uint32_t n_headers_out = t->n_headers_out;
+ struct header_out_runtime *ho = NULL;
+ uint8_t *ho_ptr = NULL;
+ uint32_t ho_nbytes = 0, i;
+
+ for (i = 0; i < n_emit; i++) {
+ uint32_t header_id = ip->io.hdr.header_id[i];
+ uint32_t struct_id = ip->io.hdr.struct_id[i];
+
+ struct header_runtime *hi = &t->headers[header_id];
+ uint8_t *hi_ptr0 = hi->ptr0;
+ uint32_t n_bytes = hi->n_bytes;
+
+ uint8_t *hi_ptr = t->structs[struct_id];
+
+ if (!MASK64_BIT_GET(valid_headers, header_id)) {
+ TRACE("[Thread %2u]: emit header %u (invalid)\n",
+ p->thread_id,
+ header_id);
+
+ continue;
+ }
+
+ TRACE("[Thread %2u]: emit header %u (valid)\n",
+ p->thread_id,
+ header_id);
+
+ /* Headers. */
+ if (!ho) {
+ if (!n_headers_out) {
+ ho = &t->headers_out[0];
+
+ ho->ptr0 = hi_ptr0;
+ ho->ptr = hi_ptr;
+
+ ho_ptr = hi_ptr;
+ ho_nbytes = n_bytes;
+
+ n_headers_out = 1;
+
+ continue;
+ } else {
+ ho = &t->headers_out[n_headers_out - 1];
+
+ ho_ptr = ho->ptr;
+ ho_nbytes = ho->n_bytes;
+ }
+ }
+
+ if (ho_ptr + ho_nbytes == hi_ptr) {
+ ho_nbytes += n_bytes;
+ } else {
+ ho->n_bytes = ho_nbytes;
+
+ ho++;
+ ho->ptr0 = hi_ptr0;
+ ho->ptr = hi_ptr;
+
+ ho_ptr = hi_ptr;
+ ho_nbytes = n_bytes;
+
+ n_headers_out++;
+ }
+ }
+
+ if (ho)
+ ho->n_bytes = ho_nbytes;
+ t->n_headers_out = n_headers_out;
+}
+
+static inline void
+__instr_hdr_emit_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ __instr_hdr_emit_many_exec(p, t, ip, 1);
+}
+
+static inline void
+__instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 1);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 2);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 3);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 4);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 5);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 6);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 7);
+ __instr_tx_exec(p, t, ip);
+}
+
+static inline void
+__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
+
+ __instr_hdr_emit_many_exec(p, t, ip, 8);
+ __instr_tx_exec(p, t, ip);
+}
+
+/*
+ * validate.
+ */
+static inline void
+__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t header_id = ip->valid.header_id;
+
+ TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
+
+ /* Headers. */
+ t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
+}
+
+/*
+ * invalidate.
+ */
+static inline void
+__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t header_id = ip->valid.header_id;
+
+ TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
+
+ /* Headers. */
+ t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
+}
+
+/*
+ * learn.
+ */
+static inline void
+__instr_learn_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t action_id = ip->learn.action_id;
+ uint32_t mf_offset = ip->learn.mf_offset;
+ uint32_t learner_id = t->learner_id;
+ struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
+ p->n_selectors + learner_id];
+ struct learner_runtime *l = &t->learners[learner_id];
+ struct learner_statistics *stats = &p->learner_stats[learner_id];
+ uint32_t status;
+
+ /* Table. */
+ status = rte_swx_table_learner_add(ts->obj,
+ l->mailbox,
+ t->time,
+ action_id,
+ &t->metadata[mf_offset]);
+
+ TRACE("[Thread %2u] learner %u learn %s\n",
+ p->thread_id,
+ learner_id,
+ status ? "ok" : "error");
+
+ stats->n_pkts_learn[status] += 1;
+}
+
+/*
+ * forget.
+ */
+static inline void
+__instr_forget_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip __rte_unused)
+{
+ uint32_t learner_id = t->learner_id;
+ struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
+ p->n_selectors + learner_id];
+ struct learner_runtime *l = &t->learners[learner_id];
+ struct learner_statistics *stats = &p->learner_stats[learner_id];
+
+ /* Table. */
+ rte_swx_table_learner_delete(ts->obj, l->mailbox);
+
+ TRACE("[Thread %2u] learner %u forget\n",
+ p->thread_id,
+ learner_id);
+
+ stats->n_pkts_forget += 1;
+}
+
+/*
+ * extern.
+ */
+static inline uint32_t
+__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t obj_id = ip->ext_obj.ext_obj_id;
+ uint32_t func_id = ip->ext_obj.func_id;
+ struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
+ rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
+ uint32_t done;
+
+ TRACE("[Thread %2u] extern obj %u member func %u\n",
+ p->thread_id,
+ obj_id,
+ func_id);
+
+ done = func(obj->obj, obj->mailbox);
+
+ return done;
+}
+
+static inline uint32_t
+__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t ext_func_id = ip->ext_func.ext_func_id;
+ struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
+ rte_swx_extern_func_t func = ext_func->func;
+ uint32_t done;
+
+ TRACE("[Thread %2u] extern func %u\n",
+ p->thread_id,
+ ext_func_id);
+
+ done = func(ext_func->mailbox);
+
+ return done;
+}
+
+/*
+ * mov.
+ */
+static inline void
+__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov\n", p->thread_id);
+
+ MOV(t, ip);
+}
+
+static inline void
+__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
+
+ MOV_MH(t, ip);
+}
+
+static inline void
+__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
+
+ MOV_HM(t, ip);
+}
+
+static inline void
+__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
+
+ MOV_HH(t, ip);
+}
+
+static inline void
+__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
+
+ MOV_I(t, ip);
+}
+
+/*
+ * dma.
+ */
+static inline void
+__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip,
+ uint32_t n_dma)
+{
+ uint8_t *action_data = t->structs[0];
+ uint64_t valid_headers = t->valid_headers;
+ uint32_t i;
+
+ for (i = 0; i < n_dma; i++) {
+ uint32_t header_id = ip->dma.dst.header_id[i];
+ uint32_t struct_id = ip->dma.dst.struct_id[i];
+ uint32_t offset = ip->dma.src.offset[i];
+ uint32_t n_bytes = ip->dma.n_bytes[i];
+
+ struct header_runtime *h = &t->headers[header_id];
+ uint8_t *h_ptr0 = h->ptr0;
+ uint8_t *h_ptr = t->structs[struct_id];
+
+ void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
+ h_ptr : h_ptr0;
+ void *src = &action_data[offset];
+
+ TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
+
+ /* Headers. */
+ memcpy(dst, src, n_bytes);
+ t->structs[struct_id] = dst;
+ valid_headers = MASK64_BIT_SET(valid_headers, header_id);
+ }
+
+ t->valid_headers = valid_headers;
+}
+
+static inline void
+__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ __instr_dma_ht_many_exec(p, t, ip, 1);
+}
+
+static inline void
+__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 2);
+}
+
+static inline void
+__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 3);
+}
+
+static inline void
+__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 4);
+}
+
+static inline void
+__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 5);
+}
+
+static inline void
+__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 6);
+}
+
+static inline void
+__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 7);
+}
+
+static inline void
+__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
+
+ __instr_dma_ht_many_exec(p, t, ip, 8);
+}
+
+/*
+ * alu.
+ */
+static inline void
+__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add\n", p->thread_id);
+
+ ALU(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (mi)\n", p->thread_id);
+
+ ALU_MI(t, ip, +);
+}
+
+static inline void
+__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] add (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, +);
+}
+
+static inline void
+__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub\n", p->thread_id);
+
+ ALU(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
+
+ ALU_MI(t, ip, -);
+}
+
+static inline void
+__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, -);
+}
+
+static inline void
+__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl\n", p->thread_id);
+
+ ALU(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
+
+ ALU_MI(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, <<);
+}
+
+static inline void
+__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr\n", p->thread_id);
+
+ ALU(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
+
+ ALU_HM(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
+
+ ALU_HH(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MI(t, ip, >>);
+}
+
+static inline void
+__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
+
+ ALU_HI(t, ip, >>);
+}
+
+static inline void
+__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and\n", p->thread_id);
+
+ ALU(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (hm)\n", p->thread_id);
+
+ ALU_HM_FAST(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (hh)\n", p->thread_id);
+
+ ALU_HH_FAST(t, ip, &);
+}
+
+static inline void
+__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] and (i)\n", p->thread_id);
+
+ ALU_I(t, ip, &);
+}
+
+static inline void
+__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or\n", p->thread_id);
+
+ ALU(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (hm)\n", p->thread_id);
+
+ ALU_HM_FAST(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (hh)\n", p->thread_id);
+
+ ALU_HH_FAST(t, ip, |);
+}
+
+static inline void
+__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] or (i)\n", p->thread_id);
+
+ ALU_I(t, ip, |);
+}
+
+static inline void
+__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor\n", p->thread_id);
+
+ ALU(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
+
+ ALU_MH(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
+
+ ALU_HM_FAST(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
+
+ ALU_HH_FAST(t, ip, ^);
+}
+
+static inline void
+__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ TRACE("[Thread %2u] xor (i)\n", p->thread_id);
+
+ ALU_I(t, ip, ^);
+}
+
+static inline void
+__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ /* Initialize the result with destination 1's complement. */
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* The first input (r) is a 16-bit number. The second and the third
+ * inputs are 32-bit numbers. In the worst case scenario, the sum of the
+ * three numbers (output r) is a 34-bit number.
+ */
+ r += (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is an 18-bit
+ * number. In the worst case scenario, the sum of the two numbers is a
+ * 19-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
+ * therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* Apply 1's complement to the result. */
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+}
+
+static inline void
+__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ /* Initialize the result with destination 1's complement. */
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
+ * the following sequence of operations in 2's complement arithmetic:
+ * a '- b = (a - b) % 0xFFFF.
+ *
+ * In order to prevent an underflow for the below subtraction, in which
+ * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
+ * minuend), we first add a multiple of the 0xFFFF modulus to the
+ * minuend. The number we add to the minuend needs to be a 34-bit number
+ * or higher, so for readability reasons we picked the 36-bit multiple.
+ * We are effectively turning the 16-bit minuend into a 36-bit number:
+ * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
+ */
+ r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
+
+ /* A 33-bit number is subtracted from a 36-bit number (the input r). The
+ * result (the output r) is a 36-bit number.
+ */
+ r -= (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* Apply 1's complement to the result. */
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+}
+
+static inline void
+__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint32_t *src32_ptr;
+ uint64_t r0, r1;
+
+ TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ /* Initialize the result with destination 1's complement. */
+ r0 = dst;
+ r0 = ~r0 & 0xFFFF;
+
+ r0 += src32_ptr[0]; /* The output r0 is a 33-bit number. */
+ r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
+ r0 += src32_ptr[2]; /* The output r0 is a 34-bit number. */
+ r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
+ r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
+
+ /* The first input is a 16-bit number. The second input is a 19-bit
+ * number. Their sum is a 20-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1000E), the output r is (0 .. 15). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* Apply 1's complement to the result. */
+ r0 = ~r0 & 0xFFFF;
+ r0 = r0 ? r0 : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r0;
+}
+
+static inline void
+__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t src_header_id = ip->alu.src.n_bits; /* The src header ID is stored here. */
+ uint32_t n_src_header_bytes = t->headers[src_header_id].n_bytes;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint32_t *src32_ptr;
+ uint64_t r;
+ uint32_t i;
+
+ if (n_src_header_bytes == 20) {
+ __instr_alu_ckadd_struct20_exec(p, t, ip);
+ return;
+ }
+
+ TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ /* Initialize the result with destination 1's complement. */
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* The max number of 32-bit words in a 32K-byte header is 2^13.
+ * Therefore, in the worst case scenario, a 45-bit number is added to a
+ * 16-bit number (the input r), so the output r is 46-bit number.
+ */
+ for (i = 0; i < n_src_header_bytes / 4; i++, src32_ptr++)
+ r += *src32_ptr;
+
+ /* The first input is a 16-bit number. The second input is a 30-bit
+ * number. Their sum is a 31-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 15-bit number (0 .. 0x7FFF). The sum is a 17-bit number (0 .. 0x17FFE).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x17FFE), the output r is (0 .. 0x7FFF). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* Apply 1's complement to the result. */
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+}
+
+/*
+ * Register array.
+ */
+static inline uint64_t *
+instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+ return r->regarray;
+}
+
+static inline uint64_t
+instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
+ uint64_t idx = idx64 & idx64_mask & r->size_mask;
+
+ return idx;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
+
+ return idx;
+}
+
+#else
+
+#define instr_regarray_idx_nbo instr_regarray_idx_hbo
+
+#endif
+
+static inline uint64_t
+instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint64_t idx = ip->regarray.idx_val & r->size_mask;
+
+ return idx;
+}
+
+static inline uint64_t
+instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return src;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
+
+ return src;
+}
+
+#else
+
+#define instr_regarray_src_nbo instr_regarray_src_hbo
+
+#endif
+
+static inline void
+instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
+{
+ uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline void
+instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
+{
+ uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+
+ src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+}
+
+#else
+
+#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
+
+#endif
+
+static inline void
+__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ rte_prefetch0(®array[idx]);
+}
+
+static inline void
+__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ rte_prefetch0(®array[idx]);
+}
+
+static inline void
+__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ rte_prefetch0(®array[idx]);
+}
+
+static inline void
+__instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+}
+
+static inline void
+__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regwr_rii_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+}
+
+static inline void
+__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+}
+
+static inline void
+__instr_regadd_rii_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
+
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+}
+
+/*
+ * metarray.
+ */
+static inline struct meter *
+instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
+ uint64_t idx = idx64 & idx64_mask & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline struct meter *
+instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+#else
+
+#define instr_meter_idx_nbo instr_meter_idx_hbo
+
+#endif
+
+static inline struct meter *
+instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint64_t idx = ip->meter.idx_val & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+static inline uint32_t
+instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return (uint32_t)src;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint32_t
+instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
+
+ return (uint32_t)src;
+}
+
+#else
+
+#define instr_meter_length_nbo instr_meter_length_hbo
+
+#endif
+
+static inline enum rte_color
+instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return (enum rte_color)src;
+}
+
+static inline void
+instr_meter_color_out_hbo_set(struct thread *t,
+ const struct instruction *ip,
+ enum rte_color color_out)
+{
+ uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
+
+ uint64_t src = (uint64_t)color_out;
+
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+}
+
+static inline void
+__instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m);
+}
+
+static inline void
+__instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m);
+}
+
+static inline void
+__instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
+ struct thread *t __rte_unused,
+ const struct instruction *ip)
+{
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m);
+}
+
+static inline void
+__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
+
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
+
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
+static inline void
+__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
+{
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
+
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+}
+
#endif
git.droids-corp.org / dpdk.git / blobdiff