+instr_alu_sub_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HH(t, ip, -);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_sub_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MI(t, ip, -);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_sub_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HI(t, ip, -);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shl_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shl\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, <<);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MH(t, ip, <<);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HM(t, ip, <<);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HH(t, ip, <<);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MI(t, ip, <<);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HI(t, ip, <<);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MH(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HM(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HH(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MI(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HI(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_and_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] and\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, &);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_and_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] and (s)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_S(t, ip, &);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_and_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] and (i)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_I(t, ip, &);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_or_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] or\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, |);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_or_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] or (s)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_S(t, ip, |);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_or_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] or (i)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_I(t, ip, |);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_xor_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] xor\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, ^);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] xor (s)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_S(t, ip, ^);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] xor (i)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_I(t, ip, ^);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* The first input (r) is a 16-bit number. The second and the third
+ * inputs are 32-bit numbers. In the worst case scenario, the sum of the
+ * three numbers (output r) is a 34-bit number.
+ */
+ r += (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is an 18-bit
+ * number. In the worst case scenario, the sum of the two numbers is a
+ * 19-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
+ * therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
+ * the following sequence of operations in 2's complement arithmetic:
+ * a '- b = (a - b) % 0xFFFF.
+ *
+ * In order to prevent an underflow for the below subtraction, in which
+ * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
+ * minuend), we first add a multiple of the 0xFFFF modulus to the
+ * minuend. The number we add to the minuend needs to be a 34-bit number
+ * or higher, so for readability reasons we picked the 36-bit multiple.
+ * We are effectively turning the 16-bit minuend into a 36-bit number:
+ * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
+ */
+ r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
+
+ /* A 33-bit number is subtracted from a 36-bit number (the input r). The
+ * result (the output r) is a 36-bit number.
+ */
+ r -= (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr;
+ uint32_t *src32_ptr;
+ uint64_t r0, r1;
+
+ TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
+ r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
+ r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
+ r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
+ r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
+
+ /* The first input is a 16-bit number. The second input is a 19-bit
+ * number. Their sum is a 20-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1000E), the output r is (0 .. 15). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ r0 = ~r0 & 0xFFFF;
+ r0 = r0 ? r0 : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r0;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr;
+ uint32_t *src32_ptr;
+ uint64_t r = 0;
+ uint32_t i;
+
+ TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
+ * Therefore, in the worst case scenario, a 35-bit number is added to a
+ * 16-bit number (the input r), so the output r is 36-bit number.
+ */
+ for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
+ r += *src32_ptr;
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+/*
+ * Register array.
+ */
+static struct regarray *
+regarray_find(struct rte_swx_pipeline *p, const char *name);
+
+static int
+instr_regprefetch_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *regarray = tokens[1], *idx = tokens[2];
+ struct regarray *r;
+ struct field *fidx;
+ uint32_t idx_struct_id, idx_val;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ r = regarray_find(p, regarray);
+ CHECK(r, EINVAL);
+
+ /* REGPREFETCH_RH, REGPREFETCH_RM. */
+ fidx = struct_field_parse(p, action, idx, &idx_struct_id);
+ if (fidx) {
+ instr->type = INSTR_REGPREFETCH_RM;
+ if (idx[0] == 'h')
+ instr->type = INSTR_REGPREFETCH_RH;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->regarray.idx.n_bits = fidx->n_bits;
+ instr->regarray.idx.offset = fidx->offset / 8;
+ instr->regarray.dstsrc_val = 0; /* Unused. */
+ return 0;
+ }
+
+ /* REGPREFETCH_RI. */
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_REGPREFETCH_RI;
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx_val = idx_val;
+ instr->regarray.dstsrc_val = 0; /* Unused. */
+ return 0;
+}
+
+static int
+instr_regrd_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3];
+ struct regarray *r;
+ struct field *fdst, *fidx;
+ uint32_t dst_struct_id, idx_struct_id, idx_val;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ r = regarray_find(p, regarray);
+ CHECK(r, EINVAL);
+
+ fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
+ CHECK(fdst, EINVAL);
+
+ /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */
+ fidx = struct_field_parse(p, action, idx, &idx_struct_id);
+ if (fidx) {
+ instr->type = INSTR_REGRD_MRM;
+ if (dst[0] == 'h' && idx[0] != 'h')
+ instr->type = INSTR_REGRD_HRM;
+ if (dst[0] != 'h' && idx[0] == 'h')
+ instr->type = INSTR_REGRD_MRH;
+ if (dst[0] == 'h' && idx[0] == 'h')
+ instr->type = INSTR_REGRD_HRH;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->regarray.idx.n_bits = fidx->n_bits;
+ instr->regarray.idx.offset = fidx->offset / 8;
+ instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
+ instr->regarray.dstsrc.n_bits = fdst->n_bits;
+ instr->regarray.dstsrc.offset = fdst->offset / 8;
+ return 0;
+ }
+
+ /* REGRD_MRI, REGRD_HRI. */
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_REGRD_MRI;
+ if (dst[0] == 'h')
+ instr->type = INSTR_REGRD_HRI;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx_val = idx_val;
+ instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id;
+ instr->regarray.dstsrc.n_bits = fdst->n_bits;
+ instr->regarray.dstsrc.offset = fdst->offset / 8;
+ return 0;
+}
+
+static int
+instr_regwr_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
+ struct regarray *r;
+ struct field *fidx, *fsrc;
+ uint64_t src_val;
+ uint32_t idx_struct_id, idx_val, src_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ r = regarray_find(p, regarray);
+ CHECK(r, EINVAL);
+
+ /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */
+ fidx = struct_field_parse(p, action, idx, &idx_struct_id);
+ fsrc = struct_field_parse(p, action, src, &src_struct_id);
+ if (fidx && fsrc) {
+ instr->type = INSTR_REGWR_RMM;
+ if (idx[0] == 'h' && src[0] != 'h')
+ instr->type = INSTR_REGWR_RHM;
+ if (idx[0] != 'h' && src[0] == 'h')
+ instr->type = INSTR_REGWR_RMH;
+ if (idx[0] == 'h' && src[0] == 'h')
+ instr->type = INSTR_REGWR_RHH;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->regarray.idx.n_bits = fidx->n_bits;
+ instr->regarray.idx.offset = fidx->offset / 8;
+ instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
+ instr->regarray.dstsrc.n_bits = fsrc->n_bits;
+ instr->regarray.dstsrc.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* REGWR_RHI, REGWR_RMI. */
+ if (fidx && !fsrc) {
+ src_val = strtoull(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ instr->type = INSTR_REGWR_RMI;
+ if (idx[0] == 'h')
+ instr->type = INSTR_REGWR_RHI;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->regarray.idx.n_bits = fidx->n_bits;
+ instr->regarray.idx.offset = fidx->offset / 8;
+ instr->regarray.dstsrc_val = src_val;
+ return 0;
+ }
+
+ /* REGWR_RIH, REGWR_RIM. */
+ if (!fidx && fsrc) {
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_REGWR_RIM;
+ if (src[0] == 'h')
+ instr->type = INSTR_REGWR_RIH;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx_val = idx_val;
+ instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
+ instr->regarray.dstsrc.n_bits = fsrc->n_bits;
+ instr->regarray.dstsrc.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* REGWR_RII. */
+ src_val = strtoull(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_REGWR_RII;
+ instr->regarray.idx_val = idx_val;
+ instr->regarray.dstsrc_val = src_val;
+
+ return 0;
+}
+
+static int
+instr_regadd_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3];
+ struct regarray *r;
+ struct field *fidx, *fsrc;
+ uint64_t src_val;
+ uint32_t idx_struct_id, idx_val, src_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ r = regarray_find(p, regarray);
+ CHECK(r, EINVAL);
+
+ /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */
+ fidx = struct_field_parse(p, action, idx, &idx_struct_id);
+ fsrc = struct_field_parse(p, action, src, &src_struct_id);
+ if (fidx && fsrc) {
+ instr->type = INSTR_REGADD_RMM;
+ if (idx[0] == 'h' && src[0] != 'h')
+ instr->type = INSTR_REGADD_RHM;
+ if (idx[0] != 'h' && src[0] == 'h')
+ instr->type = INSTR_REGADD_RMH;
+ if (idx[0] == 'h' && src[0] == 'h')
+ instr->type = INSTR_REGADD_RHH;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->regarray.idx.n_bits = fidx->n_bits;
+ instr->regarray.idx.offset = fidx->offset / 8;
+ instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
+ instr->regarray.dstsrc.n_bits = fsrc->n_bits;
+ instr->regarray.dstsrc.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* REGADD_RHI, REGADD_RMI. */
+ if (fidx && !fsrc) {
+ src_val = strtoull(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ instr->type = INSTR_REGADD_RMI;
+ if (idx[0] == 'h')
+ instr->type = INSTR_REGADD_RHI;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->regarray.idx.n_bits = fidx->n_bits;
+ instr->regarray.idx.offset = fidx->offset / 8;
+ instr->regarray.dstsrc_val = src_val;
+ return 0;
+ }
+
+ /* REGADD_RIH, REGADD_RIM. */
+ if (!fidx && fsrc) {
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_REGADD_RIM;
+ if (src[0] == 'h')
+ instr->type = INSTR_REGADD_RIH;
+
+ instr->regarray.regarray_id = r->id;
+ instr->regarray.idx_val = idx_val;
+ instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id;
+ instr->regarray.dstsrc.n_bits = fsrc->n_bits;
+ instr->regarray.dstsrc.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* REGADD_RII. */
+ src_val = strtoull(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_REGADD_RII;
+ instr->regarray.idx_val = idx_val;
+ instr->regarray.dstsrc_val = src_val;
+ return 0;
+}
+
+static inline uint64_t *
+instr_regarray_regarray(struct rte_swx_pipeline *p, struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+ return r->regarray;
+}
+
+static inline uint64_t
+instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
+ uint64_t idx = idx64 & idx64_mask & r->size_mask;
+
+ return idx;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
+
+ return idx;
+}
+
+#else
+
+#define instr_regarray_idx_nbo instr_regarray_idx_hbo
+
+#endif
+
+static inline uint64_t
+instr_regarray_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
+{
+ struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
+
+ uint64_t idx = ip->regarray.idx_val & r->size_mask;
+
+ return idx;
+}
+
+static inline uint64_t
+instr_regarray_src_hbo(struct thread *t, struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return src;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint64_t
+instr_regarray_src_nbo(struct thread *t, struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
+
+ return src;
+}
+
+#else
+
+#define instr_regarray_src_nbo instr_regarray_src_hbo
+
+#endif
+
+static inline void
+instr_regarray_dst_hbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
+{
+ uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline void
+instr_regarray_dst_nbo_src_hbo_set(struct thread *t, struct instruction *ip, uint64_t src)
+{
+ uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
+
+ src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+}
+
+#else
+
+#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
+
+#endif
+
+static inline void
+instr_regprefetch_rh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ rte_prefetch0(®array[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regprefetch_rm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ rte_prefetch0(®array[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regprefetch_ri_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ rte_prefetch0(®array[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regrd_hrh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regrd_hrm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regrd_mrh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regrd_mrm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regrd_hri_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regrd_mri_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx;
+
+ TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rhh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rhm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rmh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rmm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rhi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rmi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rih_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rim_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regwr_rii_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] = src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rhh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rhm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rmh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rmm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rhi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_nbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rmi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_hbo(p, t, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rih_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_nbo(t, ip);
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rim_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = instr_regarray_src_hbo(t, ip);
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_regadd_rii_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t *regarray, idx, src;
+
+ TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
+
+ /* Structs. */
+ regarray = instr_regarray_regarray(p, ip);
+ idx = instr_regarray_idx_imm(p, ip);
+ src = ip->regarray.dstsrc_val;
+ regarray[idx] += src;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+/*
+ * jmp.
+ */
+static struct action *
+action_find(struct rte_swx_pipeline *p, const char *name);
+
+static int
+instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_valid_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct header *h;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ h = header_parse(p, tokens[2]);
+ CHECK(h, EINVAL);
+
+ instr->type = INSTR_JMP_VALID;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.header_id = h->id;
+ return 0;
+}
+
+static int
+instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct header *h;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ h = header_parse(p, tokens[2]);
+ CHECK(h, EINVAL);
+
+ instr->type = INSTR_JMP_INVALID;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.header_id = h->id;
+ return 0;
+}
+
+static int
+instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP_HIT;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP_MISS;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct action *a;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ a = action_find(p, tokens[2]);
+ CHECK(a, EINVAL);
+
+ instr->type = INSTR_JMP_ACTION_HIT;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.action_id = a->id;
+ return 0;
+}
+
+static int
+instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct action *a;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ a = action_find(p, tokens[2]);
+ CHECK(a, EINVAL);
+
+ instr->type = INSTR_JMP_ACTION_MISS;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.action_id = a->id;
+ return 0;
+}
+
+static int
+instr_jmp_eq_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint64_t b_val;
+ uint32_t a_struct_id, b_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_EQ or JMP_EQ_S. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_EQ;
+ if ((a[0] == 'h' && b[0] != 'h') ||
+ (a[0] != 'h' && b[0] == 'h'))
+ instr->type = INSTR_JMP_EQ_S;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_EQ_I. */
+ b_val = strtoull(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ if (a[0] == 'h')
+ b_val = hton64(b_val) >> (64 - fa->n_bits);
+
+ instr->type = INSTR_JMP_EQ_I;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = b_val;
+ return 0;
+}
+
+static int
+instr_jmp_neq_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint64_t b_val;
+ uint32_t a_struct_id, b_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_NEQ or JMP_NEQ_S. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_NEQ;
+ if ((a[0] == 'h' && b[0] != 'h') ||
+ (a[0] != 'h' && b[0] == 'h'))
+ instr->type = INSTR_JMP_NEQ_S;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_NEQ_I. */
+ b_val = strtoull(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ if (a[0] == 'h')
+ b_val = hton64(b_val) >> (64 - fa->n_bits);
+
+ instr->type = INSTR_JMP_NEQ_I;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = b_val;
+ return 0;
+}
+
+static int
+instr_jmp_lt_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint64_t b_val;
+ uint32_t a_struct_id, b_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_LT;
+ if (a[0] == 'h' && b[0] != 'h')
+ instr->type = INSTR_JMP_LT_HM;
+ if (a[0] != 'h' && b[0] == 'h')
+ instr->type = INSTR_JMP_LT_MH;
+ if (a[0] == 'h' && b[0] == 'h')
+ instr->type = INSTR_JMP_LT_HH;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_LT_MI, JMP_LT_HI. */
+ b_val = strtoull(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ instr->type = INSTR_JMP_LT_MI;
+ if (a[0] == 'h')
+ instr->type = INSTR_JMP_LT_HI;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = b_val;
+ return 0;
+}
+
+static int
+instr_jmp_gt_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint64_t b_val;
+ uint32_t a_struct_id, b_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_GT;
+ if (a[0] == 'h' && b[0] != 'h')
+ instr->type = INSTR_JMP_GT_HM;
+ if (a[0] != 'h' && b[0] == 'h')
+ instr->type = INSTR_JMP_GT_MH;
+ if (a[0] == 'h' && b[0] == 'h')
+ instr->type = INSTR_JMP_GT_HH;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_GT_MI, JMP_GT_HI. */
+ b_val = strtoull(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ instr->type = INSTR_JMP_GT_MI;
+ if (a[0] == 'h')
+ instr->type = INSTR_JMP_GT_HI;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = b_val;
+ return 0;
+}
+
+static inline void
+instr_jmp_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmp\n", p->thread_id);
+
+ thread_ip_set(t, ip->jmp.ip);
+}
+
+static inline void
+instr_jmp_valid_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t header_id = ip->jmp.header_id;
+
+ TRACE("[Thread %2u] jmpv\n", p->thread_id);
+
+ t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
+}
+
+static inline void
+instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t header_id = ip->jmp.header_id;
+
+ TRACE("[Thread %2u] jmpnv\n", p->thread_id);
+
+ t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
+}
+
+static inline void
+instr_jmp_hit_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
+
+ TRACE("[Thread %2u] jmph\n", p->thread_id);
+
+ t->ip = ip_next[t->hit];
+}
+
+static inline void
+instr_jmp_miss_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
+
+ TRACE("[Thread %2u] jmpnh\n", p->thread_id);
+
+ t->ip = ip_next[t->hit];
+}
+
+static inline void
+instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpa\n", p->thread_id);
+
+ t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
+}
+
+static inline void
+instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpna\n", p->thread_id);
+
+ t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
+}
+
+static inline void
+instr_jmp_eq_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpeq\n", p->thread_id);
+
+ JMP_CMP(t, ip, ==);
+}
+
+static inline void
+instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
+
+ JMP_CMP_S(t, ip, ==);
+}
+
+static inline void
+instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
+
+ JMP_CMP_I(t, ip, ==);
+}
+
+static inline void
+instr_jmp_neq_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpneq\n", p->thread_id);
+
+ JMP_CMP(t, ip, !=);
+}
+
+static inline void
+instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
+
+ JMP_CMP_S(t, ip, !=);
+}
+
+static inline void
+instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
+
+ JMP_CMP_I(t, ip, !=);
+}
+
+static inline void
+instr_jmp_lt_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt\n", p->thread_id);
+
+ JMP_CMP(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
+
+ JMP_CMP_MH(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
+
+ JMP_CMP_HM(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
+
+ JMP_CMP_HH(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
+
+ JMP_CMP_MI(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
+
+ JMP_CMP_HI(t, ip, <);
+}
+
+static inline void
+instr_jmp_gt_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt\n", p->thread_id);
+
+ JMP_CMP(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
+
+ JMP_CMP_MH(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
+
+ JMP_CMP_HM(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)