INSTR_ALU_AND, /* dst = MEF, src = MEFT */
INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
INSTR_ALU_AND_I, /* dst = HMEF, src = I */
+
+ /* or dst src
+ * dst |= src
+ * dst = HMEF, src = HMEFTI
+ */
+ INSTR_ALU_OR, /* dst = MEF, src = MEFT */
+ INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
+ INSTR_ALU_OR_I, /* dst = HMEF, src = I */
+
+ /* xor dst src
+ * dst ^= src
+ * dst = HMEF, src = HMEFTI
+ */
+ INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
+ INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */
+ INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
+
+ /* shl dst src
+ * dst <<= src
+ * dst = HMEF, src = HMEFTI
+ */
+ INSTR_ALU_SHL, /* dst = MEF, src = MEF */
+ INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
+ INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
+ INSTR_ALU_SHL_HH, /* dst = H, src = H */
+ INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
+ INSTR_ALU_SHL_HI, /* dst = H, src = I */
+
+ /* shr dst src
+ * dst >>= src
+ * dst = HMEF, src = HMEFTI
+ */
+ INSTR_ALU_SHR, /* dst = MEF, src = MEF */
+ INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
+ INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
+ INSTR_ALU_SHR_HH, /* dst = H, src = H */
+ INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
+ INSTR_ALU_SHR_HI, /* dst = H, src = I */
+
+ /* table TABLE */
+ INSTR_TABLE,
+
+ /* extern e.obj.func */
+ INSTR_EXTERN_OBJ,
+
+ /* extern f.func */
+ INSTR_EXTERN_FUNC,
+
+ /* jmp LABEL
+ * Unconditional jump
+ */
+ INSTR_JMP,
+
+ /* jmpv LABEL h.header
+ * Jump if header is valid
+ */
+ INSTR_JMP_VALID,
+
+ /* jmpnv LABEL h.header
+ * Jump if header is invalid
+ */
+ INSTR_JMP_INVALID,
+
+ /* jmph LABEL
+ * Jump if table lookup hit
+ */
+ INSTR_JMP_HIT,
+
+ /* jmpnh LABEL
+ * Jump if table lookup miss
+ */
+ INSTR_JMP_MISS,
+
+ /* jmpa LABEL ACTION
+ * Jump if action run
+ */
+ INSTR_JMP_ACTION_HIT,
+
+ /* jmpna LABEL ACTION
+ * Jump if action not run
+ */
+ INSTR_JMP_ACTION_MISS,
+
+ /* jmpeq LABEL a b
+ * Jump is a is equal to b
+ * a = HMEFT, b = HMEFTI
+ */
+ INSTR_JMP_EQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
+ INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
+ INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
+
+ /* jmpneq LABEL a b
+ * Jump is a is not equal to b
+ * a = HMEFT, b = HMEFTI
+ */
+ INSTR_JMP_NEQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */
+ INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */
+ INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
+
+ /* jmplt LABEL a b
+ * Jump if a is less than b
+ * a = HMEFT, b = HMEFTI
+ */
+ INSTR_JMP_LT, /* a = MEF, b = MEF */
+ INSTR_JMP_LT_MH, /* a = MEF, b = H */
+ INSTR_JMP_LT_HM, /* a = H, b = MEF */
+ INSTR_JMP_LT_HH, /* a = H, b = H */
+ INSTR_JMP_LT_MI, /* a = MEF, b = I */
+ INSTR_JMP_LT_HI, /* a = H, b = I */
+
+ /* jmpgt LABEL a b
+ * Jump if a is greater than b
+ * a = HMEFT, b = HMEFTI
+ */
+ INSTR_JMP_GT, /* a = MEF, b = MEF */
+ INSTR_JMP_GT_MH, /* a = MEF, b = H */
+ INSTR_JMP_GT_HM, /* a = H, b = MEF */
+ INSTR_JMP_GT_HH, /* a = H, b = H */
+ INSTR_JMP_GT_MI, /* a = MEF, b = I */
+ INSTR_JMP_GT_HI, /* a = H, b = I */
+
+ /* return
+ * Return from action
+ */
+ INSTR_RETURN,
};
struct instr_operand {
uint8_t header_id;
};
+struct instr_table {
+ uint8_t table_id;
+};
+
+struct instr_extern_obj {
+ uint8_t ext_obj_id;
+ uint8_t func_id;
+};
+
+struct instr_extern_func {
+ uint8_t ext_func_id;
+};
+
struct instr_dst_src {
struct instr_operand dst;
union {
uint16_t n_bytes[8];
};
+struct instr_jmp {
+ struct instruction *ip;
+
+ union {
+ struct instr_operand a;
+ uint8_t header_id;
+ uint8_t action_id;
+ };
+
+ union {
+ struct instr_operand b;
+ uint32_t b_val;
+ };
+};
+
struct instruction {
enum instruction_type type;
union {
struct instr_dst_src mov;
struct instr_dma dma;
struct instr_dst_src alu;
+ struct instr_table table;
+ struct instr_extern_obj ext_obj;
+ struct instr_extern_func ext_func;
+ struct instr_jmp jmp;
};
};
#define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
#define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
+#define HEADER_VALID(thread, header_id) \
+ MASK64_BIT_GET((thread)->valid_headers, header_id)
+
#define ALU(thread, ip, operator) \
{ \
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
}
+#define JMP_CMP(thread, ip, operator) \
+{ \
+ uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
+ uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
+ uint64_t a64 = *a64_ptr; \
+ uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
+ uint64_t a = a64 & a64_mask; \
+ \
+ uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
+ uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
+ uint64_t b64 = *b64_ptr; \
+ uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
+ uint64_t b = b64 & b64_mask; \
+ \
+ (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+#define JMP_CMP_S(thread, ip, operator) \
+{ \
+ uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
+ uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
+ uint64_t a64 = *a64_ptr; \
+ uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
+ uint64_t a = a64 & a64_mask; \
+ \
+ uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
+ uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
+ uint64_t b64 = *b64_ptr; \
+ uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
+ \
+ (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
+}
+
+#define JMP_CMP_MH JMP_CMP_S
+
+#define JMP_CMP_HM(thread, ip, operator) \
+{ \
+ uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
+ uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
+ uint64_t a64 = *a64_ptr; \
+ uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
+ \
+ uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
+ uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
+ uint64_t b64 = *b64_ptr; \
+ uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
+ uint64_t b = b64 & b64_mask; \
+ \
+ (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
+}
+
+#define JMP_CMP_HH(thread, ip, operator) \
+{ \
+ uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
+ uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
+ uint64_t a64 = *a64_ptr; \
+ uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
+ \
+ uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
+ uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
+ uint64_t b64 = *b64_ptr; \
+ uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
+ \
+ (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
+}
+
+#else
+
+#define JMP_CMP_S JMP_CMP
+#define JMP_CMP_MH JMP_CMP
+#define JMP_CMP_HM JMP_CMP
+#define JMP_CMP_HH JMP_CMP
+
+#endif
+
+#define JMP_CMP_I(thread, ip, operator) \
+{ \
+ uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
+ uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
+ uint64_t a64 = *a64_ptr; \
+ uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
+ uint64_t a = a64 & a64_mask; \
+ \
+ uint64_t b = (ip)->jmp.b_val; \
+ \
+ (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
+}
+
+#define JMP_CMP_MI JMP_CMP_I
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+#define JMP_CMP_HI(thread, ip, operator) \
+{ \
+ uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
+ uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
+ uint64_t a64 = *a64_ptr; \
+ uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
+ \
+ uint64_t b = (ip)->jmp.b_val; \
+ \
+ (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
+}
+
+#else
+
+#define JMP_CMP_HI JMP_CMP_I
+
+#endif
+
#define METADATA_READ(thread, offset, n_bits) \
({ \
uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
return NULL;
}
+static struct extern_type_member_func *
+extern_obj_member_func_parse(struct rte_swx_pipeline *p,
+ const char *name,
+ struct extern_obj **obj)
+{
+ struct extern_obj *object;
+ struct extern_type_member_func *func;
+ char *object_name, *func_name;
+
+ if (name[0] != 'e' || name[1] != '.')
+ return NULL;
+
+ object_name = strdup(&name[2]);
+ if (!object_name)
+ return NULL;
+
+ func_name = strchr(object_name, '.');
+ if (!func_name) {
+ free(object_name);
+ return NULL;
+ }
+
+ *func_name = 0;
+ func_name++;
+
+ object = extern_obj_find(p, object_name);
+ if (!object) {
+ free(object_name);
+ return NULL;
+ }
+
+ func = extern_type_member_func_find(object->type, func_name);
+ if (!func) {
+ free(object_name);
+ return NULL;
+ }
+
+ if (obj)
+ *obj = object;
+
+ free(object_name);
+ return func;
+}
+
static struct field *
extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p,
const char *name,
return NULL;
}
+static struct extern_func *
+extern_func_parse(struct rte_swx_pipeline *p,
+ const char *name)
+{
+ if (name[0] != 'f' || name[1] != '.')
+ return NULL;
+
+ return extern_func_find(p, &name[2]);
+}
+
static struct field *
extern_func_mailbox_field_parse(struct rte_swx_pipeline *p,
const char *name,
/*
* Instruction.
*/
+static int
+instruction_is_jmp(struct instruction *instr)
+{
+ switch (instr->type) {
+ case INSTR_JMP:
+ case INSTR_JMP_VALID:
+ case INSTR_JMP_INVALID:
+ case INSTR_JMP_HIT:
+ case INSTR_JMP_MISS:
+ case INSTR_JMP_ACTION_HIT:
+ case INSTR_JMP_ACTION_MISS:
+ case INSTR_JMP_EQ:
+ case INSTR_JMP_EQ_S:
+ case INSTR_JMP_EQ_I:
+ case INSTR_JMP_NEQ:
+ case INSTR_JMP_NEQ_S:
+ case INSTR_JMP_NEQ_I:
+ case INSTR_JMP_LT:
+ case INSTR_JMP_LT_MH:
+ case INSTR_JMP_LT_HM:
+ case INSTR_JMP_LT_HH:
+ case INSTR_JMP_LT_MI:
+ case INSTR_JMP_LT_HI:
+ case INSTR_JMP_GT:
+ case INSTR_JMP_GT_MH:
+ case INSTR_JMP_GT_HM:
+ case INSTR_JMP_GT_HH:
+ case INSTR_JMP_GT_MI:
+ case INSTR_JMP_GT_HI:
+ return 1;
+
+ default:
+ return 0;
+ }
+}
+
static struct field *
action_field_parse(struct action *action, const char *name);
t->ip = p->instructions;
}
+static inline void
+thread_ip_set(struct thread *t, struct instruction *ip)
+{
+ t->ip = ip;
+}
+
+static inline void
+thread_ip_action_call(struct rte_swx_pipeline *p,
+ struct thread *t,
+ uint32_t action_id)
+{
+ t->ret = t->ip + 1;
+ t->ip = p->action_instructions[action_id];
+}
+
static inline void
thread_ip_inc(struct rte_swx_pipeline *p);
p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
}
+static inline void
+thread_yield_cond(struct rte_swx_pipeline *p, int cond)
+{
+ p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
+}
+
/*
* rx.
*/
thread_ip_inc(p);
}
+/*
+ * table.
+ */
+static struct table *
+table_find(struct rte_swx_pipeline *p, const char *name);
+
+static int
+instr_table_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ struct table *t;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ t = table_find(p, tokens[1]);
+ CHECK(t, EINVAL);
+
+ instr->type = INSTR_TABLE;
+ instr->table.table_id = t->id;
+ return 0;
+}
+
+static inline void
+instr_table_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t table_id = ip->table.table_id;
+ struct rte_swx_table_state *ts = &t->table_state[table_id];
+ struct table_runtime *table = &t->tables[table_id];
+ uint64_t action_id;
+ uint8_t *action_data;
+ int done, hit;
+
+ /* Table. */
+ done = table->func(ts->obj,
+ table->mailbox,
+ table->key,
+ &action_id,
+ &action_data,
+ &hit);
+ if (!done) {
+ /* Thread. */
+ TRACE("[Thread %2u] table %u (not finalized)\n",
+ p->thread_id,
+ table_id);
+
+ thread_yield(p);
+ return;
+ }
+
+ action_id = hit ? action_id : ts->default_action_id;
+ action_data = hit ? action_data : ts->default_action_data;
+
+ TRACE("[Thread %2u] table %u (%s, action %u)\n",
+ p->thread_id,
+ table_id,
+ hit ? "hit" : "miss",
+ (uint32_t)action_id);
+
+ t->action_id = action_id;
+ t->structs[0] = action_data;
+ t->hit = hit;
+
+ /* Thread. */
+ thread_ip_action_call(p, t, action_id);
+}
+
+/*
+ * extern.
+ */
+static int
+instr_extern_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *token = tokens[1];
+
+ CHECK(n_tokens == 2, EINVAL);
+
+ if (token[0] == 'e') {
+ struct extern_obj *obj;
+ struct extern_type_member_func *func;
+
+ func = extern_obj_member_func_parse(p, token, &obj);
+ CHECK(func, EINVAL);
+
+ instr->type = INSTR_EXTERN_OBJ;
+ instr->ext_obj.ext_obj_id = obj->id;
+ instr->ext_obj.func_id = func->id;
+
+ return 0;
+ }
+
+ if (token[0] == 'f') {
+ struct extern_func *func;
+
+ func = extern_func_parse(p, token);
+ CHECK(func, EINVAL);
+
+ instr->type = INSTR_EXTERN_FUNC;
+ instr->ext_func.ext_func_id = func->id;
+
+ return 0;
+ }
+
+ CHECK(0, EINVAL);
+}
+
+static inline void
+instr_extern_obj_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t obj_id = ip->ext_obj.ext_obj_id;
+ uint32_t func_id = ip->ext_obj.func_id;
+ struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
+ rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
+
+ TRACE("[Thread %2u] extern obj %u member func %u\n",
+ p->thread_id,
+ obj_id,
+ func_id);
+
+ /* Extern object member function execute. */
+ uint32_t done = func(obj->obj, obj->mailbox);
+
+ /* Thread. */
+ thread_ip_inc_cond(t, done);
+ thread_yield_cond(p, done ^ 1);
+}
+
+static inline void
+instr_extern_func_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t ext_func_id = ip->ext_func.ext_func_id;
+ struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
+ rte_swx_extern_func_t func = ext_func->func;
+
+ TRACE("[Thread %2u] extern func %u\n",
+ p->thread_id,
+ ext_func_id);
+
+ /* Extern function execute. */
+ uint32_t done = func(ext_func->mailbox);
+
+ /* Thread. */
+ thread_ip_inc_cond(t, done);
+ thread_yield_cond(p, done ^ 1);
+}
+
/*
* mov.
*/
}
static int
-instr_alu_and_translate(struct rte_swx_pipeline *p,
+instr_alu_shl_translate(struct rte_swx_pipeline *p,
struct action *action,
char **tokens,
int n_tokens,
fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
CHECK(fdst, EINVAL);
- /* AND or AND_S. */
+ /* SHL, SHL_HM, SHL_MH, SHL_HH. */
fsrc = struct_field_parse(p, action, src, &src_struct_id);
if (fsrc) {
- instr->type = INSTR_ALU_AND;
- if ((dst[0] == 'h' && src[0] != 'h') ||
- (dst[0] != 'h' && src[0] == 'h'))
- instr->type = INSTR_ALU_AND_S;
+ instr->type = INSTR_ALU_SHL;
+ if (dst[0] == 'h' && src[0] == 'm')
+ instr->type = INSTR_ALU_SHL_HM;
+ if (dst[0] == 'm' && src[0] == 'h')
+ instr->type = INSTR_ALU_SHL_MH;
+ if (dst[0] == 'h' && src[0] == 'h')
+ instr->type = INSTR_ALU_SHL_HH;
instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
instr->alu.dst.n_bits = fdst->n_bits;
return 0;
}
- /* AND_I. */
+ /* SHL_MI, SHL_HI. */
src_val = strtoul(src, &src, 0);
CHECK(!src[0], EINVAL);
+ instr->type = INSTR_ALU_SHL_MI;
if (dst[0] == 'h')
- src_val = htonl(src_val);
+ instr->type = INSTR_ALU_SHL_HI;
- instr->type = INSTR_ALU_AND_I;
instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
instr->alu.dst.n_bits = fdst->n_bits;
instr->alu.dst.offset = fdst->offset / 8;
return 0;
}
-static inline void
-instr_alu_add_exec(struct rte_swx_pipeline *p)
+static int
+instr_alu_shr_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
{
- struct thread *t = &p->threads[p->thread_id];
- struct instruction *ip = t->ip;
+ char *dst = tokens[1], *src = tokens[2];
+ struct field *fdst, *fsrc;
+ uint32_t dst_struct_id, src_struct_id, src_val;
- TRACE("[Thread %2u] add\n", p->thread_id);
+ CHECK(n_tokens == 3, EINVAL);
- /* Structs. */
- ALU(t, ip, +);
+ fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
+ CHECK(fdst, EINVAL);
- /* Thread. */
- thread_ip_inc(p);
-}
+ /* SHR, SHR_HM, SHR_MH, SHR_HH. */
+ fsrc = struct_field_parse(p, action, src, &src_struct_id);
+ if (fsrc) {
+ instr->type = INSTR_ALU_SHR;
+ if (dst[0] == 'h' && src[0] == 'm')
+ instr->type = INSTR_ALU_SHR_HM;
+ if (dst[0] == 'm' && src[0] == 'h')
+ instr->type = INSTR_ALU_SHR_MH;
+ if (dst[0] == 'h' && src[0] == 'h')
+ instr->type = INSTR_ALU_SHR_HH;
+
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src.struct_id = (uint8_t)src_struct_id;
+ instr->alu.src.n_bits = fsrc->n_bits;
+ instr->alu.src.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* SHR_MI, SHR_HI. */
+ src_val = strtoul(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ instr->type = INSTR_ALU_SHR_MI;
+ if (dst[0] == 'h')
+ instr->type = INSTR_ALU_SHR_HI;
+
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src_val = (uint32_t)src_val;
+ return 0;
+}
+
+static int
+instr_alu_and_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *dst = tokens[1], *src = tokens[2];
+ struct field *fdst, *fsrc;
+ uint32_t dst_struct_id, src_struct_id, src_val;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
+ CHECK(fdst, EINVAL);
+
+ /* AND or AND_S. */
+ fsrc = struct_field_parse(p, action, src, &src_struct_id);
+ if (fsrc) {
+ instr->type = INSTR_ALU_AND;
+ if ((dst[0] == 'h' && src[0] != 'h') ||
+ (dst[0] != 'h' && src[0] == 'h'))
+ instr->type = INSTR_ALU_AND_S;
+
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src.struct_id = (uint8_t)src_struct_id;
+ instr->alu.src.n_bits = fsrc->n_bits;
+ instr->alu.src.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* AND_I. */
+ src_val = strtoul(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ if (dst[0] == 'h')
+ src_val = htonl(src_val);
+
+ instr->type = INSTR_ALU_AND_I;
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src_val = (uint32_t)src_val;
+ return 0;
+}
+
+static int
+instr_alu_or_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *dst = tokens[1], *src = tokens[2];
+ struct field *fdst, *fsrc;
+ uint32_t dst_struct_id, src_struct_id, src_val;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
+ CHECK(fdst, EINVAL);
+
+ /* OR or OR_S. */
+ fsrc = struct_field_parse(p, action, src, &src_struct_id);
+ if (fsrc) {
+ instr->type = INSTR_ALU_OR;
+ if ((dst[0] == 'h' && src[0] != 'h') ||
+ (dst[0] != 'h' && src[0] == 'h'))
+ instr->type = INSTR_ALU_OR_S;
+
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src.struct_id = (uint8_t)src_struct_id;
+ instr->alu.src.n_bits = fsrc->n_bits;
+ instr->alu.src.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* OR_I. */
+ src_val = strtoul(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ if (dst[0] == 'h')
+ src_val = htonl(src_val);
+
+ instr->type = INSTR_ALU_OR_I;
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src_val = (uint32_t)src_val;
+ return 0;
+}
+
+static int
+instr_alu_xor_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ char *dst = tokens[1], *src = tokens[2];
+ struct field *fdst, *fsrc;
+ uint32_t dst_struct_id, src_struct_id, src_val;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ fdst = struct_field_parse(p, NULL, dst, &dst_struct_id);
+ CHECK(fdst, EINVAL);
+
+ /* XOR or XOR_S. */
+ fsrc = struct_field_parse(p, action, src, &src_struct_id);
+ if (fsrc) {
+ instr->type = INSTR_ALU_XOR;
+ if ((dst[0] == 'h' && src[0] != 'h') ||
+ (dst[0] != 'h' && src[0] == 'h'))
+ instr->type = INSTR_ALU_XOR_S;
+
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src.struct_id = (uint8_t)src_struct_id;
+ instr->alu.src.n_bits = fsrc->n_bits;
+ instr->alu.src.offset = fsrc->offset / 8;
+ return 0;
+ }
+
+ /* XOR_I. */
+ src_val = strtoul(src, &src, 0);
+ CHECK(!src[0], EINVAL);
+
+ if (dst[0] == 'h')
+ src_val = htonl(src_val);
+
+ instr->type = INSTR_ALU_XOR_I;
+ instr->alu.dst.struct_id = (uint8_t)dst_struct_id;
+ instr->alu.dst.n_bits = fdst->n_bits;
+ instr->alu.dst.offset = fdst->offset / 8;
+ instr->alu.src_val = (uint32_t)src_val;
+ return 0;
+}
+
+static inline void
+instr_alu_add_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] add\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, +);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
static inline void
instr_alu_add_mh_exec(struct rte_swx_pipeline *p)
}
static inline void
-instr_alu_and_exec(struct rte_swx_pipeline *p)
+instr_alu_shl_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
- TRACE("[Thread %2u] and\n", p->thread_id);
+ TRACE("[Thread %2u] shl\n", p->thread_id);
/* Structs. */
- ALU(t, ip, &);
+ ALU(t, ip, <<);
/* Thread. */
thread_ip_inc(p);
}
static inline void
-instr_alu_and_s_exec(struct rte_swx_pipeline *p)
+instr_alu_shl_mh_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
- TRACE("[Thread %2u] and (s)\n", p->thread_id);
+ TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
/* Structs. */
- ALU_S(t, ip, &);
+ ALU_MH(t, ip, <<);
/* Thread. */
thread_ip_inc(p);
}
static inline void
-instr_alu_and_i_exec(struct rte_swx_pipeline *p)
+instr_alu_shl_hm_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
- TRACE("[Thread %2u] and (i)\n", p->thread_id);
+ TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
/* Structs. */
- ALU_I(t, ip, &);
+ ALU_HM(t, ip, <<);
/* Thread. */
thread_ip_inc(p);
}
static inline void
-instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
+instr_alu_shl_hh_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
- uint8_t *dst_struct, *src_struct;
- uint16_t *dst16_ptr, dst;
- uint64_t *src64_ptr, src64, src64_mask, src;
- uint64_t r;
- TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
+ TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
/* Structs. */
- dst_struct = t->structs[ip->alu.dst.struct_id];
- dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
- dst = *dst16_ptr;
+ ALU_HH(t, ip, <<);
- src_struct = t->structs[ip->alu.src.struct_id];
- src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
- src64 = *src64_ptr;
- src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
- src = src64 & src64_mask;
+ /* Thread. */
+ thread_ip_inc(p);
+}
- r = dst;
- r = ~r & 0xFFFF;
+static inline void
+instr_alu_shl_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
- /* The first input (r) is a 16-bit number. The second and the third
- * inputs are 32-bit numbers. In the worst case scenario, the sum of the
- * three numbers (output r) is a 34-bit number.
- */
- r += (src >> 32) + (src & 0xFFFFFFFF);
+ TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
- /* The first input is a 16-bit number. The second input is an 18-bit
- * number. In the worst case scenario, the sum of the two numbers is a
- * 19-bit number.
- */
- r = (r & 0xFFFF) + (r >> 16);
+ /* Structs. */
+ ALU_MI(t, ip, <<);
- /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
- * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
- */
- r = (r & 0xFFFF) + (r >> 16);
+ /* Thread. */
+ thread_ip_inc(p);
+}
- /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
- * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
- * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
- * therefore the output r is always a 16-bit number.
- */
- r = (r & 0xFFFF) + (r >> 16);
+static inline void
+instr_alu_shl_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
- r = ~r & 0xFFFF;
- r = r ? r : 0xFFFF;
+ TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
- *dst16_ptr = (uint16_t)r;
+ /* Structs. */
+ ALU_HI(t, ip, <<);
/* Thread. */
thread_ip_inc(p);
}
static inline void
-instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
+instr_alu_shr_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
- uint8_t *dst_struct, *src_struct;
- uint16_t *dst16_ptr, dst;
- uint64_t *src64_ptr, src64, src64_mask, src;
- uint64_t r;
- TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
+ TRACE("[Thread %2u] shr\n", p->thread_id);
/* Structs. */
- dst_struct = t->structs[ip->alu.dst.struct_id];
- dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
- dst = *dst16_ptr;
-
- src_struct = t->structs[ip->alu.src.struct_id];
- src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
- src64 = *src64_ptr;
- src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
- src = src64 & src64_mask;
+ ALU(t, ip, >>);
- r = dst;
- r = ~r & 0xFFFF;
+ /* Thread. */
+ thread_ip_inc(p);
+}
- /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
- * the following sequence of operations in 2's complement arithmetic:
- * a '- b = (a - b) % 0xFFFF.
- *
- * In order to prevent an underflow for the below subtraction, in which
- * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
- * minuend), we first add a multiple of the 0xFFFF modulus to the
- * minuend. The number we add to the minuend needs to be a 34-bit number
- * or higher, so for readability reasons we picked the 36-bit multiple.
- * We are effectively turning the 16-bit minuend into a 36-bit number:
- * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
- */
- r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
+static inline void
+instr_alu_shr_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
- /* A 33-bit number is subtracted from a 36-bit number (the input r). The
- * result (the output r) is a 36-bit number.
- */
- r -= (src >> 32) + (src & 0xFFFFFFFF);
+ TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
- /* The first input is a 16-bit number. The second input is a 20-bit
- * number. Their sum is a 21-bit number.
- */
- r = (r & 0xFFFF) + (r >> 16);
+ /* Structs. */
+ ALU_MH(t, ip, >>);
- /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
- * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
- */
- r = (r & 0xFFFF) + (r >> 16);
+ /* Thread. */
+ thread_ip_inc(p);
+}
- /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
- * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
- * 0x1001E), the output r is (0 .. 31). So no carry bit can be
- * generated, therefore the output r is always a 16-bit number.
- */
- r = (r & 0xFFFF) + (r >> 16);
+static inline void
+instr_alu_shr_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
- r = ~r & 0xFFFF;
- r = r ? r : 0xFFFF;
+ TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
- *dst16_ptr = (uint16_t)r;
+ /* Structs. */
+ ALU_HM(t, ip, >>);
/* Thread. */
thread_ip_inc(p);
}
static inline void
-instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
+instr_alu_shr_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HH(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_MI(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_shr_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_HI(t, ip, >>);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_and_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] and\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, &);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_and_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] and (s)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_S(t, ip, &);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_and_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] and (i)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_I(t, ip, &);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_or_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] or\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, |);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_or_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] or (s)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_S(t, ip, |);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_or_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] or (i)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_I(t, ip, |);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_xor_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] xor\n", p->thread_id);
+
+ /* Structs. */
+ ALU(t, ip, ^);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_xor_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] xor (s)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_S(t, ip, ^);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_xor_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] xor (i)\n", p->thread_id);
+
+ /* Structs. */
+ ALU_I(t, ip, ^);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
uint8_t *dst_struct, *src_struct;
- uint16_t *dst16_ptr;
- uint32_t *src32_ptr;
- uint64_t r0, r1;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
- TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
+ TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
/* Structs. */
dst_struct = t->structs[ip->alu.dst.struct_id];
dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
src_struct = t->structs[ip->alu.src.struct_id];
- src32_ptr = (uint32_t *)&src_struct[0];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
- r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
- r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
- r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
- r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
- r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
+ r = dst;
+ r = ~r & 0xFFFF;
- /* The first input is a 16-bit number. The second input is a 19-bit
- * number. Their sum is a 20-bit number.
+ /* The first input (r) is a 16-bit number. The second and the third
+ * inputs are 32-bit numbers. In the worst case scenario, the sum of the
+ * three numbers (output r) is a 34-bit number.
*/
- r0 = (r0 & 0xFFFF) + (r0 >> 16);
+ r += (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is an 18-bit
+ * number. In the worst case scenario, the sum of the two numbers is a
+ * 19-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
/* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
- * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
+ * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
*/
- r0 = (r0 & 0xFFFF) + (r0 >> 16);
+ r = (r & 0xFFFF) + (r >> 16);
/* When the input r is (0 .. 0xFFFF), the output r is equal to the input
* r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
- * 0x1000E), the output r is (0 .. 15). So no carry bit can be
- * generated, therefore the output r is always a 16-bit number.
+ * 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
+ * therefore the output r is always a 16-bit number.
*/
- r0 = (r0 & 0xFFFF) + (r0 >> 16);
+ r = (r & 0xFFFF) + (r >> 16);
- r0 = ~r0 & 0xFFFF;
- r0 = r0 ? r0 : 0xFFFF;
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
- *dst16_ptr = (uint16_t)r0;
+ *dst16_ptr = (uint16_t)r;
/* Thread. */
thread_ip_inc(p);
}
static inline void
-instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
+instr_alu_cksub_field_exec(struct rte_swx_pipeline *p)
{
struct thread *t = &p->threads[p->thread_id];
struct instruction *ip = t->ip;
uint8_t *dst_struct, *src_struct;
- uint16_t *dst16_ptr;
- uint32_t *src32_ptr;
- uint64_t r = 0;
+ uint16_t *dst16_ptr, dst;
+ uint64_t *src64_ptr, src64, src64_mask, src;
+ uint64_t r;
+
+ TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+ dst = *dst16_ptr;
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
+ src64 = *src64_ptr;
+ src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
+ src = src64 & src64_mask;
+
+ r = dst;
+ r = ~r & 0xFFFF;
+
+ /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
+ * the following sequence of operations in 2's complement arithmetic:
+ * a '- b = (a - b) % 0xFFFF.
+ *
+ * In order to prevent an underflow for the below subtraction, in which
+ * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
+ * minuend), we first add a multiple of the 0xFFFF modulus to the
+ * minuend. The number we add to the minuend needs to be a 34-bit number
+ * or higher, so for readability reasons we picked the 36-bit multiple.
+ * We are effectively turning the 16-bit minuend into a 36-bit number:
+ * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
+ */
+ r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
+
+ /* A 33-bit number is subtracted from a 36-bit number (the input r). The
+ * result (the output r) is a 36-bit number.
+ */
+ r -= (src >> 32) + (src & 0xFFFFFFFF);
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr;
+ uint32_t *src32_ptr;
+ uint64_t r0, r1;
+
+ TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
+
+ /* Structs. */
+ dst_struct = t->structs[ip->alu.dst.struct_id];
+ dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
+
+ src_struct = t->structs[ip->alu.src.struct_id];
+ src32_ptr = (uint32_t *)&src_struct[0];
+
+ r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
+ r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
+ r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
+ r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
+ r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
+
+ /* The first input is a 16-bit number. The second input is a 19-bit
+ * number. Their sum is a 20-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1000E), the output r is (0 .. 15). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r0 = (r0 & 0xFFFF) + (r0 >> 16);
+
+ r0 = ~r0 & 0xFFFF;
+ r0 = r0 ? r0 : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r0;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint8_t *dst_struct, *src_struct;
+ uint16_t *dst16_ptr;
+ uint32_t *src32_ptr;
+ uint64_t r = 0;
uint32_t i;
TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
src_struct = t->structs[ip->alu.src.struct_id];
src32_ptr = (uint32_t *)&src_struct[0];
- /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
- * Therefore, in the worst case scenario, a 35-bit number is added to a
- * 16-bit number (the input r), so the output r is 36-bit number.
- */
- for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
- r += *src32_ptr;
+ /* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
+ * Therefore, in the worst case scenario, a 35-bit number is added to a
+ * 16-bit number (the input r), so the output r is 36-bit number.
+ */
+ for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
+ r += *src32_ptr;
+
+ /* The first input is a 16-bit number. The second input is a 20-bit
+ * number. Their sum is a 21-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
+ * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
+ * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
+ * 0x1001E), the output r is (0 .. 31). So no carry bit can be
+ * generated, therefore the output r is always a 16-bit number.
+ */
+ r = (r & 0xFFFF) + (r >> 16);
+
+ r = ~r & 0xFFFF;
+ r = r ? r : 0xFFFF;
+
+ *dst16_ptr = (uint16_t)r;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+/*
+ * jmp.
+ */
+static struct action *
+action_find(struct rte_swx_pipeline *p, const char *name);
+
+static int
+instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_valid_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct header *h;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ h = header_parse(p, tokens[2]);
+ CHECK(h, EINVAL);
+
+ instr->type = INSTR_JMP_VALID;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.header_id = h->id;
+ return 0;
+}
+
+static int
+instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct header *h;
+
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ h = header_parse(p, tokens[2]);
+ CHECK(h, EINVAL);
+
+ instr->type = INSTR_JMP_INVALID;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.header_id = h->id;
+ return 0;
+}
+
+static int
+instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP_HIT;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP_MISS;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct action *a;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ a = action_find(p, tokens[2]);
+ CHECK(a, EINVAL);
+
+ instr->type = INSTR_JMP_ACTION_HIT;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.action_id = a->id;
+ return 0;
+}
+
+static int
+instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct action *a;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ a = action_find(p, tokens[2]);
+ CHECK(a, EINVAL);
+
+ instr->type = INSTR_JMP_ACTION_MISS;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.action_id = a->id;
+ return 0;
+}
+
+static int
+instr_jmp_eq_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint32_t a_struct_id, b_struct_id, b_val;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_EQ or JMP_EQ_S. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_EQ;
+ if ((a[0] == 'h' && b[0] != 'h') ||
+ (a[0] != 'h' && b[0] == 'h'))
+ instr->type = INSTR_JMP_EQ_S;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_EQ_I. */
+ b_val = strtoul(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ if (a[0] == 'h')
+ b_val = htonl(b_val);
+
+ instr->type = INSTR_JMP_EQ_I;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = (uint32_t)b_val;
+ return 0;
+}
+
+static int
+instr_jmp_neq_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint32_t a_struct_id, b_struct_id, b_val;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_NEQ or JMP_NEQ_S. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_NEQ;
+ if ((a[0] == 'h' && b[0] != 'h') ||
+ (a[0] != 'h' && b[0] == 'h'))
+ instr->type = INSTR_JMP_NEQ_S;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_NEQ_I. */
+ b_val = strtoul(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ if (a[0] == 'h')
+ b_val = htonl(b_val);
+
+ instr->type = INSTR_JMP_NEQ_I;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = (uint32_t)b_val;
+ return 0;
+}
+
+static int
+instr_jmp_lt_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint32_t a_struct_id, b_struct_id, b_val;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_LT;
+ if (a[0] == 'h' && b[0] == 'm')
+ instr->type = INSTR_JMP_LT_HM;
+ if (a[0] == 'm' && b[0] == 'h')
+ instr->type = INSTR_JMP_LT_MH;
+ if (a[0] == 'h' && b[0] == 'h')
+ instr->type = INSTR_JMP_LT_HH;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_LT_MI, JMP_LT_HI. */
+ b_val = strtoul(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ instr->type = INSTR_JMP_LT_MI;
+ if (a[0] == 'h')
+ instr->type = INSTR_JMP_LT_HI;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = (uint32_t)b_val;
+ return 0;
+}
+
+static int
+instr_jmp_gt_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint32_t a_struct_id, b_struct_id, b_val;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_GT;
+ if (a[0] == 'h' && b[0] == 'm')
+ instr->type = INSTR_JMP_GT_HM;
+ if (a[0] == 'm' && b[0] == 'h')
+ instr->type = INSTR_JMP_GT_MH;
+ if (a[0] == 'h' && b[0] == 'h')
+ instr->type = INSTR_JMP_GT_HH;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_GT_MI, JMP_GT_HI. */
+ b_val = strtoul(b, &b, 0);
+ CHECK(!b[0], EINVAL);
+
+ instr->type = INSTR_JMP_GT_MI;
+ if (a[0] == 'h')
+ instr->type = INSTR_JMP_GT_HI;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b_val = (uint32_t)b_val;
+ return 0;
+}
+
+static inline void
+instr_jmp_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmp\n", p->thread_id);
+
+ thread_ip_set(t, ip->jmp.ip);
+}
+
+static inline void
+instr_jmp_valid_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t header_id = ip->jmp.header_id;
+
+ TRACE("[Thread %2u] jmpv\n", p->thread_id);
+
+ t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1);
+}
+
+static inline void
+instr_jmp_invalid_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint32_t header_id = ip->jmp.header_id;
+
+ TRACE("[Thread %2u] jmpnv\n", p->thread_id);
+
+ t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip;
+}
+
+static inline void
+instr_jmp_hit_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip};
+
+ TRACE("[Thread %2u] jmph\n", p->thread_id);
+
+ t->ip = ip_next[t->hit];
+}
+
+static inline void
+instr_jmp_miss_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1};
+
+ TRACE("[Thread %2u] jmpnh\n", p->thread_id);
+
+ t->ip = ip_next[t->hit];
+}
+
+static inline void
+instr_jmp_action_hit_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpa\n", p->thread_id);
+
+ t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1);
+}
+
+static inline void
+instr_jmp_action_miss_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpna\n", p->thread_id);
+
+ t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip;
+}
+
+static inline void
+instr_jmp_eq_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpeq\n", p->thread_id);
+
+ JMP_CMP(t, ip, ==);
+}
+
+static inline void
+instr_jmp_eq_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id);
+
+ JMP_CMP_S(t, ip, ==);
+}
+
+static inline void
+instr_jmp_eq_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id);
+
+ JMP_CMP_I(t, ip, ==);
+}
+
+static inline void
+instr_jmp_neq_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpneq\n", p->thread_id);
+
+ JMP_CMP(t, ip, !=);
+}
+
+static inline void
+instr_jmp_neq_s_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id);
+
+ JMP_CMP_S(t, ip, !=);
+}
+
+static inline void
+instr_jmp_neq_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id);
+
+ JMP_CMP_I(t, ip, !=);
+}
+
+static inline void
+instr_jmp_lt_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt\n", p->thread_id);
+
+ JMP_CMP(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id);
+
+ JMP_CMP_MH(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id);
+
+ JMP_CMP_HM(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id);
+
+ JMP_CMP_HH(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id);
+
+ JMP_CMP_MI(t, ip, <);
+}
+
+static inline void
+instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id);
+
+ JMP_CMP_HI(t, ip, <);
+}
+
+static inline void
+instr_jmp_gt_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt\n", p->thread_id);
+
+ JMP_CMP(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id);
+
+ JMP_CMP_MH(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id);
+
+ JMP_CMP_HM(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id);
+
+ JMP_CMP_HH(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id);
+
+ JMP_CMP_MI(t, ip, >);
+}
+
+static inline void
+instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id);
- /* The first input is a 16-bit number. The second input is a 20-bit
- * number. Their sum is a 21-bit number.
- */
- r = (r & 0xFFFF) + (r >> 16);
+ JMP_CMP_HI(t, ip, >);
+}
- /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
- * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
- */
- r = (r & 0xFFFF) + (r >> 16);
+/*
+ * return.
+ */
+static int
+instr_return_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens __rte_unused,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ CHECK(action, EINVAL);
+ CHECK(n_tokens == 1, EINVAL);
- /* When the input r is (0 .. 0xFFFF), the output r is equal to the input
- * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
- * 0x1001E), the output r is (0 .. 31). So no carry bit can be
- * generated, therefore the output r is always a 16-bit number.
- */
- r = (r & 0xFFFF) + (r >> 16);
+ instr->type = INSTR_RETURN;
+ return 0;
+}
- r = ~r & 0xFFFF;
- r = r ? r : 0xFFFF;
+static inline void
+instr_return_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
- *dst16_ptr = (uint16_t)r;
+ TRACE("[Thread %2u] return\n", p->thread_id);
- /* Thread. */
- thread_ip_inc(p);
+ t->ip = t->ret;
}
#define RTE_SWX_INSTRUCTION_TOKENS_MAX 16
instr,
data);
+ if (!strcmp(tokens[tpos], "or"))
+ return instr_alu_or_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "xor"))
+ return instr_alu_xor_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "shl"))
+ return instr_alu_shl_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "shr"))
+ return instr_alu_shr_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "table"))
+ return instr_table_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "extern"))
+ return instr_extern_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmp"))
+ return instr_jmp_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpv"))
+ return instr_jmp_valid_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpnv"))
+ return instr_jmp_invalid_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmph"))
+ return instr_jmp_hit_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpnh"))
+ return instr_jmp_miss_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpa"))
+ return instr_jmp_action_hit_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpna"))
+ return instr_jmp_action_miss_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpeq"))
+ return instr_jmp_eq_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpneq"))
+ return instr_jmp_neq_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmplt"))
+ return instr_jmp_lt_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "jmpgt"))
+ return instr_jmp_gt_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "return"))
+ return instr_return_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
CHECK(0, EINVAL);
}
+static struct instruction_data *
+label_find(struct instruction_data *data, uint32_t n, const char *label)
+{
+ uint32_t i;
+
+ for (i = 0; i < n; i++)
+ if (!strcmp(label, data[i].label))
+ return &data[i];
+
+ return NULL;
+}
+
static uint32_t
label_is_used(struct instruction_data *data, uint32_t n, const char *label)
{
return 0;
}
+static int
+instr_jmp_resolve(struct instruction *instructions,
+ struct instruction_data *instruction_data,
+ uint32_t n_instructions)
+{
+ uint32_t i;
+
+ for (i = 0; i < n_instructions; i++) {
+ struct instruction *instr = &instructions[i];
+ struct instruction_data *data = &instruction_data[i];
+ struct instruction_data *found;
+
+ if (!instruction_is_jmp(instr))
+ continue;
+
+ found = label_find(instruction_data,
+ n_instructions,
+ data->jmp_label);
+ CHECK(found, EINVAL);
+
+ instr->jmp.ip = &instr[found - instruction_data];
+ }
+
+ return 0;
+}
+
+static int
+instr_verify(struct rte_swx_pipeline *p __rte_unused,
+ struct action *a,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused,
+ uint32_t n_instructions)
+{
+ if (!a) {
+ enum instruction_type type;
+ uint32_t i;
+
+ /* Check that the first instruction is rx. */
+ CHECK(instr[0].type == INSTR_RX, EINVAL);
+
+ /* Check that there is at least one tx instruction. */
+ for (i = 0; i < n_instructions; i++) {
+ type = instr[i].type;
+
+ if (instr[i].type == INSTR_TX)
+ break;
+ }
+ CHECK(i < n_instructions, EINVAL);
+
+ /* Check that the last instruction is either tx or unconditional
+ * jump.
+ */
+ type = instr[n_instructions - 1].type;
+ CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
+ }
+
+ if (a) {
+ enum instruction_type type;
+ uint32_t i;
+
+ /* Check that there is at least one return or tx instruction. */
+ for (i = 0; i < n_instructions; i++) {
+ type = instr[i].type;
+
+ if ((type == INSTR_RETURN) || (type == INSTR_TX))
+ break;
+ }
+ CHECK(i < n_instructions, EINVAL);
+ }
+
+ return 0;
+}
+
+static int
+instr_pattern_extract_many_detect(struct instruction *instr,
+ struct instruction_data *data,
+ uint32_t n_instr,
+ uint32_t *n_pattern_instr)
+{
+ uint32_t i;
+
+ for (i = 0; i < n_instr; i++) {
+ if (data[i].invalid)
+ break;
+
+ if (instr[i].type != INSTR_HDR_EXTRACT)
+ break;
+
+ if (i == RTE_DIM(instr->io.hdr.header_id))
+ break;
+
+ if (i && data[i].n_users)
+ break;
+ }
+
+ if (i < 2)
+ return 0;
+
+ *n_pattern_instr = i;
+ return 1;
+}
+
+static void
+instr_pattern_extract_many_optimize(struct instruction *instr,
+ struct instruction_data *data,
+ uint32_t n_instr)
+{
+ uint32_t i;
+
+ for (i = 1; i < n_instr; i++) {
+ instr[0].type++;
+ instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
+ instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
+ instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
+
+ data[i].invalid = 1;
+ }
+}
+
+static int
+instr_pattern_emit_many_tx_detect(struct instruction *instr,
+ struct instruction_data *data,
+ uint32_t n_instr,
+ uint32_t *n_pattern_instr)
+{
+ uint32_t i;
+
+ for (i = 0; i < n_instr; i++) {
+ if (data[i].invalid)
+ break;
+
+ if (instr[i].type != INSTR_HDR_EMIT)
+ break;
+
+ if (i == RTE_DIM(instr->io.hdr.header_id))
+ break;
+
+ if (i && data[i].n_users)
+ break;
+ }
+
+ if (!i)
+ return 0;
+
+ if (instr[i].type != INSTR_TX)
+ return 0;
+
+ i++;
+
+ *n_pattern_instr = i;
+ return 1;
+}
+
+static void
+instr_pattern_emit_many_tx_optimize(struct instruction *instr,
+ struct instruction_data *data,
+ uint32_t n_instr)
+{
+ uint32_t i;
+
+ /* Any emit instruction in addition to the first one. */
+ for (i = 1; i < n_instr - 1; i++) {
+ instr[0].type++;
+ instr[0].io.hdr.header_id[i] = instr[i].io.hdr.header_id[0];
+ instr[0].io.hdr.struct_id[i] = instr[i].io.hdr.struct_id[0];
+ instr[0].io.hdr.n_bytes[i] = instr[i].io.hdr.n_bytes[0];
+
+ data[i].invalid = 1;
+ }
+
+ /* The TX instruction is the last one in the pattern. */
+ instr[0].type++;
+ instr[0].io.io.offset = instr[i].io.io.offset;
+ instr[0].io.io.n_bits = instr[i].io.io.n_bits;
+ data[i].invalid = 1;
+}
+
+static int
+instr_pattern_dma_many_detect(struct instruction *instr,
+ struct instruction_data *data,
+ uint32_t n_instr,
+ uint32_t *n_pattern_instr)
+{
+ uint32_t i;
+
+ for (i = 0; i < n_instr; i++) {
+ if (data[i].invalid)
+ break;
+
+ if (instr[i].type != INSTR_DMA_HT)
+ break;
+
+ if (i == RTE_DIM(instr->dma.dst.header_id))
+ break;
+
+ if (i && data[i].n_users)
+ break;
+ }
+
+ if (i < 2)
+ return 0;
+
+ *n_pattern_instr = i;
+ return 1;
+}
+
+static void
+instr_pattern_dma_many_optimize(struct instruction *instr,
+ struct instruction_data *data,
+ uint32_t n_instr)
+{
+ uint32_t i;
+
+ for (i = 1; i < n_instr; i++) {
+ instr[0].type++;
+ instr[0].dma.dst.header_id[i] = instr[i].dma.dst.header_id[0];
+ instr[0].dma.dst.struct_id[i] = instr[i].dma.dst.struct_id[0];
+ instr[0].dma.src.offset[i] = instr[i].dma.src.offset[0];
+ instr[0].dma.n_bytes[i] = instr[i].dma.n_bytes[0];
+
+ data[i].invalid = 1;
+ }
+}
+
+static uint32_t
+instr_optimize(struct instruction *instructions,
+ struct instruction_data *instruction_data,
+ uint32_t n_instructions)
+{
+ uint32_t i, pos = 0;
+
+ for (i = 0; i < n_instructions; ) {
+ struct instruction *instr = &instructions[i];
+ struct instruction_data *data = &instruction_data[i];
+ uint32_t n_instr = 0;
+ int detected;
+
+ /* Extract many. */
+ detected = instr_pattern_extract_many_detect(instr,
+ data,
+ n_instructions - i,
+ &n_instr);
+ if (detected) {
+ instr_pattern_extract_many_optimize(instr,
+ data,
+ n_instr);
+ i += n_instr;
+ continue;
+ }
+
+ /* Emit many + TX. */
+ detected = instr_pattern_emit_many_tx_detect(instr,
+ data,
+ n_instructions - i,
+ &n_instr);
+ if (detected) {
+ instr_pattern_emit_many_tx_optimize(instr,
+ data,
+ n_instr);
+ i += n_instr;
+ continue;
+ }
+
+ /* DMA many. */
+ detected = instr_pattern_dma_many_detect(instr,
+ data,
+ n_instructions - i,
+ &n_instr);
+ if (detected) {
+ instr_pattern_dma_many_optimize(instr, data, n_instr);
+ i += n_instr;
+ continue;
+ }
+
+ /* No pattern starting at the current instruction. */
+ i++;
+ }
+
+ /* Eliminate the invalid instructions that have been optimized out. */
+ for (i = 0; i < n_instructions; i++) {
+ struct instruction *instr = &instructions[i];
+ struct instruction_data *data = &instruction_data[i];
+
+ if (data->invalid)
+ continue;
+
+ if (i != pos) {
+ memcpy(&instructions[pos], instr, sizeof(*instr));
+ memcpy(&instruction_data[pos], data, sizeof(*data));
+ }
+
+ pos++;
+ }
+
+ return pos;
+}
+
static int
instruction_config(struct rte_swx_pipeline *p,
struct action *a,
if (err)
goto error;
+ err = instr_verify(p, a, instr, data, n_instructions);
+ if (err)
+ goto error;
+
+ n_instructions = instr_optimize(instr, data, n_instructions);
+
+ err = instr_jmp_resolve(instr, data, n_instructions);
+ if (err)
+ goto error;
+
free(data);
if (a) {
[INSTR_ALU_AND] = instr_alu_and_exec,
[INSTR_ALU_AND_S] = instr_alu_and_s_exec,
[INSTR_ALU_AND_I] = instr_alu_and_i_exec,
+
+ [INSTR_ALU_OR] = instr_alu_or_exec,
+ [INSTR_ALU_OR_S] = instr_alu_or_s_exec,
+ [INSTR_ALU_OR_I] = instr_alu_or_i_exec,
+
+ [INSTR_ALU_XOR] = instr_alu_xor_exec,
+ [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec,
+ [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec,
+
+ [INSTR_ALU_SHL] = instr_alu_shl_exec,
+ [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec,
+ [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec,
+ [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec,
+ [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec,
+ [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec,
+
+ [INSTR_ALU_SHR] = instr_alu_shr_exec,
+ [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec,
+ [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec,
+ [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec,
+ [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec,
+ [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec,
+
+ [INSTR_TABLE] = instr_table_exec,
+ [INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
+ [INSTR_EXTERN_FUNC] = instr_extern_func_exec,
+
+ [INSTR_JMP] = instr_jmp_exec,
+ [INSTR_JMP_VALID] = instr_jmp_valid_exec,
+ [INSTR_JMP_INVALID] = instr_jmp_invalid_exec,
+ [INSTR_JMP_HIT] = instr_jmp_hit_exec,
+ [INSTR_JMP_MISS] = instr_jmp_miss_exec,
+ [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec,
+ [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec,
+
+ [INSTR_JMP_EQ] = instr_jmp_eq_exec,
+ [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec,
+ [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec,
+
+ [INSTR_JMP_NEQ] = instr_jmp_neq_exec,
+ [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec,
+ [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec,
+
+ [INSTR_JMP_LT] = instr_jmp_lt_exec,
+ [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec,
+ [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec,
+ [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec,
+ [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec,
+ [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec,
+
+ [INSTR_JMP_GT] = instr_jmp_gt_exec,
+ [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec,
+ [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec,
+ [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec,
+ [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec,
+ [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec,
+
+ [INSTR_RETURN] = instr_return_exec,
};
static inline void