X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_pipeline%2Frte_swx_pipeline.c;h=d51fec821c0c7d5c8c3866cc8c95f748c12080a6;hb=75634474ca485654630e3219aeef8a80ac226533;hp=364c7d75aebe39ead397a96466088e6e9efef3a2;hpb=1e6bf5997cce82d981d7eaae630b06fa0e9f5500;p=dpdk.git diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index 364c7d75ae..d51fec821c 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -303,6 +303,139 @@ enum instruction_type { * dst = H, src = H */ INSTR_ALU_CKSUB_FIELD, + + /* and dst src + * dst &= src + * dst = HMEF, src = HMEFTI + */ + INSTR_ALU_AND, /* dst = MEF, src = MEFT */ + INSTR_ALU_AND_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */ + INSTR_ALU_AND_I, /* dst = HMEF, src = I */ + + /* or dst src + * dst |= src + * dst = HMEF, src = HMEFTI + */ + INSTR_ALU_OR, /* dst = MEF, src = MEFT */ + INSTR_ALU_OR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */ + INSTR_ALU_OR_I, /* dst = HMEF, src = I */ + + /* xor dst src + * dst ^= src + * dst = HMEF, src = HMEFTI + */ + INSTR_ALU_XOR, /* dst = MEF, src = MEFT */ + INSTR_ALU_XOR_S, /* (dst, src) = (MEF, H) or (dst, src) = (H, MEFT) */ + INSTR_ALU_XOR_I, /* dst = HMEF, src = I */ + + /* shl dst src + * dst <<= src + * dst = HMEF, src = HMEFTI + */ + INSTR_ALU_SHL, /* dst = MEF, src = MEF */ + INSTR_ALU_SHL_MH, /* dst = MEF, src = H */ + INSTR_ALU_SHL_HM, /* dst = H, src = MEF */ + INSTR_ALU_SHL_HH, /* dst = H, src = H */ + INSTR_ALU_SHL_MI, /* dst = MEF, src = I */ + INSTR_ALU_SHL_HI, /* dst = H, src = I */ + + /* shr dst src + * dst >>= src + * dst = HMEF, src = HMEFTI + */ + INSTR_ALU_SHR, /* dst = MEF, src = MEF */ + INSTR_ALU_SHR_MH, /* dst = MEF, src = H */ + INSTR_ALU_SHR_HM, /* dst = H, src = MEF */ + INSTR_ALU_SHR_HH, /* dst = H, src = H */ + INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ + INSTR_ALU_SHR_HI, /* dst = H, src = I */ + + /* table TABLE */ + INSTR_TABLE, + + /* extern e.obj.func */ + INSTR_EXTERN_OBJ, + + /* extern f.func */ + INSTR_EXTERN_FUNC, + + /* jmp LABEL + * Unconditional jump + */ + INSTR_JMP, + + /* jmpv LABEL h.header + * Jump if header is valid + */ + INSTR_JMP_VALID, + + /* jmpnv LABEL h.header + * Jump if header is invalid + */ + INSTR_JMP_INVALID, + + /* jmph LABEL + * Jump if table lookup hit + */ + INSTR_JMP_HIT, + + /* jmpnh LABEL + * Jump if table lookup miss + */ + INSTR_JMP_MISS, + + /* jmpa LABEL ACTION + * Jump if action run + */ + INSTR_JMP_ACTION_HIT, + + /* jmpna LABEL ACTION + * Jump if action not run + */ + INSTR_JMP_ACTION_MISS, + + /* jmpeq LABEL a b + * Jump is a is equal to b + * a = HMEFT, b = HMEFTI + */ + INSTR_JMP_EQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */ + INSTR_JMP_EQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */ + INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ + + /* jmpneq LABEL a b + * Jump is a is not equal to b + * a = HMEFT, b = HMEFTI + */ + INSTR_JMP_NEQ, /* (a, b) = (MEFT, MEFT) or (a, b) = (H, H) */ + INSTR_JMP_NEQ_S, /* (a, b) = (MEFT, H) or (a, b) = (H, MEFT) */ + INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */ + + /* jmplt LABEL a b + * Jump if a is less than b + * a = HMEFT, b = HMEFTI + */ + INSTR_JMP_LT, /* a = MEF, b = MEF */ + INSTR_JMP_LT_MH, /* a = MEF, b = H */ + INSTR_JMP_LT_HM, /* a = H, b = MEF */ + INSTR_JMP_LT_HH, /* a = H, b = H */ + INSTR_JMP_LT_MI, /* a = MEF, b = I */ + INSTR_JMP_LT_HI, /* a = H, b = I */ + + /* jmpgt LABEL a b + * Jump if a is greater than b + * a = HMEFT, b = HMEFTI + */ + INSTR_JMP_GT, /* a = MEF, b = MEF */ + INSTR_JMP_GT_MH, /* a = MEF, b = H */ + INSTR_JMP_GT_HM, /* a = H, b = MEF */ + INSTR_JMP_GT_HH, /* a = H, b = H */ + INSTR_JMP_GT_MI, /* a = MEF, b = I */ + INSTR_JMP_GT_HI, /* a = H, b = I */ + + /* return + * Return from action + */ + INSTR_RETURN, }; struct instr_operand { @@ -330,6 +463,19 @@ struct instr_hdr_validity { uint8_t header_id; }; +struct instr_table { + uint8_t table_id; +}; + +struct instr_extern_obj { + uint8_t ext_obj_id; + uint8_t func_id; +}; + +struct instr_extern_func { + uint8_t ext_func_id; +}; + struct instr_dst_src { struct instr_operand dst; union { @@ -351,6 +497,21 @@ struct instr_dma { uint16_t n_bytes[8]; }; +struct instr_jmp { + struct instruction *ip; + + union { + struct instr_operand a; + uint8_t header_id; + uint8_t action_id; + }; + + union { + struct instr_operand b; + uint32_t b_val; + }; +}; + struct instruction { enum instruction_type type; union { @@ -359,6 +520,10 @@ struct instruction { struct instr_dst_src mov; struct instr_dma dma; struct instr_dst_src alu; + struct instr_table table; + struct instr_extern_obj ext_obj; + struct instr_extern_func ext_func; + struct instr_jmp jmp; }; }; @@ -473,6 +638,9 @@ struct thread { #define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos))) #define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos))) +#define HEADER_VALID(thread, header_id) \ + MASK64_BIT_GET((thread)->valid_headers, header_id) + #define ALU(thread, ip, operator) \ { \ uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \ @@ -654,6 +822,118 @@ struct thread { *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \ } +#define JMP_CMP(thread, ip, operator) \ +{ \ + uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ + uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ + uint64_t a64 = *a64_ptr; \ + uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ + uint64_t a = a64 & a64_mask; \ + \ + uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ + uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ + uint64_t b64 = *b64_ptr; \ + uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ + uint64_t b = b64 & b64_mask; \ + \ + (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define JMP_CMP_S(thread, ip, operator) \ +{ \ + uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ + uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ + uint64_t a64 = *a64_ptr; \ + uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ + uint64_t a = a64 & a64_mask; \ + \ + uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ + uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ + uint64_t b64 = *b64_ptr; \ + uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ + \ + (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ +} + +#define JMP_CMP_MH JMP_CMP_S + +#define JMP_CMP_HM(thread, ip, operator) \ +{ \ + uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ + uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ + uint64_t a64 = *a64_ptr; \ + uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ + \ + uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ + uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ + uint64_t b64 = *b64_ptr; \ + uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \ + uint64_t b = b64 & b64_mask; \ + \ + (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ +} + +#define JMP_CMP_HH(thread, ip, operator) \ +{ \ + uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ + uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ + uint64_t a64 = *a64_ptr; \ + uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ + \ + uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \ + uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \ + uint64_t b64 = *b64_ptr; \ + uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \ + \ + (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ +} + +#else + +#define JMP_CMP_S JMP_CMP +#define JMP_CMP_MH JMP_CMP +#define JMP_CMP_HM JMP_CMP +#define JMP_CMP_HH JMP_CMP + +#endif + +#define JMP_CMP_I(thread, ip, operator) \ +{ \ + uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ + uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ + uint64_t a64 = *a64_ptr; \ + uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \ + uint64_t a = a64 & a64_mask; \ + \ + uint64_t b = (ip)->jmp.b_val; \ + \ + (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ +} + +#define JMP_CMP_MI JMP_CMP_I + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define JMP_CMP_HI(thread, ip, operator) \ +{ \ + uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \ + uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \ + uint64_t a64 = *a64_ptr; \ + uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \ + \ + uint64_t b = (ip)->jmp.b_val; \ + \ + (thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \ +} + +#else + +#define JMP_CMP_HI JMP_CMP_I + +#endif + #define METADATA_READ(thread, offset, n_bits) \ ({ \ uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \ @@ -1217,6 +1497,50 @@ extern_obj_find(struct rte_swx_pipeline *p, const char *name) return NULL; } +static struct extern_type_member_func * +extern_obj_member_func_parse(struct rte_swx_pipeline *p, + const char *name, + struct extern_obj **obj) +{ + struct extern_obj *object; + struct extern_type_member_func *func; + char *object_name, *func_name; + + if (name[0] != 'e' || name[1] != '.') + return NULL; + + object_name = strdup(&name[2]); + if (!object_name) + return NULL; + + func_name = strchr(object_name, '.'); + if (!func_name) { + free(object_name); + return NULL; + } + + *func_name = 0; + func_name++; + + object = extern_obj_find(p, object_name); + if (!object) { + free(object_name); + return NULL; + } + + func = extern_type_member_func_find(object->type, func_name); + if (!func) { + free(object_name); + return NULL; + } + + if (obj) + *obj = object; + + free(object_name); + return func; +} + static struct field * extern_obj_mailbox_field_parse(struct rte_swx_pipeline *p, const char *name, @@ -1499,6 +1823,16 @@ extern_func_find(struct rte_swx_pipeline *p, const char *name) return NULL; } +static struct extern_func * +extern_func_parse(struct rte_swx_pipeline *p, + const char *name) +{ + if (name[0] != 'f' || name[1] != '.') + return NULL; + + return extern_func_find(p, &name[2]); +} + static struct field * extern_func_mailbox_field_parse(struct rte_swx_pipeline *p, const char *name, @@ -1923,6 +2257,42 @@ metadata_free(struct rte_swx_pipeline *p) /* * Instruction. */ +static int +instruction_is_jmp(struct instruction *instr) +{ + switch (instr->type) { + case INSTR_JMP: + case INSTR_JMP_VALID: + case INSTR_JMP_INVALID: + case INSTR_JMP_HIT: + case INSTR_JMP_MISS: + case INSTR_JMP_ACTION_HIT: + case INSTR_JMP_ACTION_MISS: + case INSTR_JMP_EQ: + case INSTR_JMP_EQ_S: + case INSTR_JMP_EQ_I: + case INSTR_JMP_NEQ: + case INSTR_JMP_NEQ_S: + case INSTR_JMP_NEQ_I: + case INSTR_JMP_LT: + case INSTR_JMP_LT_MH: + case INSTR_JMP_LT_HM: + case INSTR_JMP_LT_HH: + case INSTR_JMP_LT_MI: + case INSTR_JMP_LT_HI: + case INSTR_JMP_GT: + case INSTR_JMP_GT_MH: + case INSTR_JMP_GT_HM: + case INSTR_JMP_GT_HH: + case INSTR_JMP_GT_MI: + case INSTR_JMP_GT_HI: + return 1; + + default: + return 0; + } +} + static struct field * action_field_parse(struct action *action, const char *name); @@ -2011,6 +2381,21 @@ thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t) t->ip = p->instructions; } +static inline void +thread_ip_set(struct thread *t, struct instruction *ip) +{ + t->ip = ip; +} + +static inline void +thread_ip_action_call(struct rte_swx_pipeline *p, + struct thread *t, + uint32_t action_id) +{ + t->ret = t->ip + 1; + t->ip = p->action_instructions[action_id]; +} + static inline void thread_ip_inc(struct rte_swx_pipeline *p); @@ -2034,6 +2419,12 @@ thread_yield(struct rte_swx_pipeline *p) p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); } +static inline void +thread_yield_cond(struct rte_swx_pipeline *p, int cond) +{ + p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1); +} + /* * rx. */ @@ -2624,6 +3015,167 @@ instr_hdr_invalidate_exec(struct rte_swx_pipeline *p) thread_ip_inc(p); } +/* + * table. + */ +static struct table * +table_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_table_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + struct table *t; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + t = table_find(p, tokens[1]); + CHECK(t, EINVAL); + + instr->type = INSTR_TABLE; + instr->table.table_id = t->id; + return 0; +} + +static inline void +instr_table_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint32_t table_id = ip->table.table_id; + struct rte_swx_table_state *ts = &t->table_state[table_id]; + struct table_runtime *table = &t->tables[table_id]; + uint64_t action_id; + uint8_t *action_data; + int done, hit; + + /* Table. */ + done = table->func(ts->obj, + table->mailbox, + table->key, + &action_id, + &action_data, + &hit); + if (!done) { + /* Thread. */ + TRACE("[Thread %2u] table %u (not finalized)\n", + p->thread_id, + table_id); + + thread_yield(p); + return; + } + + action_id = hit ? action_id : ts->default_action_id; + action_data = hit ? action_data : ts->default_action_data; + + TRACE("[Thread %2u] table %u (%s, action %u)\n", + p->thread_id, + table_id, + hit ? "hit" : "miss", + (uint32_t)action_id); + + t->action_id = action_id; + t->structs[0] = action_data; + t->hit = hit; + + /* Thread. */ + thread_ip_action_call(p, t, action_id); +} + +/* + * extern. + */ +static int +instr_extern_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + char *token = tokens[1]; + + CHECK(n_tokens == 2, EINVAL); + + if (token[0] == 'e') { + struct extern_obj *obj; + struct extern_type_member_func *func; + + func = extern_obj_member_func_parse(p, token, &obj); + CHECK(func, EINVAL); + + instr->type = INSTR_EXTERN_OBJ; + instr->ext_obj.ext_obj_id = obj->id; + instr->ext_obj.func_id = func->id; + + return 0; + } + + if (token[0] == 'f') { + struct extern_func *func; + + func = extern_func_parse(p, token); + CHECK(func, EINVAL); + + instr->type = INSTR_EXTERN_FUNC; + instr->ext_func.ext_func_id = func->id; + + return 0; + } + + CHECK(0, EINVAL); +} + +static inline void +instr_extern_obj_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint32_t obj_id = ip->ext_obj.ext_obj_id; + uint32_t func_id = ip->ext_obj.func_id; + struct extern_obj_runtime *obj = &t->extern_objs[obj_id]; + rte_swx_extern_type_member_func_t func = obj->funcs[func_id]; + + TRACE("[Thread %2u] extern obj %u member func %u\n", + p->thread_id, + obj_id, + func_id); + + /* Extern object member function execute. */ + uint32_t done = func(obj->obj, obj->mailbox); + + /* Thread. */ + thread_ip_inc_cond(t, done); + thread_yield_cond(p, done ^ 1); +} + +static inline void +instr_extern_func_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint32_t ext_func_id = ip->ext_func.ext_func_id; + struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id]; + rte_swx_extern_func_t func = ext_func->func; + + TRACE("[Thread %2u] extern func %u\n", + p->thread_id, + ext_func_id); + + /* Extern function execute. */ + uint32_t done = func(ext_func->mailbox); + + /* Thread. */ + thread_ip_inc_cond(t, done); + thread_yield_cond(p, done ^ 1); +} + /* * mov. */ @@ -3070,25 +3622,276 @@ instr_alu_cksub_translate(struct rte_swx_pipeline *p, return 0; } -static inline void -instr_alu_add_exec(struct rte_swx_pipeline *p) +static int +instr_alu_shl_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct thread *t = &p->threads[p->thread_id]; - struct instruction *ip = t->ip; + char *dst = tokens[1], *src = tokens[2]; + struct field *fdst, *fsrc; + uint32_t dst_struct_id, src_struct_id, src_val; - TRACE("[Thread %2u] add\n", p->thread_id); + CHECK(n_tokens == 3, EINVAL); - /* Structs. */ - ALU(t, ip, +); + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); - /* Thread. */ - thread_ip_inc(p); -} + /* SHL, SHL_HM, SHL_MH, SHL_HH. */ + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fsrc) { + instr->type = INSTR_ALU_SHL; + if (dst[0] == 'h' && src[0] == 'm') + instr->type = INSTR_ALU_SHL_HM; + if (dst[0] == 'm' && src[0] == 'h') + instr->type = INSTR_ALU_SHL_MH; + if (dst[0] == 'h' && src[0] == 'h') + instr->type = INSTR_ALU_SHL_HH; -static inline void -instr_alu_add_mh_exec(struct rte_swx_pipeline *p) -{ - struct thread *t = &p->threads[p->thread_id]; + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src.struct_id = (uint8_t)src_struct_id; + instr->alu.src.n_bits = fsrc->n_bits; + instr->alu.src.offset = fsrc->offset / 8; + return 0; + } + + /* SHL_MI, SHL_HI. */ + src_val = strtoul(src, &src, 0); + CHECK(!src[0], EINVAL); + + instr->type = INSTR_ALU_SHL_MI; + if (dst[0] == 'h') + instr->type = INSTR_ALU_SHL_HI; + + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src_val = (uint32_t)src_val; + return 0; +} + +static int +instr_alu_shr_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + char *dst = tokens[1], *src = tokens[2]; + struct field *fdst, *fsrc; + uint32_t dst_struct_id, src_struct_id, src_val; + + CHECK(n_tokens == 3, EINVAL); + + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); + + /* SHR, SHR_HM, SHR_MH, SHR_HH. */ + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fsrc) { + instr->type = INSTR_ALU_SHR; + if (dst[0] == 'h' && src[0] == 'm') + instr->type = INSTR_ALU_SHR_HM; + if (dst[0] == 'm' && src[0] == 'h') + instr->type = INSTR_ALU_SHR_MH; + if (dst[0] == 'h' && src[0] == 'h') + instr->type = INSTR_ALU_SHR_HH; + + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src.struct_id = (uint8_t)src_struct_id; + instr->alu.src.n_bits = fsrc->n_bits; + instr->alu.src.offset = fsrc->offset / 8; + return 0; + } + + /* SHR_MI, SHR_HI. */ + src_val = strtoul(src, &src, 0); + CHECK(!src[0], EINVAL); + + instr->type = INSTR_ALU_SHR_MI; + if (dst[0] == 'h') + instr->type = INSTR_ALU_SHR_HI; + + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src_val = (uint32_t)src_val; + return 0; +} + +static int +instr_alu_and_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + char *dst = tokens[1], *src = tokens[2]; + struct field *fdst, *fsrc; + uint32_t dst_struct_id, src_struct_id, src_val; + + CHECK(n_tokens == 3, EINVAL); + + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); + + /* AND or AND_S. */ + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fsrc) { + instr->type = INSTR_ALU_AND; + if ((dst[0] == 'h' && src[0] != 'h') || + (dst[0] != 'h' && src[0] == 'h')) + instr->type = INSTR_ALU_AND_S; + + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src.struct_id = (uint8_t)src_struct_id; + instr->alu.src.n_bits = fsrc->n_bits; + instr->alu.src.offset = fsrc->offset / 8; + return 0; + } + + /* AND_I. */ + src_val = strtoul(src, &src, 0); + CHECK(!src[0], EINVAL); + + if (dst[0] == 'h') + src_val = htonl(src_val); + + instr->type = INSTR_ALU_AND_I; + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src_val = (uint32_t)src_val; + return 0; +} + +static int +instr_alu_or_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + char *dst = tokens[1], *src = tokens[2]; + struct field *fdst, *fsrc; + uint32_t dst_struct_id, src_struct_id, src_val; + + CHECK(n_tokens == 3, EINVAL); + + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); + + /* OR or OR_S. */ + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fsrc) { + instr->type = INSTR_ALU_OR; + if ((dst[0] == 'h' && src[0] != 'h') || + (dst[0] != 'h' && src[0] == 'h')) + instr->type = INSTR_ALU_OR_S; + + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src.struct_id = (uint8_t)src_struct_id; + instr->alu.src.n_bits = fsrc->n_bits; + instr->alu.src.offset = fsrc->offset / 8; + return 0; + } + + /* OR_I. */ + src_val = strtoul(src, &src, 0); + CHECK(!src[0], EINVAL); + + if (dst[0] == 'h') + src_val = htonl(src_val); + + instr->type = INSTR_ALU_OR_I; + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src_val = (uint32_t)src_val; + return 0; +} + +static int +instr_alu_xor_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + char *dst = tokens[1], *src = tokens[2]; + struct field *fdst, *fsrc; + uint32_t dst_struct_id, src_struct_id, src_val; + + CHECK(n_tokens == 3, EINVAL); + + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); + + /* XOR or XOR_S. */ + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fsrc) { + instr->type = INSTR_ALU_XOR; + if ((dst[0] == 'h' && src[0] != 'h') || + (dst[0] != 'h' && src[0] == 'h')) + instr->type = INSTR_ALU_XOR_S; + + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src.struct_id = (uint8_t)src_struct_id; + instr->alu.src.n_bits = fsrc->n_bits; + instr->alu.src.offset = fsrc->offset / 8; + return 0; + } + + /* XOR_I. */ + src_val = strtoul(src, &src, 0); + CHECK(!src[0], EINVAL); + + if (dst[0] == 'h') + src_val = htonl(src_val); + + instr->type = INSTR_ALU_XOR_I; + instr->alu.dst.struct_id = (uint8_t)dst_struct_id; + instr->alu.dst.n_bits = fdst->n_bits; + instr->alu.dst.offset = fdst->offset / 8; + instr->alu.src_val = (uint32_t)src_val; + return 0; +} + +static inline void +instr_alu_add_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] add\n", p->thread_id); + + /* Structs. */ + ALU(t, ip, +); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_add_mh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; TRACE("[Thread %2u] add (mh)\n", p->thread_id); @@ -3251,75 +4054,331 @@ instr_alu_sub_hi_exec(struct rte_swx_pipeline *p) } static inline void -instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p) +instr_alu_shl_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint8_t *dst_struct, *src_struct; - uint16_t *dst16_ptr, dst; - uint64_t *src64_ptr, src64, src64_mask, src; - uint64_t r; - TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); + TRACE("[Thread %2u] shl\n", p->thread_id); /* Structs. */ - dst_struct = t->structs[ip->alu.dst.struct_id]; - dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; - dst = *dst16_ptr; - - src_struct = t->structs[ip->alu.src.struct_id]; - src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; - src64 = *src64_ptr; - src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); - src = src64 & src64_mask; - - r = dst; - r = ~r & 0xFFFF; - - /* The first input (r) is a 16-bit number. The second and the third - * inputs are 32-bit numbers. In the worst case scenario, the sum of the - * three numbers (output r) is a 34-bit number. - */ - r += (src >> 32) + (src & 0xFFFFFFFF); - - /* The first input is a 16-bit number. The second input is an 18-bit - * number. In the worst case scenario, the sum of the two numbers is a - * 19-bit number. - */ - r = (r & 0xFFFF) + (r >> 16); - - /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is - * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). - */ - r = (r & 0xFFFF) + (r >> 16); - - /* When the input r is (0 .. 0xFFFF), the output r is equal to the input - * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. - * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, - * therefore the output r is always a 16-bit number. - */ - r = (r & 0xFFFF) + (r >> 16); - - r = ~r & 0xFFFF; - r = r ? r : 0xFFFF; - - *dst16_ptr = (uint16_t)r; + ALU(t, ip, <<); /* Thread. */ thread_ip_inc(p); } static inline void -instr_alu_cksub_field_exec(struct rte_swx_pipeline *p) +instr_alu_shl_mh_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint8_t *dst_struct, *src_struct; + + TRACE("[Thread %2u] shl (mh)\n", p->thread_id); + + /* Structs. */ + ALU_MH(t, ip, <<); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shl_hm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shl (hm)\n", p->thread_id); + + /* Structs. */ + ALU_HM(t, ip, <<); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shl_hh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shl (hh)\n", p->thread_id); + + /* Structs. */ + ALU_HH(t, ip, <<); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shl_mi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shl (mi)\n", p->thread_id); + + /* Structs. */ + ALU_MI(t, ip, <<); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shl_hi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shl (hi)\n", p->thread_id); + + /* Structs. */ + ALU_HI(t, ip, <<); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shr_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shr\n", p->thread_id); + + /* Structs. */ + ALU(t, ip, >>); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shr_mh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shr (mh)\n", p->thread_id); + + /* Structs. */ + ALU_MH(t, ip, >>); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shr_hm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shr (hm)\n", p->thread_id); + + /* Structs. */ + ALU_HM(t, ip, >>); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shr_hh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shr (hh)\n", p->thread_id); + + /* Structs. */ + ALU_HH(t, ip, >>); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shr_mi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shr (mi)\n", p->thread_id); + + /* Structs. */ + ALU_MI(t, ip, >>); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_shr_hi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] shr (hi)\n", p->thread_id); + + /* Structs. */ + ALU_HI(t, ip, >>); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_and_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] and\n", p->thread_id); + + /* Structs. */ + ALU(t, ip, &); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_and_s_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] and (s)\n", p->thread_id); + + /* Structs. */ + ALU_S(t, ip, &); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_and_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] and (i)\n", p->thread_id); + + /* Structs. */ + ALU_I(t, ip, &); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_or_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] or\n", p->thread_id); + + /* Structs. */ + ALU(t, ip, |); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_or_s_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] or (s)\n", p->thread_id); + + /* Structs. */ + ALU_S(t, ip, |); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_or_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] or (i)\n", p->thread_id); + + /* Structs. */ + ALU_I(t, ip, |); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_xor_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] xor\n", p->thread_id); + + /* Structs. */ + ALU(t, ip, ^); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_xor_s_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] xor (s)\n", p->thread_id); + + /* Structs. */ + ALU_S(t, ip, ^); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_xor_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] xor (i)\n", p->thread_id); + + /* Structs. */ + ALU_I(t, ip, ^); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint8_t *dst_struct, *src_struct; uint16_t *dst16_ptr, dst; uint64_t *src64_ptr, src64, src64_mask, src; uint64_t r; - TRACE("[Thread %2u] cksub (field)\n", p->thread_id); + TRACE("[Thread %2u] ckadd (field)\n", p->thread_id); /* Structs. */ dst_struct = t->structs[ip->alu.dst.struct_id]; @@ -3335,153 +4394,890 @@ instr_alu_cksub_field_exec(struct rte_swx_pipeline *p) r = dst; r = ~r & 0xFFFF; - /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as - * the following sequence of operations in 2's complement arithmetic: - * a '- b = (a - b) % 0xFFFF. - * - * In order to prevent an underflow for the below subtraction, in which - * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the - * minuend), we first add a multiple of the 0xFFFF modulus to the - * minuend. The number we add to the minuend needs to be a 34-bit number - * or higher, so for readability reasons we picked the 36-bit multiple. - * We are effectively turning the 16-bit minuend into a 36-bit number: - * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. - */ + /* The first input (r) is a 16-bit number. The second and the third + * inputs are 32-bit numbers. In the worst case scenario, the sum of the + * three numbers (output r) is a 34-bit number. + */ + r += (src >> 32) + (src & 0xFFFFFFFF); + + /* The first input is a 16-bit number. The second input is an 18-bit + * number. In the worst case scenario, the sum of the two numbers is a + * 19-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x10006), the output r is (0 .. 7). So no carry bit can be generated, + * therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_cksub_field_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr, dst; + uint64_t *src64_ptr, src64, src64_mask, src; + uint64_t r; + + TRACE("[Thread %2u] cksub (field)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + dst = *dst16_ptr; + + src_struct = t->structs[ip->alu.src.struct_id]; + src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset]; + src64 = *src64_ptr; + src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits); + src = src64 & src64_mask; + + r = dst; + r = ~r & 0xFFFF; + + /* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as + * the following sequence of operations in 2's complement arithmetic: + * a '- b = (a - b) % 0xFFFF. + * + * In order to prevent an underflow for the below subtraction, in which + * a 33-bit number (the subtrahend) is taken out of a 16-bit number (the + * minuend), we first add a multiple of the 0xFFFF modulus to the + * minuend. The number we add to the minuend needs to be a 34-bit number + * or higher, so for readability reasons we picked the 36-bit multiple. + * We are effectively turning the 16-bit minuend into a 36-bit number: + * (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF. + */ r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */ - /* A 33-bit number is subtracted from a 36-bit number (the input r). The - * result (the output r) is a 36-bit number. - */ - r -= (src >> 32) + (src & 0xFFFFFFFF); + /* A 33-bit number is subtracted from a 36-bit number (the input r). The + * result (the output r) is a 36-bit number. + */ + r -= (src >> 32) + (src & 0xFFFFFFFF); + + /* The first input is a 16-bit number. The second input is a 20-bit + * number. Their sum is a 21-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1001E), the output r is (0 .. 31). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr; + uint32_t *src32_ptr; + uint64_t r0, r1; + + TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + + src_struct = t->structs[ip->alu.src.struct_id]; + src32_ptr = (uint32_t *)&src_struct[0]; + + r0 = src32_ptr[0]; /* r0 is a 32-bit number. */ + r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ + r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */ + r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ + r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ + + /* The first input is a 16-bit number. The second input is a 19-bit + * number. Their sum is a 20-bit number. + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1000E), the output r is (0 .. 15). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r0 = (r0 & 0xFFFF) + (r0 >> 16); + + r0 = ~r0 & 0xFFFF; + r0 = r0 ? r0 : 0xFFFF; + + *dst16_ptr = (uint16_t)r0; + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint8_t *dst_struct, *src_struct; + uint16_t *dst16_ptr; + uint32_t *src32_ptr; + uint64_t r = 0; + uint32_t i; + + TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); + + /* Structs. */ + dst_struct = t->structs[ip->alu.dst.struct_id]; + dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + + src_struct = t->structs[ip->alu.src.struct_id]; + src32_ptr = (uint32_t *)&src_struct[0]; + + /* The max number of 32-bit words in a 256-byte header is 8 = 2^3. + * Therefore, in the worst case scenario, a 35-bit number is added to a + * 16-bit number (the input r), so the output r is 36-bit number. + */ + for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++) + r += *src32_ptr; + + /* The first input is a 16-bit number. The second input is a 20-bit + * number. Their sum is a 21-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is + * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E). + */ + r = (r & 0xFFFF) + (r >> 16); + + /* When the input r is (0 .. 0xFFFF), the output r is equal to the input + * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. + * 0x1001E), the output r is (0 .. 31). So no carry bit can be + * generated, therefore the output r is always a 16-bit number. + */ + r = (r & 0xFFFF) + (r >> 16); + + r = ~r & 0xFFFF; + r = r ? r : 0xFFFF; + + *dst16_ptr = (uint16_t)r; + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint32_t a_struct_id, b_struct_id, b_val; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoul(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = htonl(b_val); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = (uint32_t)b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint32_t a_struct_id, b_struct_id, b_val; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_NEQ or JMP_NEQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_NEQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_NEQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_NEQ_I. */ + b_val = strtoul(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = htonl(b_val); + + instr->type = INSTR_JMP_NEQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = (uint32_t)b_val; + return 0; +} + +static int +instr_jmp_lt_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint32_t a_struct_id, b_struct_id, b_val; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_LT, JMP_LT_MH, JMP_LT_HM, JMP_LT_HH. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_LT; + if (a[0] == 'h' && b[0] == 'm') + instr->type = INSTR_JMP_LT_HM; + if (a[0] == 'm' && b[0] == 'h') + instr->type = INSTR_JMP_LT_MH; + if (a[0] == 'h' && b[0] == 'h') + instr->type = INSTR_JMP_LT_HH; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_LT_MI, JMP_LT_HI. */ + b_val = strtoul(b, &b, 0); + CHECK(!b[0], EINVAL); + + instr->type = INSTR_JMP_LT_MI; + if (a[0] == 'h') + instr->type = INSTR_JMP_LT_HI; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = (uint32_t)b_val; + return 0; +} + +static int +instr_jmp_gt_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint32_t a_struct_id, b_struct_id, b_val; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_GT, JMP_GT_MH, JMP_GT_HM, JMP_GT_HH. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_GT; + if (a[0] == 'h' && b[0] == 'm') + instr->type = INSTR_JMP_GT_HM; + if (a[0] == 'm' && b[0] == 'h') + instr->type = INSTR_JMP_GT_MH; + if (a[0] == 'h' && b[0] == 'h') + instr->type = INSTR_JMP_GT_HH; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_GT_MI, JMP_GT_HI. */ + b_val = strtoul(b, &b, 0); + CHECK(!b[0], EINVAL); + + instr->type = INSTR_JMP_GT_MI; + if (a[0] == 'h') + instr->type = INSTR_JMP_GT_HI; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = (uint32_t)b_val; + return 0; +} + +static inline void +instr_jmp_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmp\n", p->thread_id); + + thread_ip_set(t, ip->jmp.ip); +} + +static inline void +instr_jmp_valid_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint32_t header_id = ip->jmp.header_id; + + TRACE("[Thread %2u] jmpv\n", p->thread_id); + + t->ip = HEADER_VALID(t, header_id) ? ip->jmp.ip : (t->ip + 1); +} + +static inline void +instr_jmp_invalid_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + uint32_t header_id = ip->jmp.header_id; + + TRACE("[Thread %2u] jmpnv\n", p->thread_id); + + t->ip = HEADER_VALID(t, header_id) ? (t->ip + 1) : ip->jmp.ip; +} + +static inline void +instr_jmp_hit_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct instruction *ip_next[] = {t->ip + 1, ip->jmp.ip}; + + TRACE("[Thread %2u] jmph\n", p->thread_id); + + t->ip = ip_next[t->hit]; +} + +static inline void +instr_jmp_miss_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + struct instruction *ip_next[] = {ip->jmp.ip, t->ip + 1}; + + TRACE("[Thread %2u] jmpnh\n", p->thread_id); + + t->ip = ip_next[t->hit]; +} + +static inline void +instr_jmp_action_hit_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpa\n", p->thread_id); + + t->ip = (ip->jmp.action_id == t->action_id) ? ip->jmp.ip : (t->ip + 1); +} + +static inline void +instr_jmp_action_miss_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpna\n", p->thread_id); + + t->ip = (ip->jmp.action_id == t->action_id) ? (t->ip + 1) : ip->jmp.ip; +} + +static inline void +instr_jmp_eq_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpeq\n", p->thread_id); + + JMP_CMP(t, ip, ==); +} + +static inline void +instr_jmp_eq_s_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpeq (s)\n", p->thread_id); + + JMP_CMP_S(t, ip, ==); +} + +static inline void +instr_jmp_eq_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpeq (i)\n", p->thread_id); + + JMP_CMP_I(t, ip, ==); +} + +static inline void +instr_jmp_neq_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpneq\n", p->thread_id); + + JMP_CMP(t, ip, !=); +} + +static inline void +instr_jmp_neq_s_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpneq (s)\n", p->thread_id); + + JMP_CMP_S(t, ip, !=); +} + +static inline void +instr_jmp_neq_i_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmpneq (i)\n", p->thread_id); + + JMP_CMP_I(t, ip, !=); +} + +static inline void +instr_jmp_lt_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmplt\n", p->thread_id); + + JMP_CMP(t, ip, <); +} + +static inline void +instr_jmp_lt_mh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmplt (mh)\n", p->thread_id); + + JMP_CMP_MH(t, ip, <); +} + +static inline void +instr_jmp_lt_hm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmplt (hm)\n", p->thread_id); + + JMP_CMP_HM(t, ip, <); +} + +static inline void +instr_jmp_lt_hh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] jmplt (hh)\n", p->thread_id); + + JMP_CMP_HH(t, ip, <); +} - /* The first input is a 16-bit number. The second input is a 20-bit - * number. Their sum is a 21-bit number. - */ - r = (r & 0xFFFF) + (r >> 16); +static inline void +instr_jmp_lt_mi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; - /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is - * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E). - */ - r = (r & 0xFFFF) + (r >> 16); + TRACE("[Thread %2u] jmplt (mi)\n", p->thread_id); - /* When the input r is (0 .. 0xFFFF), the output r is equal to the input - * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. - * 0x1001E), the output r is (0 .. 31). So no carry bit can be - * generated, therefore the output r is always a 16-bit number. - */ - r = (r & 0xFFFF) + (r >> 16); + JMP_CMP_MI(t, ip, <); +} - r = ~r & 0xFFFF; - r = r ? r : 0xFFFF; +static inline void +instr_jmp_lt_hi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; - *dst16_ptr = (uint16_t)r; + TRACE("[Thread %2u] jmplt (hi)\n", p->thread_id); - /* Thread. */ - thread_ip_inc(p); + JMP_CMP_HI(t, ip, <); } static inline void -instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p) +instr_jmp_gt_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint8_t *dst_struct, *src_struct; - uint16_t *dst16_ptr; - uint32_t *src32_ptr; - uint64_t r0, r1; - TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id); + TRACE("[Thread %2u] jmpgt\n", p->thread_id); - /* Structs. */ - dst_struct = t->structs[ip->alu.dst.struct_id]; - dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + JMP_CMP(t, ip, >); +} - src_struct = t->structs[ip->alu.src.struct_id]; - src32_ptr = (uint32_t *)&src_struct[0]; +static inline void +instr_jmp_gt_mh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; - r0 = src32_ptr[0]; /* r0 is a 32-bit number. */ - r1 = src32_ptr[1]; /* r1 is a 32-bit number. */ - r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */ - r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */ - r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */ + TRACE("[Thread %2u] jmpgt (mh)\n", p->thread_id); - /* The first input is a 16-bit number. The second input is a 19-bit - * number. Their sum is a 20-bit number. - */ - r0 = (r0 & 0xFFFF) + (r0 >> 16); + JMP_CMP_MH(t, ip, >); +} - /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is - * a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E). - */ - r0 = (r0 & 0xFFFF) + (r0 >> 16); +static inline void +instr_jmp_gt_hm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; - /* When the input r is (0 .. 0xFFFF), the output r is equal to the input - * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. - * 0x1000E), the output r is (0 .. 15). So no carry bit can be - * generated, therefore the output r is always a 16-bit number. - */ - r0 = (r0 & 0xFFFF) + (r0 >> 16); + TRACE("[Thread %2u] jmpgt (hm)\n", p->thread_id); - r0 = ~r0 & 0xFFFF; - r0 = r0 ? r0 : 0xFFFF; + JMP_CMP_HM(t, ip, >); +} - *dst16_ptr = (uint16_t)r0; +static inline void +instr_jmp_gt_hh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; - /* Thread. */ - thread_ip_inc(p); + TRACE("[Thread %2u] jmpgt (hh)\n", p->thread_id); + + JMP_CMP_HH(t, ip, >); } static inline void -instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) +instr_jmp_gt_mi_exec(struct rte_swx_pipeline *p) { struct thread *t = &p->threads[p->thread_id]; struct instruction *ip = t->ip; - uint8_t *dst_struct, *src_struct; - uint16_t *dst16_ptr; - uint32_t *src32_ptr; - uint64_t r = 0; - uint32_t i; - TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id); + TRACE("[Thread %2u] jmpgt (mi)\n", p->thread_id); - /* Structs. */ - dst_struct = t->structs[ip->alu.dst.struct_id]; - dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset]; + JMP_CMP_MI(t, ip, >); +} - src_struct = t->structs[ip->alu.src.struct_id]; - src32_ptr = (uint32_t *)&src_struct[0]; +static inline void +instr_jmp_gt_hi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; - /* The max number of 32-bit words in a 256-byte header is 8 = 2^3. - * Therefore, in the worst case scenario, a 35-bit number is added to a - * 16-bit number (the input r), so the output r is 36-bit number. - */ - for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++) - r += *src32_ptr; + TRACE("[Thread %2u] jmpgt (hi)\n", p->thread_id); - /* The first input is a 16-bit number. The second input is a 20-bit - * number. Their sum is a 21-bit number. - */ - r = (r & 0xFFFF) + (r >> 16); + JMP_CMP_HI(t, ip, >); +} - /* The first input is a 16-bit number (0 .. 0xFFFF). The second input is - * a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E). - */ - r = (r & 0xFFFF) + (r >> 16); +/* + * return. + */ +static int +instr_return_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens __rte_unused, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) +{ + CHECK(action, EINVAL); + CHECK(n_tokens == 1, EINVAL); - /* When the input r is (0 .. 0xFFFF), the output r is equal to the input - * r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 .. - * 0x1001E), the output r is (0 .. 31). So no carry bit can be - * generated, therefore the output r is always a 16-bit number. - */ - r = (r & 0xFFFF) + (r >> 16); + instr->type = INSTR_RETURN; + return 0; +} - r = ~r & 0xFFFF; - r = r ? r : 0xFFFF; +static inline void +instr_return_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; - *dst16_ptr = (uint16_t)r; + TRACE("[Thread %2u] return\n", p->thread_id); - /* Thread. */ - thread_ip_inc(p); + t->ip = t->ret; } #define RTE_SWX_INSTRUCTION_TOKENS_MAX 16 @@ -3617,9 +5413,173 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "and")) + return instr_alu_and_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "or")) + return instr_alu_or_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "xor")) + return instr_alu_xor_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "shl")) + return instr_alu_shl_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "shr")) + return instr_alu_shr_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "table")) + return instr_table_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "extern")) + return instr_extern_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmp")) + return instr_jmp_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpv")) + return instr_jmp_valid_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpnv")) + return instr_jmp_invalid_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmph")) + return instr_jmp_hit_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpnh")) + return instr_jmp_miss_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpa")) + return instr_jmp_action_hit_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpna")) + return instr_jmp_action_miss_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpeq")) + return instr_jmp_eq_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpneq")) + return instr_jmp_neq_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmplt")) + return instr_jmp_lt_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "jmpgt")) + return instr_jmp_gt_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "return")) + return instr_return_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + CHECK(0, EINVAL); } +static struct instruction_data * +label_find(struct instruction_data *data, uint32_t n, const char *label) +{ + uint32_t i; + + for (i = 0; i < n; i++) + if (!strcmp(label, data[i].label)) + return &data[i]; + + return NULL; +} + static uint32_t label_is_used(struct instruction_data *data, uint32_t n, const char *label) { @@ -3667,6 +5627,79 @@ instr_label_check(struct instruction_data *instruction_data, return 0; } +static int +instr_jmp_resolve(struct instruction *instructions, + struct instruction_data *instruction_data, + uint32_t n_instructions) +{ + uint32_t i; + + for (i = 0; i < n_instructions; i++) { + struct instruction *instr = &instructions[i]; + struct instruction_data *data = &instruction_data[i]; + struct instruction_data *found; + + if (!instruction_is_jmp(instr)) + continue; + + found = label_find(instruction_data, + n_instructions, + data->jmp_label); + CHECK(found, EINVAL); + + instr->jmp.ip = &instr[found - instruction_data]; + } + + return 0; +} + +static int +instr_verify(struct rte_swx_pipeline *p __rte_unused, + struct action *a, + struct instruction *instr, + struct instruction_data *data __rte_unused, + uint32_t n_instructions) +{ + if (!a) { + enum instruction_type type; + uint32_t i; + + /* Check that the first instruction is rx. */ + CHECK(instr[0].type == INSTR_RX, EINVAL); + + /* Check that there is at least one tx instruction. */ + for (i = 0; i < n_instructions; i++) { + type = instr[i].type; + + if (instr[i].type == INSTR_TX) + break; + } + CHECK(i < n_instructions, EINVAL); + + /* Check that the last instruction is either tx or unconditional + * jump. + */ + type = instr[n_instructions - 1].type; + CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL); + } + + if (a) { + enum instruction_type type; + uint32_t i; + + /* Check that there is at least one return or tx instruction. */ + for (i = 0; i < n_instructions; i++) { + type = instr[i].type; + + if ((type == INSTR_RETURN) || (type == INSTR_TX)) + break; + } + CHECK(i < n_instructions, EINVAL); + } + + return 0; +} + static int instruction_config(struct rte_swx_pipeline *p, struct action *a, @@ -3715,6 +5748,14 @@ instruction_config(struct rte_swx_pipeline *p, if (err) goto error; + err = instr_verify(p, a, instr, data, n_instructions); + if (err) + goto error; + + err = instr_jmp_resolve(instr, data, n_instructions); + if (err) + goto error; + free(data); if (a) { @@ -3793,6 +5834,68 @@ static instr_exec_t instruction_table[] = { [INSTR_ALU_CKADD_STRUCT] = instr_alu_ckadd_struct_exec, [INSTR_ALU_CKADD_STRUCT20] = instr_alu_ckadd_struct20_exec, [INSTR_ALU_CKSUB_FIELD] = instr_alu_cksub_field_exec, + + [INSTR_ALU_AND] = instr_alu_and_exec, + [INSTR_ALU_AND_S] = instr_alu_and_s_exec, + [INSTR_ALU_AND_I] = instr_alu_and_i_exec, + + [INSTR_ALU_OR] = instr_alu_or_exec, + [INSTR_ALU_OR_S] = instr_alu_or_s_exec, + [INSTR_ALU_OR_I] = instr_alu_or_i_exec, + + [INSTR_ALU_XOR] = instr_alu_xor_exec, + [INSTR_ALU_XOR_S] = instr_alu_xor_s_exec, + [INSTR_ALU_XOR_I] = instr_alu_xor_i_exec, + + [INSTR_ALU_SHL] = instr_alu_shl_exec, + [INSTR_ALU_SHL_MH] = instr_alu_shl_mh_exec, + [INSTR_ALU_SHL_HM] = instr_alu_shl_hm_exec, + [INSTR_ALU_SHL_HH] = instr_alu_shl_hh_exec, + [INSTR_ALU_SHL_MI] = instr_alu_shl_mi_exec, + [INSTR_ALU_SHL_HI] = instr_alu_shl_hi_exec, + + [INSTR_ALU_SHR] = instr_alu_shr_exec, + [INSTR_ALU_SHR_MH] = instr_alu_shr_mh_exec, + [INSTR_ALU_SHR_HM] = instr_alu_shr_hm_exec, + [INSTR_ALU_SHR_HH] = instr_alu_shr_hh_exec, + [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec, + [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec, + + [INSTR_TABLE] = instr_table_exec, + [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, + [INSTR_EXTERN_FUNC] = instr_extern_func_exec, + + [INSTR_JMP] = instr_jmp_exec, + [INSTR_JMP_VALID] = instr_jmp_valid_exec, + [INSTR_JMP_INVALID] = instr_jmp_invalid_exec, + [INSTR_JMP_HIT] = instr_jmp_hit_exec, + [INSTR_JMP_MISS] = instr_jmp_miss_exec, + [INSTR_JMP_ACTION_HIT] = instr_jmp_action_hit_exec, + [INSTR_JMP_ACTION_MISS] = instr_jmp_action_miss_exec, + + [INSTR_JMP_EQ] = instr_jmp_eq_exec, + [INSTR_JMP_EQ_S] = instr_jmp_eq_s_exec, + [INSTR_JMP_EQ_I] = instr_jmp_eq_i_exec, + + [INSTR_JMP_NEQ] = instr_jmp_neq_exec, + [INSTR_JMP_NEQ_S] = instr_jmp_neq_s_exec, + [INSTR_JMP_NEQ_I] = instr_jmp_neq_i_exec, + + [INSTR_JMP_LT] = instr_jmp_lt_exec, + [INSTR_JMP_LT_MH] = instr_jmp_lt_mh_exec, + [INSTR_JMP_LT_HM] = instr_jmp_lt_hm_exec, + [INSTR_JMP_LT_HH] = instr_jmp_lt_hh_exec, + [INSTR_JMP_LT_MI] = instr_jmp_lt_mi_exec, + [INSTR_JMP_LT_HI] = instr_jmp_lt_hi_exec, + + [INSTR_JMP_GT] = instr_jmp_gt_exec, + [INSTR_JMP_GT_MH] = instr_jmp_gt_mh_exec, + [INSTR_JMP_GT_HM] = instr_jmp_gt_hm_exec, + [INSTR_JMP_GT_HH] = instr_jmp_gt_hh_exec, + [INSTR_JMP_GT_MI] = instr_jmp_gt_mi_exec, + [INSTR_JMP_GT_HI] = instr_jmp_gt_hi_exec, + + [INSTR_RETURN] = instr_return_exec, }; static inline void