#include <stdio.h>
#include <errno.h>
#include <arpa/inet.h>
+#include <dlfcn.h>
#include "rte_swx_pipeline_internal.h"
}
}
+static int
+instruction_does_thread_yield(struct instruction *instr)
+{
+ switch (instr->type) {
+ case INSTR_RX:
+ case INSTR_TABLE:
+ case INSTR_TABLE_AF:
+ case INSTR_SELECTOR:
+ case INSTR_LEARNER:
+ case INSTR_LEARNER_AF:
+ case INSTR_EXTERN_OBJ:
+ case INSTR_EXTERN_FUNC:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
static struct field *
action_field_parse(struct action *action, const char *name);
static int
action_has_nbo_args(struct action *a);
+static int
+learner_action_args_check(struct rte_swx_pipeline *p, struct action *a, const char *mf_name);
+
static int
instr_learn_translate(struct rte_swx_pipeline *p,
struct action *action,
struct instruction_data *data __rte_unused)
{
struct action *a;
+ const char *mf_name;
+ uint32_t mf_offset = 0;
CHECK(action, EINVAL);
- CHECK(n_tokens == 2, EINVAL);
+ CHECK((n_tokens == 2) || (n_tokens == 3), EINVAL);
a = action_find(p, tokens[1]);
CHECK(a, EINVAL);
CHECK(!action_has_nbo_args(a), EINVAL);
+ mf_name = (n_tokens > 2) ? tokens[2] : NULL;
+ CHECK(!learner_action_args_check(p, a, mf_name), EINVAL);
+
+ if (mf_name) {
+ struct field *mf;
+
+ mf = metadata_field_parse(p, mf_name);
+ CHECK(mf, EINVAL);
+
+ mf_offset = mf->offset / 8;
+ }
+
instr->type = INSTR_LEARNER_LEARN;
instr->learn.action_id = a->id;
+ instr->learn.mf_offset = mf_offset;
return 0;
}
instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
instr->meter.color_out.n_bits = fcout->n_bits;
instr->meter.color_out.offset = fcout->offset / 8;
-
- return 0;
}
/* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
instr->meter.color_out.n_bits = fcout->n_bits;
instr->meter.color_out.offset = fcout->offset / 8;
-
- return 0;
}
/* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
instr->meter.color_out.n_bits = fcout->n_bits;
instr->meter.color_out.offset = fcout->offset / 8;
-
- return 0;
}
/* index = I, length = HMEFT, color_in = I, color_out = MEF. */
instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
instr->meter.color_out.n_bits = fcout->n_bits;
instr->meter.color_out.offset = fcout->offset / 8;
-
- return 0;
}
- CHECK(0, EINVAL);
+ return 0;
}
static inline void
instr,
data);
- CHECK(0, EINVAL);
+ return -EINVAL;
}
static struct instruction_data *
continue;
for (j = i + 1; j < n_instructions; j++)
- CHECK(strcmp(label, data[j].label), EINVAL);
+ CHECK(strcmp(label, instruction_data[j].label), EINVAL);
}
/* Get users for each instruction label. */
uint32_t size)
{
struct table_type *type;
- struct table *t;
+ struct table *t = NULL;
struct action *default_action;
struct header *header = NULL;
uint32_t action_data_size_max = 0, i;
const char *action_name = params->action_names[i];
struct action *a;
uint32_t action_data_size;
+ int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
CHECK_NAME(action_name, EINVAL);
action_data_size = a->st ? a->st->n_bits / 8 : 0;
if (action_data_size > action_data_size_max)
action_data_size_max = action_data_size;
+
+ if (params->action_is_for_table_entries)
+ action_is_for_table_entries = params->action_is_for_table_entries[i];
+ if (params->action_is_for_default_entry)
+ action_is_for_default_entry = params->action_is_for_default_entry[i];
+ CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
}
CHECK_NAME(params->default_action_name, EINVAL);
params->default_action_name))
break;
CHECK(i < params->n_actions, EINVAL);
+ CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
+ EINVAL);
+
default_action = action_find(p, params->default_action_name);
CHECK((default_action->st && params->default_action_data) ||
!params->default_action_data, EINVAL);
/* Memory allocation. */
t = calloc(1, sizeof(struct table));
- CHECK(t, ENOMEM);
+ if (!t)
+ goto nomem;
t->fields = calloc(params->n_fields, sizeof(struct match_field));
- if (!t->fields) {
- free(t);
- CHECK(0, ENOMEM);
- }
+ if (!t->fields)
+ goto nomem;
t->actions = calloc(params->n_actions, sizeof(struct action *));
- if (!t->actions) {
- free(t->fields);
- free(t);
- CHECK(0, ENOMEM);
- }
+ if (!t->actions)
+ goto nomem;
if (action_data_size_max) {
t->default_action_data = calloc(1, action_data_size_max);
- if (!t->default_action_data) {
- free(t->actions);
- free(t->fields);
- free(t);
- CHECK(0, ENOMEM);
- }
+ if (!t->default_action_data)
+ goto nomem;
}
+ t->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
+ if (!t->action_is_for_table_entries)
+ goto nomem;
+
+ t->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
+ if (!t->action_is_for_default_entry)
+ goto nomem;
+
/* Node initialization. */
strcpy(t->name, name);
if (args && args[0])
t->n_fields = params->n_fields;
t->header = header;
- for (i = 0; i < params->n_actions; i++)
+ for (i = 0; i < params->n_actions; i++) {
+ int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
+
+ if (params->action_is_for_table_entries)
+ action_is_for_table_entries = params->action_is_for_table_entries[i];
+ if (params->action_is_for_default_entry)
+ action_is_for_default_entry = params->action_is_for_default_entry[i];
+
t->actions[i] = action_find(p, params->action_names[i]);
+ t->action_is_for_table_entries[i] = action_is_for_table_entries;
+ t->action_is_for_default_entry[i] = action_is_for_default_entry;
+ }
t->default_action = default_action;
if (default_action->st)
memcpy(t->default_action_data,
p->n_tables++;
return 0;
+
+nomem:
+ if (!t)
+ return -ENOMEM;
+
+ free(t->action_is_for_default_entry);
+ free(t->action_is_for_table_entries);
+ free(t->default_action_data);
+ free(t->actions);
+ free(t->fields);
+ free(t);
+
+ return -ENOMEM;
}
static struct rte_swx_table_params *
/* Action checks. */
CHECK(params->n_actions, EINVAL);
-
CHECK(params->action_names, EINVAL);
for (i = 0; i < params->n_actions; i++) {
const char *action_name = params->action_names[i];
- const char *action_field_name = params->action_field_names[i];
struct action *a;
uint32_t action_data_size;
+ int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
CHECK_NAME(action_name, EINVAL);
a = action_find(p, action_name);
CHECK(a, EINVAL);
- status = learner_action_args_check(p, a, action_field_name);
- if (status)
- return status;
-
status = learner_action_learning_check(p,
a,
params->action_names,
action_data_size = a->st ? a->st->n_bits / 8 : 0;
if (action_data_size > action_data_size_max)
action_data_size_max = action_data_size;
+
+ if (params->action_is_for_table_entries)
+ action_is_for_table_entries = params->action_is_for_table_entries[i];
+ if (params->action_is_for_default_entry)
+ action_is_for_default_entry = params->action_is_for_default_entry[i];
+ CHECK(action_is_for_table_entries || action_is_for_default_entry, EINVAL);
}
CHECK_NAME(params->default_action_name, EINVAL);
params->default_action_name))
break;
CHECK(i < params->n_actions, EINVAL);
+ CHECK(!params->action_is_for_default_entry || params->action_is_for_default_entry[i],
+ EINVAL);
default_action = action_find(p, params->default_action_name);
CHECK((default_action->st && params->default_action_data) ||
if (!l->actions)
goto nomem;
- l->action_arg = calloc(params->n_actions, sizeof(struct field *));
- if (!l->action_arg)
- goto nomem;
-
if (action_data_size_max) {
l->default_action_data = calloc(1, action_data_size_max);
if (!l->default_action_data)
goto nomem;
}
+ l->action_is_for_table_entries = calloc(params->n_actions, sizeof(int));
+ if (!l->action_is_for_table_entries)
+ goto nomem;
+
+ l->action_is_for_default_entry = calloc(params->n_actions, sizeof(int));
+ if (!l->action_is_for_default_entry)
+ goto nomem;
+
/* Node initialization. */
strcpy(l->name, name);
l->header = header;
for (i = 0; i < params->n_actions; i++) {
- const char *mf_name = params->action_field_names[i];
+ int action_is_for_table_entries = 1, action_is_for_default_entry = 1;
- l->actions[i] = action_find(p, params->action_names[i]);
+ if (params->action_is_for_table_entries)
+ action_is_for_table_entries = params->action_is_for_table_entries[i];
+ if (params->action_is_for_default_entry)
+ action_is_for_default_entry = params->action_is_for_default_entry[i];
- l->action_arg[i] = mf_name ? metadata_field_parse(p, mf_name) : NULL;
+ l->actions[i] = action_find(p, params->action_names[i]);
+ l->action_is_for_table_entries[i] = action_is_for_table_entries;
+ l->action_is_for_default_entry[i] = action_is_for_default_entry;
}
l->default_action = default_action;
if (!l)
return -ENOMEM;
- free(l->action_arg);
+ free(l->action_is_for_default_entry);
+ free(l->action_is_for_table_entries);
+ free(l->default_action_data);
free(l->actions);
free(l->fields);
free(l);
struct learner_runtime *r = &t->learners[j];
free(r->mailbox);
- free(r->action_data);
}
free(t->learners);
TAILQ_FOREACH(l, &p->learners, node) {
struct learner_runtime *r = &t->learners[l->id];
uint64_t size;
- uint32_t j;
/* r->mailbox. */
size = rte_swx_table_learner_mailbox_size_get();
r->key = l->header ?
&t->structs[l->header->struct_id] :
&t->structs[p->metadata_struct_id];
-
- /* r->action_data. */
- r->action_data = calloc(p->n_actions, sizeof(uint8_t *));
- if (!r->action_data) {
- status = -ENOMEM;
- goto error;
- }
-
- for (j = 0; j < l->n_actions; j++) {
- struct action *a = l->actions[j];
- struct field *mf = l->action_arg[j];
- uint8_t *m = t->structs[p->metadata_struct_id];
-
- r->action_data[a->id] = mf ? &m[mf->offset / 8] : NULL;
- }
}
}
TAILQ_REMOVE(&p->learners, l, node);
free(l->fields);
free(l->actions);
- free(l->action_arg);
free(l->default_action_data);
free(l);
}
void
rte_swx_pipeline_free(struct rte_swx_pipeline *p)
{
+ void *lib;
+
if (!p)
return;
+ lib = p->lib;
+
free(p->instruction_data);
free(p->instructions);
struct_free(p);
free(p);
+
+ if (lib)
+ dlclose(lib);
}
int
table_action->action_id = t->actions[table_action_id]->id;
+ table_action->action_is_for_table_entries = t->action_is_for_table_entries[table_action_id];
+ table_action->action_is_for_default_entry = t->action_is_for_default_entry[table_action_id];
+
return 0;
}
learner_action->action_id = l->actions[learner_action_id]->id;
+ learner_action->action_is_for_table_entries =
+ l->action_is_for_table_entries[learner_action_id];
+
+ learner_action->action_is_for_default_entry =
+ l->action_is_for_default_entry[learner_action_id];
+
return 0;
}
fprintf(f, "}\n\n");
}
+struct instruction_group {
+ TAILQ_ENTRY(instruction_group) node;
+
+ uint32_t group_id;
+
+ uint32_t first_instr_id;
+
+ uint32_t last_instr_id;
+
+ instr_exec_t func;
+};
+
+TAILQ_HEAD(instruction_group_list, instruction_group);
+
+static struct instruction_group *
+instruction_group_list_group_find(struct instruction_group_list *igl, uint32_t instruction_id)
+{
+ struct instruction_group *g;
+
+ TAILQ_FOREACH(g, igl, node)
+ if ((g->first_instr_id <= instruction_id) && (instruction_id <= g->last_instr_id))
+ return g;
+
+ return NULL;
+}
+
+static void
+instruction_group_list_free(struct instruction_group_list *igl)
+{
+ if (!igl)
+ return;
+
+ for ( ; ; ) {
+ struct instruction_group *g;
+
+ g = TAILQ_FIRST(igl);
+ if (!g)
+ break;
+
+ TAILQ_REMOVE(igl, g, node);
+ free(g);
+ }
+
+ free(igl);
+}
+
+static struct instruction_group_list *
+instruction_group_list_create(struct rte_swx_pipeline *p)
+{
+ struct instruction_group_list *igl = NULL;
+ struct instruction_group *g = NULL;
+ uint32_t n_groups = 0, i;
+
+ if (!p || !p->instructions || !p->instruction_data || !p->n_instructions)
+ goto error;
+
+ /* List init. */
+ igl = calloc(1, sizeof(struct instruction_group_list));
+ if (!igl)
+ goto error;
+
+ TAILQ_INIT(igl);
+
+ /* Allocate the first group. */
+ g = calloc(1, sizeof(struct instruction_group));
+ if (!g)
+ goto error;
+
+ /* Iteration 1: Separate the instructions into groups based on the thread yield
+ * instructions. Do not worry about the jump instructions at this point.
+ */
+ for (i = 0; i < p->n_instructions; i++) {
+ struct instruction *instr = &p->instructions[i];
+
+ /* Check for thread yield instructions. */
+ if (!instruction_does_thread_yield(instr))
+ continue;
+
+ /* If the current group contains at least one instruction, then finalize it (with
+ * the previous instruction), add it to the list and allocate a new group (that
+ * starts with the current instruction).
+ */
+ if (i - g->first_instr_id) {
+ /* Finalize the group. */
+ g->last_instr_id = i - 1;
+
+ /* Add the group to the list. Advance the number of groups. */
+ TAILQ_INSERT_TAIL(igl, g, node);
+ n_groups++;
+
+ /* Allocate a new group. */
+ g = calloc(1, sizeof(struct instruction_group));
+ if (!g)
+ goto error;
+
+ /* Initialize the new group. */
+ g->group_id = n_groups;
+ g->first_instr_id = i;
+ }
+
+ /* Finalize the current group (with the current instruction, therefore this group
+ * contains just the current thread yield instruction), add it to the list and
+ * allocate a new group (that starts with the next instruction).
+ */
+
+ /* Finalize the group. */
+ g->last_instr_id = i;
+
+ /* Add the group to the list. Advance the number of groups. */
+ TAILQ_INSERT_TAIL(igl, g, node);
+ n_groups++;
+
+ /* Allocate a new group. */
+ g = calloc(1, sizeof(struct instruction_group));
+ if (!g)
+ goto error;
+
+ /* Initialize the new group. */
+ g->group_id = n_groups;
+ g->first_instr_id = i + 1;
+ }
+
+ /* Handle the last group. */
+ if (i - g->first_instr_id) {
+ /* Finalize the group. */
+ g->last_instr_id = i - 1;
+
+ /* Add the group to the list. Advance the number of groups. */
+ TAILQ_INSERT_TAIL(igl, g, node);
+ n_groups++;
+ } else
+ free(g);
+
+ g = NULL;
+
+ /* Iteration 2: Handle jumps. If the current group contains an instruction which represents
+ * the destination of a jump instruction located in a different group ("far jump"), then the
+ * current group has to be split, so that the instruction representing the far jump
+ * destination is at the start of its group.
+ */
+ for ( ; ; ) {
+ int is_modified = 0;
+
+ for (i = 0; i < p->n_instructions; i++) {
+ struct instruction_data *data = &p->instruction_data[i];
+ struct instruction_group *g;
+ uint32_t j;
+
+ /* Continue when the current instruction is not a jump destination. */
+ if (!data->n_users)
+ continue;
+
+ g = instruction_group_list_group_find(igl, i);
+ if (!g)
+ goto error;
+
+ /* Find out all the jump instructions with this destination. */
+ for (j = 0; j < p->n_instructions; j++) {
+ struct instruction *jmp_instr = &p->instructions[j];
+ struct instruction_data *jmp_data = &p->instruction_data[j];
+ struct instruction_group *jmp_g, *new_g;
+
+ /* Continue when not a jump instruction. Even when jump instruction,
+ * continue when the jump destination is not this instruction.
+ */
+ if (!instruction_is_jmp(jmp_instr) ||
+ strcmp(jmp_data->jmp_label, data->label))
+ continue;
+
+ jmp_g = instruction_group_list_group_find(igl, j);
+ if (!jmp_g)
+ goto error;
+
+ /* Continue when both the jump instruction and the jump destination
+ * instruction are in the same group. Even when in different groups,
+ * still continue if the jump destination instruction is already the
+ * first instruction of its group.
+ */
+ if ((jmp_g->group_id == g->group_id) || (g->first_instr_id == i))
+ continue;
+
+ /* Split the group of the current jump destination instruction to
+ * make this instruction the first instruction of a new group.
+ */
+ new_g = calloc(1, sizeof(struct instruction_group));
+ if (!new_g)
+ goto error;
+
+ new_g->group_id = n_groups;
+ new_g->first_instr_id = i;
+ new_g->last_instr_id = g->last_instr_id;
+
+ g->last_instr_id = i - 1;
+
+ TAILQ_INSERT_AFTER(igl, g, new_g, node);
+ n_groups++;
+ is_modified = 1;
+
+ /* The decision to split this group (to make the current instruction
+ * the first instruction of a new group) is already taken and fully
+ * implemented, so no need to search for more reasons to do it.
+ */
+ break;
+ }
+ }
+
+ /* Re-evaluate everything, as at least one group got split, so some jumps that were
+ * previously considered local (i.e. the jump destination is in the same group as
+ * the jump instruction) can now be "far jumps" (i.e. the jump destination is in a
+ * different group than the jump instruction). Wost case scenario: each instruction
+ * that is a jump destination ends up as the first instruction of its group.
+ */
+ if (!is_modified)
+ break;
+ }
+
+ /* Re-assign the group IDs to be in incremental order. */
+ i = 0;
+ TAILQ_FOREACH(g, igl, node) {
+ g->group_id = i;
+
+ i++;
+ }
+
+ return igl;
+
+error:
+ instruction_group_list_free(igl);
+
+ free(g);
+
+ return NULL;
+}
+
+static void
+pipeline_instr_does_tx_codegen(struct rte_swx_pipeline *p __rte_unused,
+ uint32_t instr_pos,
+ struct instruction *instr,
+ FILE *f)
+{
+ fprintf(f,
+ "%s(p, t, &pipeline_instructions[%u]);\n"
+ "\tthread_ip_reset(p, t);\n"
+ "\tinstr_rx_exec(p);\n"
+ "\treturn;\n",
+ instr_type_to_func(instr),
+ instr_pos);
+}
+
static int
-pipeline_codegen(struct rte_swx_pipeline *p)
+pipeline_instr_jmp_codegen(struct rte_swx_pipeline *p,
+ struct instruction_group_list *igl,
+ uint32_t jmp_instr_id,
+ struct instruction *jmp_instr,
+ struct instruction_data *jmp_data,
+ FILE *f)
+{
+ struct instruction_group *jmp_g, *g;
+ struct instruction_data *data;
+ uint32_t instr_id;
+
+ switch (jmp_instr->type) {
+ case INSTR_JMP:
+ break;
+
+ case INSTR_JMP_VALID:
+ fprintf(f,
+ "if (HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_INVALID:
+ fprintf(f,
+ "if (!HEADER_VALID(t, pipeline_instructions[%u].jmp.header_id))",
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_HIT:
+ fprintf(f,
+ "if (t->hit)\n");
+ break;
+
+ case INSTR_JMP_MISS:
+ fprintf(f,
+ "if (!t->hit)\n");
+ break;
+
+ case INSTR_JMP_ACTION_HIT:
+ fprintf(f,
+ "if (t->action_id == pipeline_instructions[%u].jmp.action_id)",
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_ACTION_MISS:
+ fprintf(f,
+ "if (t->action_id != pipeline_instructions[%u].jmp.action_id)",
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_EQ:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_EQ_MH:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_EQ_HM:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_EQ_HH:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) == "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_EQ_I:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) == "
+ "pipeline_instructions[%u].jmp.b_val)",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_NEQ:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_NEQ_MH:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_NEQ_HM:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_NEQ_HH:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) != "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_NEQ_I:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) != "
+ "pipeline_instructions[%u].jmp.b_val)",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_LT:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_LT_MH:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_LT_HM:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_LT_HH:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_LT_MI:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) < "
+ "pipeline_instructions[%u].jmp.b_val)",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_LT_HI:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) < "
+ "pipeline_instructions[%u].jmp.b_val)",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_GT:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_GT_MH:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_GT_HM:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
+ "instr_operand_hbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_GT_HH:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
+ "instr_operand_nbo(t, &pipeline_instructions[%u].jmp.b))",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_GT_MI:
+ fprintf(f,
+ "if (instr_operand_hbo(t, &pipeline_instructions[%u].jmp.a) > "
+ "pipeline_instructions[%u].jmp.b_val)",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ case INSTR_JMP_GT_HI:
+ fprintf(f,
+ "if (instr_operand_nbo(t, &pipeline_instructions[%u].jmp.a) > "
+ "pipeline_instructions[%u].jmp.b_val)",
+ jmp_instr_id,
+ jmp_instr_id);
+ break;
+
+ default:
+ break;
+ }
+
+ /* Find the instruction group of the jump instruction. */
+ jmp_g = instruction_group_list_group_find(igl, jmp_instr_id);
+ if (!jmp_g)
+ return -EINVAL;
+
+ /* Find the instruction group of the jump destination instruction. */
+ data = label_find(p->instruction_data, p->n_instructions, jmp_data->jmp_label);
+ if (!data)
+ return -EINVAL;
+
+ instr_id = data - p->instruction_data;
+
+ g = instruction_group_list_group_find(igl, instr_id);
+ if (!g)
+ return -EINVAL;
+
+ /* Code generation for "near" jump (same instruction group) or "far" jump (different
+ * instruction group).
+ */
+ if (g->group_id == jmp_g->group_id)
+ fprintf(f,
+ "\n\t\tgoto %s;\n",
+ jmp_data->jmp_label);
+ else
+ fprintf(f,
+ " {\n"
+ "\t\tthread_ip_set(t, &p->instructions[%u]);\n"
+ "\t\treturn;\n"
+ "\t}\n\n",
+ g->group_id);
+
+ return 0;
+}
+
+static void
+instruction_group_list_codegen(struct instruction_group_list *igl,
+ struct rte_swx_pipeline *p,
+ FILE *f)
+{
+ struct instruction_group *g;
+ uint32_t i;
+ int is_required = 0;
+
+ /* Check if code generation is required. */
+ TAILQ_FOREACH(g, igl, node)
+ if (g->first_instr_id < g->last_instr_id)
+ is_required = 1;
+
+ if (!is_required)
+ return;
+
+ /* Generate the code for the pipeline instruction array. */
+ fprintf(f,
+ "static const struct instruction pipeline_instructions[] = {\n");
+
+ for (i = 0; i < p->n_instructions; i++) {
+ struct instruction *instr = &p->instructions[i];
+ instruction_export_t func = export_table[instr->type];
+
+ func(instr, f);
+ }
+
+ fprintf(f, "};\n\n");
+
+ /* Generate the code for the pipeline functions: one function for each instruction group
+ * that contains more than one instruction.
+ */
+ TAILQ_FOREACH(g, igl, node) {
+ struct instruction *last_instr;
+ uint32_t j;
+
+ /* Skip if group contains a single instruction. */
+ if (g->last_instr_id == g->first_instr_id)
+ continue;
+
+ /* Generate new pipeline function. */
+ fprintf(f,
+ "void\n"
+ "pipeline_func_%u(struct rte_swx_pipeline *p)\n"
+ "{\n"
+ "\tstruct thread *t = &p->threads[p->thread_id];\n"
+ "\n",
+ g->group_id);
+
+ /* Generate the code for each pipeline instruction. */
+ for (j = g->first_instr_id; j <= g->last_instr_id; j++) {
+ struct instruction *instr = &p->instructions[j];
+ struct instruction_data *data = &p->instruction_data[j];
+
+ /* Label, if present. */
+ if (data->label[0])
+ fprintf(f, "\n%s : ", data->label);
+ else
+ fprintf(f, "\n\t");
+
+ /* TX instruction type. */
+ if (instruction_does_tx(instr)) {
+ pipeline_instr_does_tx_codegen(p, j, instr, f);
+ continue;
+ }
+
+ /* Jump instruction type. */
+ if (instruction_is_jmp(instr)) {
+ pipeline_instr_jmp_codegen(p, igl, j, instr, data, f);
+ continue;
+ }
+
+ /* Any other instruction type. */
+ fprintf(f,
+ "%s(p, t, &pipeline_instructions[%u]);\n",
+ instr_type_to_func(instr),
+ j);
+ }
+
+ /* Finalize the generated pipeline function. For some instructions such as TX,
+ * emit-many-and-TX and unconditional jump, the next instruction has been already
+ * decided unconditionally and the instruction pointer of the current thread set
+ * accordingly; for all the other instructions, the instruction pointer must be
+ * incremented now.
+ */
+ last_instr = &p->instructions[g->last_instr_id];
+
+ if (!instruction_does_tx(last_instr) && (last_instr->type != INSTR_JMP))
+ fprintf(f,
+ "thread_ip_inc(p);\n");
+
+ fprintf(f,
+ "}\n"
+ "\n");
+ }
+}
+
+static uint32_t
+instruction_group_list_custom_instructions_count(struct instruction_group_list *igl)
+{
+ struct instruction_group *g;
+ uint32_t n_custom_instr = 0;
+
+ /* Groups with a single instruction: no function is generated for this group, the group
+ * keeps its current instruction. Groups with more than two instructions: one function and
+ * the associated custom instruction get generated for each such group.
+ */
+ TAILQ_FOREACH(g, igl, node) {
+ if (g->first_instr_id == g->last_instr_id)
+ continue;
+
+ n_custom_instr++;
+ }
+
+ return n_custom_instr;
+}
+
+static int
+pipeline_codegen(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
{
struct action *a;
FILE *f = NULL;
- if (!p)
- return -EINVAL;
-
/* Create the .c file. */
f = fopen("/tmp/pipeline.c", "w");
if (!f)
fprintf(f, "\n");
}
+ /* Add the pipeline code. */
+ instruction_group_list_codegen(igl, p, f);
+
/* Close the .c file. */
fclose(f);
return 0;
}
+#ifndef RTE_SWX_PIPELINE_CMD_MAX_SIZE
+#define RTE_SWX_PIPELINE_CMD_MAX_SIZE 4096
+#endif
+
+static int
+pipeline_libload(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
+{
+ struct action *a;
+ struct instruction_group *g;
+ char *dir_in, *buffer = NULL;
+ const char *dir_out;
+ int status = 0;
+
+ /* Get the environment variables. */
+ dir_in = getenv("RTE_INSTALL_DIR");
+ if (!dir_in) {
+ status = -EINVAL;
+ goto free;
+ }
+
+ dir_out = "/tmp";
+
+ /* Memory allocation for the command buffer. */
+ buffer = malloc(RTE_SWX_PIPELINE_CMD_MAX_SIZE);
+ if (!buffer) {
+ status = -ENOMEM;
+ goto free;
+ }
+
+ snprintf(buffer,
+ RTE_SWX_PIPELINE_CMD_MAX_SIZE,
+ "gcc -c -O3 -fpic -Wno-deprecated-declarations -o %s/pipeline.o %s/pipeline.c "
+ "-I %s/lib/pipeline "
+ "-I %s/lib/eal/include "
+ "-I %s/lib/eal/x86/include "
+ "-I %s/lib/eal/include/generic "
+ "-I %s/lib/meter "
+ "-I %s/lib/port "
+ "-I %s/lib/table "
+ "-I %s/lib/pipeline "
+ "-I %s/config "
+ "-I %s/build "
+ "-I %s/lib/eal/linux/include "
+ ">%s/pipeline.log 2>&1 "
+ "&& "
+ "gcc -shared %s/pipeline.o -o %s/libpipeline.so "
+ ">>%s/pipeline.log 2>&1",
+ dir_out,
+ dir_out,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_in,
+ dir_out,
+ dir_out,
+ dir_out,
+ dir_out);
+
+ /* Build the shared object library. */
+ status = system(buffer);
+ if (status)
+ goto free;
+
+ /* Open library. */
+ snprintf(buffer,
+ RTE_SWX_PIPELINE_CMD_MAX_SIZE,
+ "%s/libpipeline.so",
+ dir_out);
+
+ p->lib = dlopen(buffer, RTLD_LAZY);
+ if (!p->lib) {
+ status = -EIO;
+ goto free;
+ }
+
+ /* Get the action function symbols. */
+ TAILQ_FOREACH(a, &p->actions, node) {
+ snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "action_%s_run", a->name);
+
+ p->action_funcs[a->id] = dlsym(p->lib, buffer);
+ if (!p->action_funcs[a->id]) {
+ status = -EINVAL;
+ goto free;
+ }
+ }
+
+ /* Get the pipeline function symbols. */
+ TAILQ_FOREACH(g, igl, node) {
+ if (g->first_instr_id == g->last_instr_id)
+ continue;
+
+ snprintf(buffer, RTE_SWX_PIPELINE_CMD_MAX_SIZE, "pipeline_func_%u", g->group_id);
+
+ g->func = dlsym(p->lib, buffer);
+ if (!g->func) {
+ status = -EINVAL;
+ goto free;
+ }
+ }
+
+free:
+ if (status && p->lib) {
+ dlclose(p->lib);
+ p->lib = NULL;
+ }
+
+ free(buffer);
+
+ return status;
+}
+
+static int
+pipeline_adjust_check(struct rte_swx_pipeline *p __rte_unused,
+ struct instruction_group_list *igl)
+{
+ uint32_t n_custom_instr = instruction_group_list_custom_instructions_count(igl);
+
+ /* Check that enough space is available within the pipeline instruction table to store all
+ * the custom instructions.
+ */
+ if (INSTR_CUSTOM_0 + n_custom_instr > RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX)
+ return -ENOSPC;
+
+ return 0;
+}
+
+static void
+pipeline_adjust(struct rte_swx_pipeline *p, struct instruction_group_list *igl)
+{
+ struct instruction_group *g;
+ uint32_t i;
+
+ /* Pipeline table instructions. */
+ for (i = 0; i < p->n_instructions; i++) {
+ struct instruction *instr = &p->instructions[i];
+
+ if (instr->type == INSTR_TABLE)
+ instr->type = INSTR_TABLE_AF;
+
+ if (instr->type == INSTR_LEARNER)
+ instr->type = INSTR_LEARNER_AF;
+ }
+
+ /* Pipeline custom instructions. */
+ i = 0;
+ TAILQ_FOREACH(g, igl, node) {
+ struct instruction *instr = &p->instructions[g->first_instr_id];
+ uint32_t j;
+
+ if (g->first_instr_id == g->last_instr_id)
+ continue;
+
+ /* Install a new custom instruction. */
+ p->instruction_table[INSTR_CUSTOM_0 + i] = g->func;
+
+ /* First instruction of the group: change its type to the new custom instruction. */
+ instr->type = INSTR_CUSTOM_0 + i;
+
+ /* All the subsequent instructions of the group: invalidate. */
+ for (j = g->first_instr_id + 1; j <= g->last_instr_id; j++) {
+ struct instruction_data *data = &p->instruction_data[j];
+
+ data->invalid = 1;
+ }
+
+ i++;
+ }
+
+ /* Remove the invalidated instructions. */
+ p->n_instructions = instr_compact(p->instructions, p->instruction_data, p->n_instructions);
+
+ /* Resolve the jump destination for any "standalone" jump instructions (i.e. those jump
+ * instructions that are the only instruction within their group, so they were left
+ * unmodified).
+ */
+ instr_jmp_resolve(p->instructions, p->instruction_data, p->n_instructions);
+}
+
static int
pipeline_compile(struct rte_swx_pipeline *p)
{
+ struct instruction_group_list *igl = NULL;
int status = 0;
+ igl = instruction_group_list_create(p);
+ if (!igl) {
+ status = -ENOMEM;
+ goto free;
+ }
+
/* Code generation. */
- status = pipeline_codegen(p);
+ status = pipeline_codegen(p, igl);
if (status)
- return status;
+ goto free;
+
+ /* Build and load the shared object library. */
+ status = pipeline_libload(p, igl);
+ if (status)
+ goto free;
+
+ /* Adjust instructions. */
+ status = pipeline_adjust_check(p, igl);
+ if (status)
+ goto free;
+
+ pipeline_adjust(p, igl);
+
+free:
+ instruction_group_list_free(igl);
return status;
}
git.droids-corp.org / dpdk.git / blobdiff