#include <rte_common.h>
#include <rte_prefetch.h>
#include <rte_byteorder.h>
+#include <rte_cycles.h>
+#include <rte_meter.h>
#include "rte_swx_pipeline.h"
#include "rte_swx_ctl.h"
/* rx m.port_in */
INSTR_RX,
- /* tx m.port_out */
- INSTR_TX,
+ /* tx port_out
+ * port_out = MI
+ */
+ INSTR_TX, /* port_out = M */
+ INSTR_TX_I, /* port_out = I */
/* extract h.header */
INSTR_HDR_EXTRACT,
INSTR_REGADD_RIM, /* index = I, src = MEFT */
INSTR_REGADD_RII, /* index = I, src = I */
+ /* metprefetch METARRAY index
+ * prefetch METARRAY[index]
+ * index = HMEFTI
+ */
+ INSTR_METPREFETCH_H, /* index = H */
+ INSTR_METPREFETCH_M, /* index = MEFT */
+ INSTR_METPREFETCH_I, /* index = I */
+
+ /* meter METARRAY index length color_in color_out
+ * color_out = meter(METARRAY[index], length, color_in)
+ * index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
+ */
+ INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
+ INSTR_METER_HHI, /* index = H, length = H, color_in = I */
+ INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
+ INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
+ INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
+ INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
+ INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
+ INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
+ INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
+ INSTR_METER_IHI, /* index = I, length = H, color_in = I */
+ INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
+ INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
+
/* table TABLE */
INSTR_TABLE,
struct instr_io {
struct {
- uint8_t offset;
- uint8_t n_bits;
- uint8_t pad[2];
+ union {
+ struct {
+ uint8_t offset;
+ uint8_t n_bits;
+ uint8_t pad[2];
+ };
+
+ uint32_t val;
+ };
} io;
struct {
};
};
+struct instr_meter {
+ uint8_t metarray_id;
+ uint8_t pad[3];
+
+ union {
+ struct instr_operand idx;
+ uint32_t idx_val;
+ };
+
+ struct instr_operand length;
+
+ union {
+ struct instr_operand color_in;
+ uint32_t color_in_val;
+ };
+
+ struct instr_operand color_out;
+};
+
struct instr_dma {
struct {
uint8_t header_id[8];
struct instr_hdr_validity valid;
struct instr_dst_src mov;
struct instr_regarray regarray;
+ struct instr_meter meter;
struct instr_dma dma;
struct instr_dst_src alu;
struct instr_table table;
/* Match. */
struct match_field *fields;
uint32_t n_fields;
- int is_header; /* Only valid when n_fields > 0. */
struct header *header; /* Only valid when n_fields > 0. */
/* Action. */
uint8_t **key;
};
+struct table_statistics {
+ uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
+ uint64_t *n_pkts_action;
+};
+
/*
* Register array.
*/
uint32_t size_mask;
};
+/*
+ * Meter array.
+ */
+struct meter_profile {
+ TAILQ_ENTRY(meter_profile) node;
+ char name[RTE_SWX_NAME_SIZE];
+ struct rte_meter_trtcm_params params;
+ struct rte_meter_trtcm_profile profile;
+ uint32_t n_users;
+};
+
+TAILQ_HEAD(meter_profile_tailq, meter_profile);
+
+struct metarray {
+ TAILQ_ENTRY(metarray) node;
+ char name[RTE_SWX_NAME_SIZE];
+ uint32_t size;
+ uint32_t id;
+};
+
+TAILQ_HEAD(metarray_tailq, metarray);
+
+struct meter {
+ struct rte_meter_trtcm m;
+ struct meter_profile *profile;
+ enum rte_color color_mask;
+ uint8_t pad[20];
+
+ uint64_t n_pkts[RTE_COLORS];
+ uint64_t n_bytes[RTE_COLORS];
+};
+
+struct metarray_runtime {
+ struct meter *metarray;
+ uint32_t size_mask;
+};
+
/*
* Pipeline.
*/
struct table_type_tailq table_types;
struct table_tailq tables;
struct regarray_tailq regarrays;
+ struct meter_profile_tailq meter_profiles;
+ struct metarray_tailq metarrays;
struct port_in_runtime *in;
struct port_out_runtime *out;
struct instruction **action_instructions;
struct rte_swx_table_state *table_state;
+ struct table_statistics *table_stats;
struct regarray_runtime *regarray_runtime;
+ struct metarray_runtime *metarray_runtime;
struct instruction *instructions;
struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
uint32_t n_actions;
uint32_t n_tables;
uint32_t n_regarrays;
+ uint32_t n_metarrays;
uint32_t n_headers;
uint32_t thread_id;
uint32_t port_id;
/*
* Instruction.
*/
+static int
+instruction_is_tx(enum instruction_type type)
+{
+ switch (type) {
+ case INSTR_TX:
+ case INSTR_TX_I:
+ return 1;
+
+ default:
+ return 0;
+ }
+}
+
static int
instruction_is_jmp(struct instruction *instr)
{
struct instruction *instr,
struct instruction_data *data __rte_unused)
{
+ char *port = tokens[1];
struct field *f;
+ uint32_t port_val;
CHECK(n_tokens == 2, EINVAL);
- f = metadata_field_parse(p, tokens[1]);
- CHECK(f, EINVAL);
+ f = metadata_field_parse(p, port);
+ if (f) {
+ instr->type = INSTR_TX;
+ instr->io.io.offset = f->offset / 8;
+ instr->io.io.n_bits = f->n_bits;
+ return 0;
+ }
- instr->type = INSTR_TX;
- instr->io.io.offset = f->offset / 8;
- instr->io.io.n_bits = f->n_bits;
+ /* TX_I. */
+ port_val = strtoul(port, &port, 0);
+ CHECK(!port[0], EINVAL);
+
+ instr->type = INSTR_TX_I;
+ instr->io.io.val = port_val;
+ return 0;
+}
+
+static int
+instr_drop_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens __rte_unused,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ CHECK(n_tokens == 1, EINVAL);
+
+ /* TX_I. */
+ instr->type = INSTR_TX_I;
+ instr->io.io.val = p->n_ports_out - 1;
return 0;
}
instr_rx_exec(p);
}
+static inline void
+instr_tx_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ uint64_t port_id = ip->io.io.val;
+ struct port_out_runtime *port = &p->out[port_id];
+ struct rte_swx_pkt *pkt = &t->pkt;
+
+ TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
+ p->thread_id,
+ (uint32_t)port_id);
+
+ /* Headers. */
+ emit_handler(t);
+
+ /* Packet. */
+ port->pkt_tx(port->obj, pkt);
+
+ /* Thread. */
+ thread_ip_reset(p, t);
+ instr_rx_exec(p);
+}
+
/*
* extract.
*/
uint32_t table_id = ip->table.table_id;
struct rte_swx_table_state *ts = &t->table_state[table_id];
struct table_runtime *table = &t->tables[table_id];
- uint64_t action_id;
+ struct table_statistics *stats = &p->table_stats[table_id];
+ uint64_t action_id, n_pkts_hit, n_pkts_action;
uint8_t *action_data;
int done, hit;
action_id = hit ? action_id : ts->default_action_id;
action_data = hit ? action_data : ts->default_action_data;
+ n_pkts_hit = stats->n_pkts_hit[hit];
+ n_pkts_action = stats->n_pkts_action[action_id];
TRACE("[Thread %2u] table %u (%s, action %u)\n",
p->thread_id,
t->action_id = action_id;
t->structs[0] = action_data;
t->hit = hit;
+ stats->n_pkts_hit[hit] = n_pkts_hit + 1;
+ stats->n_pkts_action[action_id] = n_pkts_action + 1;
/* Thread. */
thread_ip_action_call(p, t, action_id);
}
/*
- * jmp.
+ * metarray.
*/
-static struct action *
-action_find(struct rte_swx_pipeline *p, const char *name);
-
-static int
-instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
- struct action *action __rte_unused,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
-{
- CHECK(n_tokens == 2, EINVAL);
-
- strcpy(data->jmp_label, tokens[1]);
-
- instr->type = INSTR_JMP;
- instr->jmp.ip = NULL; /* Resolved later. */
- return 0;
-}
-
-static int
-instr_jmp_valid_translate(struct rte_swx_pipeline *p,
- struct action *action __rte_unused,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
-{
- struct header *h;
-
- CHECK(n_tokens == 3, EINVAL);
-
- strcpy(data->jmp_label, tokens[1]);
-
- h = header_parse(p, tokens[2]);
- CHECK(h, EINVAL);
-
- instr->type = INSTR_JMP_VALID;
- instr->jmp.ip = NULL; /* Resolved later. */
- instr->jmp.header_id = h->id;
- return 0;
-}
+static struct metarray *
+metarray_find(struct rte_swx_pipeline *p, const char *name);
static int
-instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
- struct action *action __rte_unused,
+instr_metprefetch_translate(struct rte_swx_pipeline *p,
+ struct action *action,
char **tokens,
int n_tokens,
struct instruction *instr,
- struct instruction_data *data)
+ struct instruction_data *data __rte_unused)
{
- struct header *h;
+ char *metarray = tokens[1], *idx = tokens[2];
+ struct metarray *m;
+ struct field *fidx;
+ uint32_t idx_struct_id, idx_val;
CHECK(n_tokens == 3, EINVAL);
- strcpy(data->jmp_label, tokens[1]);
+ m = metarray_find(p, metarray);
+ CHECK(m, EINVAL);
- h = header_parse(p, tokens[2]);
- CHECK(h, EINVAL);
+ /* METPREFETCH_H, METPREFETCH_M. */
+ fidx = struct_field_parse(p, action, idx, &idx_struct_id);
+ if (fidx) {
+ instr->type = INSTR_METPREFETCH_M;
+ if (idx[0] == 'h')
+ instr->type = INSTR_METPREFETCH_H;
- instr->type = INSTR_JMP_INVALID;
- instr->jmp.ip = NULL; /* Resolved later. */
- instr->jmp.header_id = h->id;
+ instr->meter.metarray_id = m->id;
+ instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->meter.idx.n_bits = fidx->n_bits;
+ instr->meter.idx.offset = fidx->offset / 8;
+ return 0;
+ }
+
+ /* METPREFETCH_I. */
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_METPREFETCH_I;
+ instr->meter.metarray_id = m->id;
+ instr->meter.idx_val = idx_val;
return 0;
}
static int
-instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
- struct action *action,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
+instr_meter_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
{
- CHECK(!action, EINVAL);
- CHECK(n_tokens == 2, EINVAL);
+ char *metarray = tokens[1], *idx = tokens[2], *length = tokens[3];
+ char *color_in = tokens[4], *color_out = tokens[5];
+ struct metarray *m;
+ struct field *fidx, *flength, *fcin, *fcout;
+ uint32_t idx_struct_id, length_struct_id;
+ uint32_t color_in_struct_id, color_out_struct_id;
- strcpy(data->jmp_label, tokens[1]);
+ CHECK(n_tokens == 6, EINVAL);
- instr->type = INSTR_JMP_HIT;
- instr->jmp.ip = NULL; /* Resolved later. */
- return 0;
-}
+ m = metarray_find(p, metarray);
+ CHECK(m, EINVAL);
-static int
-instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
- struct action *action,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
-{
- CHECK(!action, EINVAL);
- CHECK(n_tokens == 2, EINVAL);
+ fidx = struct_field_parse(p, action, idx, &idx_struct_id);
- strcpy(data->jmp_label, tokens[1]);
+ flength = struct_field_parse(p, action, length, &length_struct_id);
+ CHECK(flength, EINVAL);
- instr->type = INSTR_JMP_MISS;
- instr->jmp.ip = NULL; /* Resolved later. */
- return 0;
-}
+ fcin = struct_field_parse(p, action, color_in, &color_in_struct_id);
-static int
-instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
- struct action *action,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
-{
- struct action *a;
+ fcout = struct_field_parse(p, NULL, color_out, &color_out_struct_id);
+ CHECK(fcout, EINVAL);
- CHECK(!action, EINVAL);
- CHECK(n_tokens == 3, EINVAL);
+ /* index = HMEFT, length = HMEFT, color_in = MEFT, color_out = MEF. */
+ if (fidx && fcin) {
+ instr->type = INSTR_METER_MMM;
+ if (idx[0] == 'h' && length[0] == 'h')
+ instr->type = INSTR_METER_HHM;
+ if (idx[0] == 'h' && length[0] != 'h')
+ instr->type = INSTR_METER_HMM;
+ if (idx[0] != 'h' && length[0] == 'h')
+ instr->type = INSTR_METER_MHM;
- strcpy(data->jmp_label, tokens[1]);
+ instr->meter.metarray_id = m->id;
- a = action_find(p, tokens[2]);
- CHECK(a, EINVAL);
+ instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->meter.idx.n_bits = fidx->n_bits;
+ instr->meter.idx.offset = fidx->offset / 8;
- instr->type = INSTR_JMP_ACTION_HIT;
- instr->jmp.ip = NULL; /* Resolved later. */
- instr->jmp.action_id = a->id;
- return 0;
-}
+ instr->meter.length.struct_id = (uint8_t)length_struct_id;
+ instr->meter.length.n_bits = flength->n_bits;
+ instr->meter.length.offset = flength->offset / 8;
-static int
-instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
- struct action *action,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
-{
- struct action *a;
+ instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
+ instr->meter.color_in.n_bits = fcin->n_bits;
+ instr->meter.color_in.offset = fcin->offset / 8;
- CHECK(!action, EINVAL);
- CHECK(n_tokens == 3, EINVAL);
+ instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
+ instr->meter.color_out.n_bits = fcout->n_bits;
+ instr->meter.color_out.offset = fcout->offset / 8;
- strcpy(data->jmp_label, tokens[1]);
+ return 0;
+ }
- a = action_find(p, tokens[2]);
- CHECK(a, EINVAL);
+ /* index = HMEFT, length = HMEFT, color_in = I, color_out = MEF. */
+ if (fidx && !fcin) {
+ uint32_t color_in_val = strtoul(color_in, &color_in, 0);
+ CHECK(!color_in[0], EINVAL);
- instr->type = INSTR_JMP_ACTION_MISS;
- instr->jmp.ip = NULL; /* Resolved later. */
- instr->jmp.action_id = a->id;
- return 0;
-}
+ instr->type = INSTR_METER_MMI;
+ if (idx[0] == 'h' && length[0] == 'h')
+ instr->type = INSTR_METER_HHI;
+ if (idx[0] == 'h' && length[0] != 'h')
+ instr->type = INSTR_METER_HMI;
+ if (idx[0] != 'h' && length[0] == 'h')
+ instr->type = INSTR_METER_MHI;
-static int
-instr_jmp_eq_translate(struct rte_swx_pipeline *p,
- struct action *action,
- char **tokens,
- int n_tokens,
- struct instruction *instr,
- struct instruction_data *data)
-{
- char *a = tokens[2], *b = tokens[3];
- struct field *fa, *fb;
- uint64_t b_val;
- uint32_t a_struct_id, b_struct_id;
+ instr->meter.metarray_id = m->id;
- CHECK(n_tokens == 4, EINVAL);
+ instr->meter.idx.struct_id = (uint8_t)idx_struct_id;
+ instr->meter.idx.n_bits = fidx->n_bits;
+ instr->meter.idx.offset = fidx->offset / 8;
- strcpy(data->jmp_label, tokens[1]);
+ instr->meter.length.struct_id = (uint8_t)length_struct_id;
+ instr->meter.length.n_bits = flength->n_bits;
+ instr->meter.length.offset = flength->offset / 8;
- fa = struct_field_parse(p, action, a, &a_struct_id);
- CHECK(fa, EINVAL);
+ instr->meter.color_in_val = color_in_val;
- /* JMP_EQ or JMP_EQ_S. */
- fb = struct_field_parse(p, action, b, &b_struct_id);
- if (fb) {
- instr->type = INSTR_JMP_EQ;
- if ((a[0] == 'h' && b[0] != 'h') ||
- (a[0] != 'h' && b[0] == 'h'))
- instr->type = INSTR_JMP_EQ_S;
- instr->jmp.ip = NULL; /* Resolved later. */
+ instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
+ instr->meter.color_out.n_bits = fcout->n_bits;
+ instr->meter.color_out.offset = fcout->offset / 8;
- instr->jmp.a.struct_id = (uint8_t)a_struct_id;
- instr->jmp.a.n_bits = fa->n_bits;
- instr->jmp.a.offset = fa->offset / 8;
- instr->jmp.b.struct_id = (uint8_t)b_struct_id;
- instr->jmp.b.n_bits = fb->n_bits;
- instr->jmp.b.offset = fb->offset / 8;
return 0;
}
- /* JMP_EQ_I. */
- b_val = strtoull(b, &b, 0);
+ /* index = I, length = HMEFT, color_in = MEFT, color_out = MEF. */
+ if (!fidx && fcin) {
+ uint32_t idx_val;
+
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ instr->type = INSTR_METER_IMM;
+ if (length[0] == 'h')
+ instr->type = INSTR_METER_IHM;
+
+ instr->meter.metarray_id = m->id;
+
+ instr->meter.idx_val = idx_val;
+
+ instr->meter.length.struct_id = (uint8_t)length_struct_id;
+ instr->meter.length.n_bits = flength->n_bits;
+ instr->meter.length.offset = flength->offset / 8;
+
+ instr->meter.color_in.struct_id = (uint8_t)color_in_struct_id;
+ instr->meter.color_in.n_bits = fcin->n_bits;
+ instr->meter.color_in.offset = fcin->offset / 8;
+
+ instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
+ instr->meter.color_out.n_bits = fcout->n_bits;
+ instr->meter.color_out.offset = fcout->offset / 8;
+
+ return 0;
+ }
+
+ /* index = I, length = HMEFT, color_in = I, color_out = MEF. */
+ if (!fidx && !fcin) {
+ uint32_t idx_val, color_in_val;
+
+ idx_val = strtoul(idx, &idx, 0);
+ CHECK(!idx[0], EINVAL);
+
+ color_in_val = strtoul(color_in, &color_in, 0);
+ CHECK(!color_in[0], EINVAL);
+
+ instr->type = INSTR_METER_IMI;
+ if (length[0] == 'h')
+ instr->type = INSTR_METER_IHI;
+
+ instr->meter.metarray_id = m->id;
+
+ instr->meter.idx_val = idx_val;
+
+ instr->meter.length.struct_id = (uint8_t)length_struct_id;
+ instr->meter.length.n_bits = flength->n_bits;
+ instr->meter.length.offset = flength->offset / 8;
+
+ instr->meter.color_in_val = color_in_val;
+
+ instr->meter.color_out.struct_id = (uint8_t)color_out_struct_id;
+ instr->meter.color_out.n_bits = fcout->n_bits;
+ instr->meter.color_out.offset = fcout->offset / 8;
+
+ return 0;
+ }
+
+ CHECK(0, EINVAL);
+}
+
+static inline struct meter *
+instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
+ uint64_t idx = idx64 & idx64_mask & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline struct meter *
+instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
+ uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
+ uint64_t idx64 = *idx64_ptr;
+ uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+#else
+
+#define instr_meter_idx_nbo instr_meter_idx_hbo
+
+#endif
+
+static inline struct meter *
+instr_meter_idx_imm(struct rte_swx_pipeline *p, struct instruction *ip)
+{
+ struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
+
+ uint64_t idx = ip->meter.idx_val & r->size_mask;
+
+ return &r->metarray[idx];
+}
+
+static inline uint32_t
+instr_meter_length_hbo(struct thread *t, struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return (uint32_t)src;
+}
+
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+
+static inline uint32_t
+instr_meter_length_nbo(struct thread *t, struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
+
+ return (uint32_t)src;
+}
+
+#else
+
+#define instr_meter_length_nbo instr_meter_length_hbo
+
+#endif
+
+static inline enum rte_color
+instr_meter_color_in_hbo(struct thread *t, struct instruction *ip)
+{
+ uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
+ uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
+ uint64_t src64 = *src64_ptr;
+ uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
+ uint64_t src = src64 & src64_mask;
+
+ return (enum rte_color)src;
+}
+
+static inline void
+instr_meter_color_out_hbo_set(struct thread *t, struct instruction *ip, enum rte_color color_out)
+{
+ uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
+ uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
+ uint64_t dst64 = *dst64_ptr;
+ uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
+
+ uint64_t src = (uint64_t)color_out;
+
+ *dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
+}
+
+static inline void
+instr_metprefetch_h_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_metprefetch_m_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_metprefetch_i_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+
+ TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_hhm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_hhi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_hmm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+static inline void
+instr_meter_hmi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_nbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_mhm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_mhi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_mmm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_mmi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_hbo(p, t, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_ihm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_ihi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_nbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+static inline void
+instr_meter_imm_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = instr_meter_color_in_hbo(t, ip);
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+static inline void
+instr_meter_imi_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+ struct meter *m;
+ uint64_t time, n_pkts, n_bytes;
+ uint32_t length;
+ enum rte_color color_in, color_out;
+
+ TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
+
+ /* Structs. */
+ m = instr_meter_idx_imm(p, ip);
+ rte_prefetch0(m->n_pkts);
+ time = rte_get_tsc_cycles();
+ length = instr_meter_length_hbo(t, ip);
+ color_in = (enum rte_color)ip->meter.color_in_val;
+
+ color_out = rte_meter_trtcm_color_aware_check(&m->m,
+ &m->profile->profile,
+ time,
+ length,
+ color_in);
+
+ color_out &= m->color_mask;
+
+ n_pkts = m->n_pkts[color_out];
+ n_bytes = m->n_bytes[color_out];
+
+ instr_meter_color_out_hbo_set(t, ip, color_out);
+
+ m->n_pkts[color_out] = n_pkts + 1;
+ m->n_bytes[color_out] = n_bytes + length;
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
+/*
+ * jmp.
+ */
+static struct action *
+action_find(struct rte_swx_pipeline *p, const char *name);
+
+static int
+instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_valid_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct header *h;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ h = header_parse(p, tokens[2]);
+ CHECK(h, EINVAL);
+
+ instr->type = INSTR_JMP_VALID;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.header_id = h->id;
+ return 0;
+}
+
+static int
+instr_jmp_invalid_translate(struct rte_swx_pipeline *p,
+ struct action *action __rte_unused,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct header *h;
+
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ h = header_parse(p, tokens[2]);
+ CHECK(h, EINVAL);
+
+ instr->type = INSTR_JMP_INVALID;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.header_id = h->id;
+ return 0;
+}
+
+static int
+instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP_HIT;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 2, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ instr->type = INSTR_JMP_MISS;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ return 0;
+}
+
+static int
+instr_jmp_action_hit_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct action *a;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ a = action_find(p, tokens[2]);
+ CHECK(a, EINVAL);
+
+ instr->type = INSTR_JMP_ACTION_HIT;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.action_id = a->id;
+ return 0;
+}
+
+static int
+instr_jmp_action_miss_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ struct action *a;
+
+ CHECK(!action, EINVAL);
+ CHECK(n_tokens == 3, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ a = action_find(p, tokens[2]);
+ CHECK(a, EINVAL);
+
+ instr->type = INSTR_JMP_ACTION_MISS;
+ instr->jmp.ip = NULL; /* Resolved later. */
+ instr->jmp.action_id = a->id;
+ return 0;
+}
+
+static int
+instr_jmp_eq_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data)
+{
+ char *a = tokens[2], *b = tokens[3];
+ struct field *fa, *fb;
+ uint64_t b_val;
+ uint32_t a_struct_id, b_struct_id;
+
+ CHECK(n_tokens == 4, EINVAL);
+
+ strcpy(data->jmp_label, tokens[1]);
+
+ fa = struct_field_parse(p, action, a, &a_struct_id);
+ CHECK(fa, EINVAL);
+
+ /* JMP_EQ or JMP_EQ_S. */
+ fb = struct_field_parse(p, action, b, &b_struct_id);
+ if (fb) {
+ instr->type = INSTR_JMP_EQ;
+ if ((a[0] == 'h' && b[0] != 'h') ||
+ (a[0] != 'h' && b[0] == 'h'))
+ instr->type = INSTR_JMP_EQ_S;
+ instr->jmp.ip = NULL; /* Resolved later. */
+
+ instr->jmp.a.struct_id = (uint8_t)a_struct_id;
+ instr->jmp.a.n_bits = fa->n_bits;
+ instr->jmp.a.offset = fa->offset / 8;
+ instr->jmp.b.struct_id = (uint8_t)b_struct_id;
+ instr->jmp.b.n_bits = fb->n_bits;
+ instr->jmp.b.offset = fb->offset / 8;
+ return 0;
+ }
+
+ /* JMP_EQ_I. */
+ b_val = strtoull(b, &b, 0);
CHECK(!b[0], EINVAL);
if (a[0] == 'h')
instr,
data);
+ if (!strcmp(tokens[tpos], "drop"))
+ return instr_drop_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
if (!strcmp(tokens[tpos], "extract"))
return instr_hdr_extract_translate(p,
action,
instr,
data);
+ if (!strcmp(tokens[tpos], "metprefetch"))
+ return instr_metprefetch_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
+ if (!strcmp(tokens[tpos], "meter"))
+ return instr_meter_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
if (!strcmp(tokens[tpos], "table"))
return instr_table_translate(p,
action,
for (i = 0; i < n_instructions; i++) {
type = instr[i].type;
- if (type == INSTR_TX)
+ if (instruction_is_tx(type))
break;
}
CHECK(i < n_instructions, EINVAL);
* jump.
*/
type = instr[n_instructions - 1].type;
- CHECK((type == INSTR_TX) || (type == INSTR_JMP), EINVAL);
+ CHECK(instruction_is_tx(type) || (type == INSTR_JMP), EINVAL);
}
if (a) {
for (i = 0; i < n_instructions; i++) {
type = instr[i].type;
- if ((type == INSTR_RETURN) || (type == INSTR_TX))
+ if ((type == INSTR_RETURN) || instruction_is_tx(type))
break;
}
CHECK(i < n_instructions, EINVAL);
if (!i)
return 0;
- if (instr[i].type != INSTR_TX)
+ if (!instruction_is_tx(instr[i].type))
return 0;
if (data[i].n_users)
static instr_exec_t instruction_table[] = {
[INSTR_RX] = instr_rx_exec,
[INSTR_TX] = instr_tx_exec,
+ [INSTR_TX_I] = instr_tx_i_exec,
[INSTR_HDR_EXTRACT] = instr_hdr_extract_exec,
[INSTR_HDR_EXTRACT2] = instr_hdr_extract2_exec,
[INSTR_REGADD_RIM] = instr_regadd_rim_exec,
[INSTR_REGADD_RII] = instr_regadd_rii_exec,
+ [INSTR_METPREFETCH_H] = instr_metprefetch_h_exec,
+ [INSTR_METPREFETCH_M] = instr_metprefetch_m_exec,
+ [INSTR_METPREFETCH_I] = instr_metprefetch_i_exec,
+
+ [INSTR_METER_HHM] = instr_meter_hhm_exec,
+ [INSTR_METER_HHI] = instr_meter_hhi_exec,
+ [INSTR_METER_HMM] = instr_meter_hmm_exec,
+ [INSTR_METER_HMI] = instr_meter_hmi_exec,
+ [INSTR_METER_MHM] = instr_meter_mhm_exec,
+ [INSTR_METER_MHI] = instr_meter_mhi_exec,
+ [INSTR_METER_MMM] = instr_meter_mmm_exec,
+ [INSTR_METER_MMI] = instr_meter_mmi_exec,
+ [INSTR_METER_IHM] = instr_meter_ihm_exec,
+ [INSTR_METER_IHI] = instr_meter_ihi_exec,
+ [INSTR_METER_IMM] = instr_meter_imm_exec,
+ [INSTR_METER_IMI] = instr_meter_imi_exec,
+
[INSTR_TABLE] = instr_table_exec,
[INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
[INSTR_EXTERN_FUNC] = instr_extern_func_exec,
CHECK(ops->lkp, EINVAL);
CHECK(ops->free, EINVAL);
- /* Node allocation. */
- elem = calloc(1, sizeof(struct table_type));
- CHECK(elem, ENOMEM);
+ /* Node allocation. */
+ elem = calloc(1, sizeof(struct table_type));
+ CHECK(elem, ENOMEM);
+
+ /* Node initialization. */
+ strcpy(elem->name, name);
+ elem->match_type = match_type;
+ memcpy(&elem->ops, ops, sizeof(*ops));
+
+ /* Node add to tailq. */
+ TAILQ_INSERT_TAIL(&p->table_types, elem, node);
+
+ return 0;
+}
+
+static enum rte_swx_table_match_type
+table_match_type_resolve(struct rte_swx_match_field_params *fields,
+ uint32_t n_fields,
+ uint32_t max_offset_field_id)
+{
+ uint32_t n_fields_em = 0, i;
+
+ for (i = 0; i < n_fields; i++)
+ if (fields[i].match_type == RTE_SWX_TABLE_MATCH_EXACT)
+ n_fields_em++;
+
+ if (n_fields_em == n_fields)
+ return RTE_SWX_TABLE_MATCH_EXACT;
+
+ if ((n_fields_em == n_fields - 1) &&
+ (fields[max_offset_field_id].match_type == RTE_SWX_TABLE_MATCH_LPM))
+ return RTE_SWX_TABLE_MATCH_LPM;
+
+ return RTE_SWX_TABLE_MATCH_WILDCARD;
+}
+
+static int
+table_match_fields_check(struct rte_swx_pipeline *p,
+ struct rte_swx_pipeline_table_params *params,
+ struct header **header,
+ uint32_t *min_offset_field_id,
+ uint32_t *max_offset_field_id)
+{
+ struct header *h0 = NULL;
+ struct field *hf, *mf;
+ uint32_t *offset = NULL, min_offset, max_offset, min_offset_pos, max_offset_pos, i;
+ int status = 0;
+
+ /* Return if no match fields. */
+ if (!params->n_fields) {
+ if (params->fields) {
+ status = -EINVAL;
+ goto end;
+ }
+
+ return 0;
+ }
+
+ /* Memory allocation. */
+ offset = calloc(params->n_fields, sizeof(uint32_t));
+ if (!offset) {
+ status = -ENOMEM;
+ goto end;
+ }
+
+ /* Check that all the match fields belong to either the same header or
+ * to the meta-data.
+ */
+ hf = header_field_parse(p, params->fields[0].name, &h0);
+ mf = metadata_field_parse(p, params->fields[0].name);
+ if (!hf && !mf) {
+ status = -EINVAL;
+ goto end;
+ }
+
+ offset[0] = h0 ? hf->offset : mf->offset;
+
+ for (i = 1; i < params->n_fields; i++)
+ if (h0) {
+ struct header *h;
+
+ hf = header_field_parse(p, params->fields[i].name, &h);
+ if (!hf || (h->id != h0->id)) {
+ status = -EINVAL;
+ goto end;
+ }
+
+ offset[i] = hf->offset;
+ } else {
+ mf = metadata_field_parse(p, params->fields[i].name);
+ if (!mf) {
+ status = -EINVAL;
+ goto end;
+ }
+
+ offset[i] = mf->offset;
+ }
+
+ /* Check that there are no duplicated match fields. */
+ for (i = 0; i < params->n_fields; i++) {
+ uint32_t j;
- /* Node initialization. */
- strcpy(elem->name, name);
- elem->match_type = match_type;
- memcpy(&elem->ops, ops, sizeof(*ops));
+ for (j = 0; j < i; j++)
+ if (offset[j] == offset[i]) {
+ status = -EINVAL;
+ goto end;
+ }
+ }
- /* Node add to tailq. */
- TAILQ_INSERT_TAIL(&p->table_types, elem, node);
+ /* Find the min and max offset fields. */
+ min_offset = offset[0];
+ max_offset = offset[0];
+ min_offset_pos = 0;
+ max_offset_pos = 0;
- return 0;
-}
+ for (i = 1; i < params->n_fields; i++) {
+ if (offset[i] < min_offset) {
+ min_offset = offset[i];
+ min_offset_pos = i;
+ }
-static enum rte_swx_table_match_type
-table_match_type_resolve(struct rte_swx_match_field_params *fields,
- uint32_t n_fields)
-{
- uint32_t i;
+ if (offset[i] > max_offset) {
+ max_offset = offset[i];
+ max_offset_pos = i;
+ }
+ }
- for (i = 0; i < n_fields; i++)
- if (fields[i].match_type != RTE_SWX_TABLE_MATCH_EXACT)
- break;
+ /* Return. */
+ if (header)
+ *header = h0;
- if (i == n_fields)
- return RTE_SWX_TABLE_MATCH_EXACT;
+ if (min_offset_field_id)
+ *min_offset_field_id = min_offset_pos;
- if ((i == n_fields - 1) &&
- (fields[i].match_type == RTE_SWX_TABLE_MATCH_LPM))
- return RTE_SWX_TABLE_MATCH_LPM;
+ if (max_offset_field_id)
+ *max_offset_field_id = max_offset_pos;
- return RTE_SWX_TABLE_MATCH_WILDCARD;
+end:
+ free(offset);
+ return status;
}
int
struct table *t;
struct action *default_action;
struct header *header = NULL;
- int is_header = 0;
- uint32_t offset_prev = 0, action_data_size_max = 0, i;
+ uint32_t action_data_size_max = 0, min_offset_field_id = 0, max_offset_field_id = 0, i;
+ int status = 0;
CHECK(p, EINVAL);
CHECK(params, EINVAL);
/* Match checks. */
- CHECK(!params->n_fields || params->fields, EINVAL);
- for (i = 0; i < params->n_fields; i++) {
- struct rte_swx_match_field_params *field = ¶ms->fields[i];
- struct header *h;
- struct field *hf, *mf;
- uint32_t offset;
-
- CHECK_NAME(field->name, EINVAL);
-
- hf = header_field_parse(p, field->name, &h);
- mf = metadata_field_parse(p, field->name);
- CHECK(hf || mf, EINVAL);
-
- offset = hf ? hf->offset : mf->offset;
-
- if (i == 0) {
- is_header = hf ? 1 : 0;
- header = hf ? h : NULL;
- offset_prev = offset;
-
- continue;
- }
-
- CHECK((is_header && hf && (h->id == header->id)) ||
- (!is_header && mf), EINVAL);
-
- CHECK(offset > offset_prev, EINVAL);
- offset_prev = offset;
- }
+ status = table_match_fields_check(p,
+ params,
+ &header,
+ &min_offset_field_id,
+ &max_offset_field_id);
+ if (status)
+ return status;
/* Action checks. */
CHECK(params->n_actions, EINVAL);
enum rte_swx_table_match_type match_type;
match_type = table_match_type_resolve(params->fields,
- params->n_fields);
+ params->n_fields,
+ max_offset_field_id);
type = table_type_resolve(p,
recommended_table_type_name,
match_type);
struct match_field *f = &t->fields[i];
f->match_type = field->match_type;
- f->field = is_header ?
+ f->field = header ?
header_field_parse(p, field->name, NULL) :
metadata_field_parse(p, field->name);
}
t->n_fields = params->n_fields;
- t->is_header = is_header;
t->header = header;
for (i = 0; i < params->n_actions; i++)
if (!params)
return NULL;
- /* Key offset and size. */
+ /* Find first (smallest offset) and last (biggest offset) match fields. */
first = table->fields[0].field;
- last = table->fields[table->n_fields - 1].field;
+ last = table->fields[0].field;
+
+ for (i = 0; i < table->n_fields; i++) {
+ struct field *f = table->fields[i].field;
+
+ if (f->offset < first->offset)
+ first = f;
+
+ if (f->offset > last->offset)
+ last = f;
+ }
+
+ /* Key offset and size. */
key_offset = first->offset / 8;
key_size = (last->offset + last->n_bits - first->offset) / 8;
{
uint32_t i;
+ /* Per pipeline: table statistics. */
+ p->table_stats = calloc(p->n_tables, sizeof(struct table_statistics));
+ CHECK(p->table_stats, ENOMEM);
+
+ for (i = 0; i < p->n_tables; i++) {
+ p->table_stats[i].n_pkts_action = calloc(p->n_actions, sizeof(uint64_t));
+ CHECK(p->table_stats[i].n_pkts_action, ENOMEM);
+ }
+
+ /* Per thread: table runt-time. */
for (i = 0; i < RTE_SWX_PIPELINE_THREADS_MAX; i++) {
struct thread *t = &p->threads[i];
struct table *table;
}
/* r->key. */
- r->key = table->is_header ?
+ r->key = table->header ?
&t->structs[table->header->struct_id] :
&t->structs[p->metadata_struct_id];
} else {
free(t->tables);
t->tables = NULL;
}
+
+ if (p->table_stats) {
+ for (i = 0; i < p->n_tables; i++)
+ free(p->table_stats[i].n_pkts_action);
+
+ free(p->table_stats);
+ }
}
static void
}
}
+/*
+ * Meter array.
+ */
+static struct meter_profile *
+meter_profile_find(struct rte_swx_pipeline *p, const char *name)
+{
+ struct meter_profile *elem;
+
+ TAILQ_FOREACH(elem, &p->meter_profiles, node)
+ if (!strcmp(elem->name, name))
+ return elem;
+
+ return NULL;
+}
+
+static struct metarray *
+metarray_find(struct rte_swx_pipeline *p, const char *name)
+{
+ struct metarray *elem;
+
+ TAILQ_FOREACH(elem, &p->metarrays, node)
+ if (!strcmp(elem->name, name))
+ return elem;
+
+ return NULL;
+}
+
+static struct metarray *
+metarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id)
+{
+ struct metarray *elem = NULL;
+
+ TAILQ_FOREACH(elem, &p->metarrays, node)
+ if (elem->id == id)
+ return elem;
+
+ return NULL;
+}
+
+int
+rte_swx_pipeline_metarray_config(struct rte_swx_pipeline *p,
+ const char *name,
+ uint32_t size)
+{
+ struct metarray *m;
+
+ CHECK(p, EINVAL);
+
+ CHECK_NAME(name, EINVAL);
+ CHECK(!metarray_find(p, name), EEXIST);
+
+ CHECK(size, EINVAL);
+ size = rte_align32pow2(size);
+
+ /* Memory allocation. */
+ m = calloc(1, sizeof(struct metarray));
+ CHECK(m, ENOMEM);
+
+ /* Node initialization. */
+ strcpy(m->name, name);
+ m->size = size;
+ m->id = p->n_metarrays;
+
+ /* Node add to tailq. */
+ TAILQ_INSERT_TAIL(&p->metarrays, m, node);
+ p->n_metarrays++;
+
+ return 0;
+}
+
+struct meter_profile meter_profile_default = {
+ .node = {0},
+ .name = "",
+ .params = {0},
+
+ .profile = {
+ .cbs = 10000,
+ .pbs = 10000,
+ .cir_period = 1,
+ .cir_bytes_per_period = 1,
+ .pir_period = 1,
+ .pir_bytes_per_period = 1,
+ },
+
+ .n_users = 0,
+};
+
+static void
+meter_init(struct meter *m)
+{
+ memset(m, 0, sizeof(struct meter));
+ rte_meter_trtcm_config(&m->m, &meter_profile_default.profile);
+ m->profile = &meter_profile_default;
+ m->color_mask = RTE_COLOR_GREEN;
+
+ meter_profile_default.n_users++;
+}
+
+static int
+metarray_build(struct rte_swx_pipeline *p)
+{
+ struct metarray *m;
+
+ if (!p->n_metarrays)
+ return 0;
+
+ p->metarray_runtime = calloc(p->n_metarrays, sizeof(struct metarray_runtime));
+ CHECK(p->metarray_runtime, ENOMEM);
+
+ TAILQ_FOREACH(m, &p->metarrays, node) {
+ struct metarray_runtime *r = &p->metarray_runtime[m->id];
+ uint32_t i;
+
+ r->metarray = env_malloc(m->size * sizeof(struct meter),
+ RTE_CACHE_LINE_SIZE,
+ p->numa_node);
+ CHECK(r->metarray, ENOMEM);
+
+ for (i = 0; i < m->size; i++)
+ meter_init(&r->metarray[i]);
+
+ r->size_mask = m->size - 1;
+ }
+
+ return 0;
+}
+
+static void
+metarray_build_free(struct rte_swx_pipeline *p)
+{
+ uint32_t i;
+
+ if (!p->metarray_runtime)
+ return;
+
+ for (i = 0; i < p->n_metarrays; i++) {
+ struct metarray *m = metarray_find_by_id(p, i);
+ struct metarray_runtime *r = &p->metarray_runtime[i];
+
+ env_free(r->metarray, m->size * sizeof(struct meter));
+ }
+
+ free(p->metarray_runtime);
+ p->metarray_runtime = NULL;
+}
+
+static void
+metarray_free(struct rte_swx_pipeline *p)
+{
+ metarray_build_free(p);
+
+ /* Meter arrays. */
+ for ( ; ; ) {
+ struct metarray *elem;
+
+ elem = TAILQ_FIRST(&p->metarrays);
+ if (!elem)
+ break;
+
+ TAILQ_REMOVE(&p->metarrays, elem, node);
+ free(elem);
+ }
+
+ /* Meter profiles. */
+ for ( ; ; ) {
+ struct meter_profile *elem;
+
+ elem = TAILQ_FIRST(&p->meter_profiles);
+ if (!elem)
+ break;
+
+ TAILQ_REMOVE(&p->meter_profiles, elem, node);
+ free(elem);
+ }
+}
+
/*
* Pipeline.
*/
TAILQ_INIT(&pipeline->table_types);
TAILQ_INIT(&pipeline->tables);
TAILQ_INIT(&pipeline->regarrays);
+ TAILQ_INIT(&pipeline->meter_profiles);
+ TAILQ_INIT(&pipeline->metarrays);
pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */
pipeline->numa_node = numa_node;
free(p->instructions);
+ metarray_free(p);
regarray_free(p);
table_state_free(p);
table_free(p);
if (status)
goto error;
+ status = metarray_build(p);
+ if (status)
+ goto error;
+
p->build_done = 1;
return 0;
error:
+ metarray_build_free(p);
regarray_build_free(p);
table_state_build_free(p);
table_build_free(p);
pipeline->n_actions = n_actions;
pipeline->n_tables = n_tables;
pipeline->n_regarrays = p->n_regarrays;
+ pipeline->n_metarrays = p->n_metarrays;
return 0;
}
f = &t->fields[match_field_id];
match_field->match_type = f->match_type;
- match_field->is_header = t->is_header;
+ match_field->is_header = t->header ? 1 : 0;
match_field->n_bits = f->field->n_bits;
match_field->offset = f->field->offset;
return 0;
}
+int
+rte_swx_ctl_pipeline_table_stats_read(struct rte_swx_pipeline *p,
+ const char *table_name,
+ struct rte_swx_table_stats *stats)
+{
+ struct table *table;
+ struct table_statistics *table_stats;
+
+ if (!p || !table_name || !table_name[0] || !stats || !stats->n_pkts_action)
+ return -EINVAL;
+
+ table = table_find(p, table_name);
+ if (!table)
+ return -EINVAL;
+
+ table_stats = &p->table_stats[table->id];
+
+ memcpy(&stats->n_pkts_action,
+ &table_stats->n_pkts_action,
+ p->n_actions * sizeof(uint64_t));
+
+ stats->n_pkts_hit = table_stats->n_pkts_hit[1];
+ stats->n_pkts_miss = table_stats->n_pkts_hit[0];
+
+ return 0;
+}
+
int
rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p,
uint32_t regarray_id,
r->regarray[regarray_index] = value;
return 0;
}
+
+int
+rte_swx_ctl_metarray_info_get(struct rte_swx_pipeline *p,
+ uint32_t metarray_id,
+ struct rte_swx_ctl_metarray_info *metarray)
+{
+ struct metarray *m;
+
+ if (!p || !metarray)
+ return -EINVAL;
+
+ m = metarray_find_by_id(p, metarray_id);
+ if (!m)
+ return -EINVAL;
+
+ strcpy(metarray->name, m->name);
+ metarray->size = m->size;
+ return 0;
+}
+
+int
+rte_swx_ctl_meter_profile_add(struct rte_swx_pipeline *p,
+ const char *name,
+ struct rte_meter_trtcm_params *params)
+{
+ struct meter_profile *mp;
+ int status;
+
+ CHECK(p, EINVAL);
+ CHECK_NAME(name, EINVAL);
+ CHECK(params, EINVAL);
+ CHECK(!meter_profile_find(p, name), EEXIST);
+
+ /* Node allocation. */
+ mp = calloc(1, sizeof(struct meter_profile));
+ CHECK(mp, ENOMEM);
+
+ /* Node initialization. */
+ strcpy(mp->name, name);
+ memcpy(&mp->params, params, sizeof(struct rte_meter_trtcm_params));
+ status = rte_meter_trtcm_profile_config(&mp->profile, params);
+ if (status) {
+ free(mp);
+ CHECK(0, EINVAL);
+ }
+
+ /* Node add to tailq. */
+ TAILQ_INSERT_TAIL(&p->meter_profiles, mp, node);
+
+ return 0;
+}
+
+int
+rte_swx_ctl_meter_profile_delete(struct rte_swx_pipeline *p,
+ const char *name)
+{
+ struct meter_profile *mp;
+
+ CHECK(p, EINVAL);
+ CHECK_NAME(name, EINVAL);
+
+ mp = meter_profile_find(p, name);
+ CHECK(mp, EINVAL);
+ CHECK(!mp->n_users, EBUSY);
+
+ /* Remove node from tailq. */
+ TAILQ_REMOVE(&p->meter_profiles, mp, node);
+ free(mp);
+
+ return 0;
+}
+
+int
+rte_swx_ctl_meter_reset(struct rte_swx_pipeline *p,
+ const char *metarray_name,
+ uint32_t metarray_index)
+{
+ struct meter_profile *mp_old;
+ struct metarray *metarray;
+ struct metarray_runtime *metarray_runtime;
+ struct meter *m;
+
+ CHECK(p, EINVAL);
+ CHECK_NAME(metarray_name, EINVAL);
+
+ metarray = metarray_find(p, metarray_name);
+ CHECK(metarray, EINVAL);
+ CHECK(metarray_index < metarray->size, EINVAL);
+
+ metarray_runtime = &p->metarray_runtime[metarray->id];
+ m = &metarray_runtime->metarray[metarray_index];
+ mp_old = m->profile;
+
+ meter_init(m);
+
+ mp_old->n_users--;
+
+ return 0;
+}
+
+int
+rte_swx_ctl_meter_set(struct rte_swx_pipeline *p,
+ const char *metarray_name,
+ uint32_t metarray_index,
+ const char *profile_name)
+{
+ struct meter_profile *mp, *mp_old;
+ struct metarray *metarray;
+ struct metarray_runtime *metarray_runtime;
+ struct meter *m;
+
+ CHECK(p, EINVAL);
+ CHECK_NAME(metarray_name, EINVAL);
+
+ metarray = metarray_find(p, metarray_name);
+ CHECK(metarray, EINVAL);
+ CHECK(metarray_index < metarray->size, EINVAL);
+
+ mp = meter_profile_find(p, profile_name);
+ CHECK(mp, EINVAL);
+
+ metarray_runtime = &p->metarray_runtime[metarray->id];
+ m = &metarray_runtime->metarray[metarray_index];
+ mp_old = m->profile;
+
+ memset(m, 0, sizeof(struct meter));
+ rte_meter_trtcm_config(&m->m, &mp->profile);
+ m->profile = mp;
+ m->color_mask = RTE_COLORS;
+
+ mp->n_users++;
+ mp_old->n_users--;
+
+ return 0;
+}
+
+int
+rte_swx_ctl_meter_stats_read(struct rte_swx_pipeline *p,
+ const char *metarray_name,
+ uint32_t metarray_index,
+ struct rte_swx_ctl_meter_stats *stats)
+{
+ struct metarray *metarray;
+ struct metarray_runtime *metarray_runtime;
+ struct meter *m;
+
+ CHECK(p, EINVAL);
+ CHECK_NAME(metarray_name, EINVAL);
+
+ metarray = metarray_find(p, metarray_name);
+ CHECK(metarray, EINVAL);
+ CHECK(metarray_index < metarray->size, EINVAL);
+
+ CHECK(stats, EINVAL);
+
+ metarray_runtime = &p->metarray_runtime[metarray->id];
+ m = &metarray_runtime->metarray[metarray_index];
+
+ memcpy(stats->n_pkts, m->n_pkts, sizeof(m->n_pkts));
+ memcpy(stats->n_bytes, m->n_bytes, sizeof(m->n_bytes));
+
+ return 0;
+}
git.droids-corp.org / dpdk.git / blobdiff