#define MLX5_IPV4 4
#define MLX5_IPV6 6
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+struct ibv_counter_set_init_attr {
+ int dummy;
+};
+struct ibv_flow_spec_counter_action {
+ int dummy;
+};
+struct ibv_counter_set {
+ int dummy;
+};
+
+static inline int
+ibv_destroy_counter_set(struct ibv_counter_set *cs)
+{
+ (void)cs;
+ return -ENOTSUP;
+}
+#endif
+
+/* Dev ops structure defined in mlx5.c */
+extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_dev_ops_isolate;
+
static int
mlx5_flow_create_eth(const struct rte_flow_item *item,
const void *default_mask,
static int
mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
+static int
+mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
+
/* Hash RX queue types. */
enum hash_rxq_type {
HASH_RXQ_TCPV4,
/* Number of entries in hash_rxq_init[]. */
const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init);
+/** Structure for holding counter stats. */
+struct mlx5_flow_counter_stats {
+ uint64_t hits; /**< Number of packets matched by the rule. */
+ uint64_t bytes; /**< Number of bytes matched by the rule. */
+};
+
/** Structure for Drop queue. */
struct mlx5_hrxq_drop {
struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
uint16_t (*queues)[]; /**< Queues indexes to use. */
struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
uint8_t rss_key[40]; /**< copy of the RSS key. */
+ struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+ struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
union {
struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
/**< Flow with Rx queue. */
RTE_FLOW_ACTION_TYPE_QUEUE,
RTE_FLOW_ACTION_TYPE_MARK,
RTE_FLOW_ACTION_TYPE_FLAG,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+ RTE_FLOW_ACTION_TYPE_COUNT,
+#endif
RTE_FLOW_ACTION_TYPE_END,
};
/**< Whether resources should remain after a validate. */
uint32_t drop:1; /**< Target is a drop queue. */
uint32_t mark:1; /**< Mark is present in the flow. */
+ uint32_t count:1; /**< Count is present in the flow. */
uint32_t mark_id; /**< Mark identifier. */
uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
uint16_t queues_n; /**< Number of entries in queue[]. */
struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
uint8_t rss_key[40]; /**< copy of the RSS key. */
enum hash_rxq_type layer; /**< Last pattern layer detected. */
+ struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
union {
struct {
struct ibv_flow_attr *ibv_attr;
.create = mlx5_flow_create,
.destroy = mlx5_flow_destroy,
.flush = mlx5_flow_flush,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+ .query = mlx5_flow_query,
+#else
.query = NULL,
+#endif
.isolate = mlx5_flow_isolate,
};
struct rte_flow_action actions[2];
struct rte_flow_item items[4];
struct rte_flow_item_eth l2;
+ struct rte_flow_item_eth l2_mask;
union {
struct rte_flow_item_ipv4 ipv4;
struct rte_flow_item_ipv6 ipv6;
parser->mark_id = mark->id;
} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
parser->mark = 1;
+ } else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
+ priv->counter_set_supported) {
+ parser->count = 1;
} else {
goto exit_action_not_supported;
}
}
+ if (parser->drop && parser->mark)
+ parser->mark = 0;
if (!parser->queues_n && !parser->drop) {
rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
NULL, "no valid action");
" VXLAN encapsulations");
return -rte_errno;
}
- parser->inner = 1;
+ parser->inner = IBV_FLOW_SPEC_INNER;
}
if (parser->drop) {
parser->drop_q.offset += cur_item->dst_sz;
parser->queue[i].offset +=
sizeof(struct ibv_flow_spec_action_tag);
}
+ if (parser->count) {
+ unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+
+ if (parser->drop) {
+ parser->drop_q.offset += size;
+ } else {
+ for (i = 0; i != hash_rxq_init_n; ++i)
+ parser->queue[i].offset += size;
+ }
+ }
return 0;
exit_item_not_supported:
rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
size = sizeof(struct ibv_flow_spec_ipv4_ext);
specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
- .type = IBV_FLOW_SPEC_IPV4_EXT |
- parser->inner,
+ .type = IBV_FLOW_SPEC_IPV4_EXT,
.size = size,
};
} else {
size = sizeof(struct ibv_flow_spec_ipv6);
specs.ipv6 = (struct ibv_flow_spec_ipv6){
- .type = IBV_FLOW_SPEC_IPV6 |
- parser->inner,
+ .type = IBV_FLOW_SPEC_IPV6,
.size = size,
};
}
.type = ((i == HASH_RXQ_UDPV4 ||
i == HASH_RXQ_UDPV6) ?
IBV_FLOW_SPEC_UDP :
- IBV_FLOW_SPEC_TCP) |
- parser->inner,
+ IBV_FLOW_SPEC_TCP),
.size = size,
};
if (parser->queue[i].ibv_attr) {
cur_item->mask),
parser);
if (ret) {
- rte_flow_error_set(error, ENOTSUP,
+ rte_flow_error_set(error, ret,
RTE_FLOW_ERROR_TYPE_ITEM,
items, "item not supported");
goto exit_free;
}
if (parser->mark)
mlx5_flow_create_flag_mark(parser, parser->mark_id);
+ if (parser->count && parser->create) {
+ mlx5_flow_create_count(priv, parser);
+ if (!parser->cs)
+ goto exit_count_error;
+ }
/*
* Last step. Complete missing specification to reach the RSS
* configuration.
*/
- if (parser->queues_n > 1)
+ if (parser->drop) {
+ /*
+ * Drop queue priority needs to be adjusted to
+ * their most specific layer priority.
+ */
+ parser->drop_q.ibv_attr->priority =
+ attr->priority +
+ hash_rxq_init[parser->layer].flow_priority;
+ } else if (parser->queues_n > 1) {
priv_flow_convert_finalise(priv, parser);
+ } else {
+ /*
+ * Action queue have their priority overridden with
+ * Ethernet priority, this priority needs to be adjusted to
+ * their most specific layer priority.
+ */
+ parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
+ attr->priority +
+ hash_rxq_init[parser->layer].flow_priority;
+ }
exit_free:
/* Only verification is expected, all resources should be released. */
if (!parser->create) {
rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "cannot allocate verbs spec attributes.");
return ret;
+exit_count_error:
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "cannot create counter.");
+ return rte_errno;
}
/**
.size = eth_size,
};
- parser->layer = HASH_RXQ_ETH;
+ /* Don't update layer for the inner pattern. */
+ if (!parser->inner)
+ parser->layer = HASH_RXQ_ETH;
if (spec) {
unsigned int i;
.size = ipv4_size,
};
- parser->layer = HASH_RXQ_IPV4;
+ /* Don't update layer for the inner pattern. */
+ if (!parser->inner)
+ parser->layer = HASH_RXQ_IPV4;
if (spec) {
if (!mask)
mask = default_mask;
.size = ipv6_size,
};
- parser->layer = HASH_RXQ_IPV6;
+ /* Don't update layer for the inner pattern. */
+ if (!parser->inner)
+ parser->layer = HASH_RXQ_IPV6;
if (spec) {
unsigned int i;
.size = udp_size,
};
- if (parser->layer == HASH_RXQ_IPV4)
- parser->layer = HASH_RXQ_UDPV4;
- else
- parser->layer = HASH_RXQ_UDPV6;
+ /* Don't update layer for the inner pattern. */
+ if (!parser->inner) {
+ if (parser->layer == HASH_RXQ_IPV4)
+ parser->layer = HASH_RXQ_UDPV4;
+ else
+ parser->layer = HASH_RXQ_UDPV6;
+ }
if (spec) {
if (!mask)
mask = default_mask;
.size = tcp_size,
};
- if (parser->layer == HASH_RXQ_IPV4)
- parser->layer = HASH_RXQ_TCPV4;
- else
- parser->layer = HASH_RXQ_TCPV6;
+ /* Don't update layer for the inner pattern. */
+ if (!parser->inner) {
+ if (parser->layer == HASH_RXQ_IPV4)
+ parser->layer = HASH_RXQ_TCPV4;
+ else
+ parser->layer = HASH_RXQ_TCPV6;
+ }
if (spec) {
if (!mask)
mask = default_mask;
/* Remove unwanted bits from values. */
vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
}
+ /*
+ * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+ * layer is defined in the Verbs specification it is interpreted as
+ * wildcard and all packets will match this rule, if it follows a full
+ * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+ * before will also match this rule.
+ * To avoid such situation, VNI 0 is currently refused.
+ */
+ if (!vxlan.val.tunnel_id)
+ return EINVAL;
mlx5_flow_create_copy(parser, &vxlan, size);
return 0;
}
return 0;
}
+/**
+ * Convert count action to Verbs specification.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param parser
+ * Pointer to MLX5 flow parser structure.
+ *
+ * @return
+ * 0 on success, errno value on failure.
+ */
+static int
+mlx5_flow_create_count(struct priv *priv __rte_unused,
+ struct mlx5_flow_parse *parser __rte_unused)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+ unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+ struct ibv_counter_set_init_attr init_attr = {0};
+ struct ibv_flow_spec_counter_action counter = {
+ .type = IBV_FLOW_SPEC_ACTION_COUNT,
+ .size = size,
+ .counter_set_handle = 0,
+ };
+
+ init_attr.counter_set_id = 0;
+ parser->cs = ibv_create_counter_set(priv->ctx, &init_attr);
+ if (!parser->cs)
+ return EINVAL;
+ counter.counter_set_handle = parser->cs->handle;
+ mlx5_flow_create_copy(parser, &counter, size);
+#endif
+ return 0;
+}
+
/**
* Complete flow rule creation with a drop queue.
*
};
++parser->drop_q.ibv_attr->num_of_specs;
parser->drop_q.offset += size;
+ flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
+ if (parser->count)
+ flow->cs = parser->cs;
if (!priv->dev->data->dev_started)
return 0;
- flow->drxq.ibv_attr = parser->drop_q.ibv_attr;
parser->drop_q.ibv_attr = NULL;
flow->drxq.ibv_flow = ibv_create_flow(priv->flow_drop_queue->qp,
flow->drxq.ibv_attr);
rte_free(flow->drxq.ibv_attr);
flow->drxq.ibv_attr = NULL;
}
+ if (flow->cs) {
+ claim_zero(ibv_destroy_counter_set(flow->cs));
+ flow->cs = NULL;
+ parser->cs = NULL;
+ }
return err;
}
flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
parser->queue[i].ibv_attr = NULL;
hash_fields = hash_rxq_init[i].hash_fields;
+ if (!priv->dev->data->dev_started)
+ continue;
flow->frxq[i].hrxq =
mlx5_priv_hrxq_get(priv,
parser->rss_conf.rss_key,
parser->rss_conf.rss_key_len,
hash_fields,
parser->queues,
- hash_fields ? parser->queues_n : 1);
+ parser->queues_n);
if (flow->frxq[i].hrxq)
continue;
flow->frxq[i].hrxq =
parser->rss_conf.rss_key_len,
hash_fields,
parser->queues,
- hash_fields ? parser->queues_n : 1);
+ parser->queues_n);
if (!flow->frxq[i].hrxq) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_HANDLE,
err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
if (err)
goto error;
+ if (parser->count)
+ flow->cs = parser->cs;
if (!priv->dev->data->dev_started)
return 0;
for (i = 0; i != hash_rxq_init_n; ++i) {
if (flow->frxq[i].ibv_attr)
rte_free(flow->frxq[i].ibv_attr);
}
+ if (flow->cs) {
+ claim_zero(ibv_destroy_counter_set(flow->cs));
+ flow->cs = NULL;
+ parser->cs = NULL;
+ }
return err;
}
rte_free(frxq->ibv_attr);
}
}
+ if (flow->cs) {
+ claim_zero(ibv_destroy_counter_set(flow->cs));
+ flow->cs = NULL;
+ }
TAILQ_REMOVE(list, flow, next);
DEBUG("Flow destroyed %p", (void *)flow);
rte_free(flow);
return 0;
}
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+/**
+ * Query flow counter.
+ *
+ * @param cs
+ * the counter set.
+ * @param counter_value
+ * returned data from the counter.
+ *
+ * @return
+ * 0 on success, a errno value otherwise and rte_errno is set.
+ */
+static int
+priv_flow_query_count(struct ibv_counter_set *cs,
+ struct mlx5_flow_counter_stats *counter_stats,
+ struct rte_flow_query_count *query_count,
+ struct rte_flow_error *error)
+{
+ uint64_t counters[2];
+ struct ibv_query_counter_set_attr query_cs_attr = {
+ .cs = cs,
+ .query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+ };
+ struct ibv_counter_set_data query_out = {
+ .out = counters,
+ .outlen = 2 * sizeof(uint64_t),
+ };
+ int res = ibv_query_counter_set(&query_cs_attr, &query_out);
+
+ if (res) {
+ rte_flow_error_set(error, -res,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "cannot read counter");
+ return -res;
+ }
+ query_count->hits_set = 1;
+ query_count->bytes_set = 1;
+ query_count->hits = counters[0] - counter_stats->hits;
+ query_count->bytes = counters[1] - counter_stats->bytes;
+ if (query_count->reset) {
+ counter_stats->hits = counters[0];
+ counter_stats->bytes = counters[1];
+ }
+ return 0;
+}
+
+/**
+ * Query a flows.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_query(struct rte_eth_dev *dev,
+ struct rte_flow *flow,
+ enum rte_flow_action_type action __rte_unused,
+ void *data,
+ struct rte_flow_error *error)
+{
+ struct priv *priv = dev->data->dev_private;
+ int res = EINVAL;
+
+ priv_lock(priv);
+ if (flow->cs) {
+ res = priv_flow_query_count(flow->cs,
+ &flow->counter_stats,
+ (struct rte_flow_query_count *)data,
+ error);
+ } else {
+ rte_flow_error_set(error, res,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "no counter found for flow");
+ }
+ priv_unlock(priv);
+ return -res;
+}
+#endif
+
/**
* Isolated mode.
*
return -rte_errno;
}
priv->isolated = !!enable;
+ if (enable)
+ priv->dev->dev_ops = &mlx5_dev_ops_isolate;
+ else
+ priv->dev->dev_ops = &mlx5_dev_ops;
priv_unlock(priv);
return 0;
}
ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
return EINVAL;
}
- /* Validate the behavior. */
- if (fdir_filter->action.behavior != RTE_ETH_FDIR_ACCEPT) {
- ERROR("invalid behavior %d", fdir_filter->action.behavior);
- return ENOTSUP;
- }
attributes->attr.ingress = 1;
attributes->items[0] = (struct rte_flow_item) {
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = &attributes->l2,
+ .mask = &attributes->l2_mask,
};
- attributes->actions[0] = (struct rte_flow_action){
- .type = RTE_FLOW_ACTION_TYPE_QUEUE,
- .conf = &attributes->queue,
- };
+ switch (fdir_filter->action.behavior) {
+ case RTE_ETH_FDIR_ACCEPT:
+ attributes->actions[0] = (struct rte_flow_action){
+ .type = RTE_FLOW_ACTION_TYPE_QUEUE,
+ .conf = &attributes->queue,
+ };
+ break;
+ case RTE_ETH_FDIR_REJECT:
+ attributes->actions[0] = (struct rte_flow_action){
+ .type = RTE_FLOW_ACTION_TYPE_DROP,
+ };
+ break;
+ default:
+ ERROR("invalid behavior %d", fdir_filter->action.behavior);
+ return ENOTSUP;
+ }
attributes->queue.index = fdir_filter->action.rx_queue;
switch (fdir_filter->input.flow_type) {
case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
.spec = &attributes->l3,
};
attributes->items[2] = (struct rte_flow_item){
- .type = RTE_FLOW_ITEM_TYPE_UDP,
+ .type = RTE_FLOW_ITEM_TYPE_TCP,
.spec = &attributes->l4,
};
break;
{
struct mlx5_fdir attributes = {
.attr.group = 0,
+ .l2_mask = {
+ .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .type = 0,
+ },
};
struct mlx5_flow_parse parser = {
.layer = HASH_RXQ_ETH,
attributes.actions,
&error);
if (flow) {
- TAILQ_INSERT_TAIL(&priv->flows, flow, next);
DEBUG("FDIR created %p", (void *)flow);
return 0;
}
priv_fdir_filter_delete(struct priv *priv,
const struct rte_eth_fdir_filter *fdir_filter)
{
- struct mlx5_fdir attributes;
+ struct mlx5_fdir attributes = {
+ .attr.group = 0,
+ };
struct mlx5_flow_parse parser = {
.create = 1,
.layer = HASH_RXQ_ETH,
attributes.actions, &error, &parser);
if (ret)
goto exit;
+ /*
+ * Special case for drop action which is only set in the
+ * specifications when the flow is created. In this situation the
+ * drop specification is missing.
+ */
+ if (parser.drop) {
+ struct ibv_flow_spec_action_drop *drop;
+
+ drop = (void *)((uintptr_t)parser.drop_q.ibv_attr +
+ parser.drop_q.offset);
+ *drop = (struct ibv_flow_spec_action_drop){
+ .type = IBV_FLOW_SPEC_ACTION_DROP,
+ .size = sizeof(struct ibv_flow_spec_action_drop),
+ };
+ parser.drop_q.ibv_attr->num_of_specs++;
+ }
TAILQ_FOREACH(flow, &priv->flows, next) {
struct ibv_flow_attr *attr;
struct ibv_spec_header *attr_h;
flow_h = flow_spec;
if (memcmp(spec, flow_spec,
RTE_MIN(attr_h->size, flow_h->size)))
- continue;
- spec = (void *)((uintptr_t)attr + attr_h->size);
- flow_spec = (void *)((uintptr_t)flow_attr +
+ goto wrong_flow;
+ spec = (void *)((uintptr_t)spec + attr_h->size);
+ flow_spec = (void *)((uintptr_t)flow_spec +
flow_h->size);
}
/* At this point, the flow match. */
break;
+wrong_flow:
+ /* The flow does not match. */
+ continue;
}
if (flow)
priv_flow_destroy(priv, &priv->flows, flow);