#include <rte_byteorder.h>
#include <rte_errno.h>
#include <rte_eth_ctrl.h>
-#include <rte_ethdev.h>
+#include <rte_ethdev_driver.h>
#include <rte_ether.h>
#include <rte_flow.h>
#include <rte_flow_driver.h>
uint32_t refcnt; /**< Reference count. */
};
+/**
+ * Convert DPDK RSS hash fields to their Verbs equivalent.
+ *
+ * This function returns the supported (default) set when @p rss_hf has
+ * special value (uint64_t)-1.
+ *
+ * @param priv
+ * Pointer to private structure.
+ * @param rss_hf
+ * Hash fields in DPDK format (see struct rte_eth_rss_conf).
+ *
+ * @return
+ * A valid Verbs RSS hash fields mask for mlx4 on success, (uint64_t)-1
+ * otherwise and rte_errno is set.
+ */
+uint64_t
+mlx4_conv_rss_hf(struct priv *priv, uint64_t rss_hf)
+{
+ enum { IPV4, IPV6, TCP, UDP, };
+ const uint64_t in[] = {
+ [IPV4] = (ETH_RSS_IPV4 |
+ ETH_RSS_FRAG_IPV4 |
+ ETH_RSS_NONFRAG_IPV4_TCP |
+ ETH_RSS_NONFRAG_IPV4_UDP |
+ ETH_RSS_NONFRAG_IPV4_OTHER),
+ [IPV6] = (ETH_RSS_IPV6 |
+ ETH_RSS_FRAG_IPV6 |
+ ETH_RSS_NONFRAG_IPV6_TCP |
+ ETH_RSS_NONFRAG_IPV6_UDP |
+ ETH_RSS_NONFRAG_IPV6_OTHER |
+ ETH_RSS_IPV6_EX |
+ ETH_RSS_IPV6_TCP_EX |
+ ETH_RSS_IPV6_UDP_EX),
+ [TCP] = (ETH_RSS_NONFRAG_IPV4_TCP |
+ ETH_RSS_NONFRAG_IPV6_TCP |
+ ETH_RSS_IPV6_TCP_EX),
+ [UDP] = (ETH_RSS_NONFRAG_IPV4_UDP |
+ ETH_RSS_NONFRAG_IPV6_UDP |
+ ETH_RSS_IPV6_UDP_EX),
+ };
+ const uint64_t out[RTE_DIM(in)] = {
+ [IPV4] = IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4,
+ [IPV6] = IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6,
+ [TCP] = IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP,
+ [UDP] = IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP,
+ };
+ uint64_t seen = 0;
+ uint64_t conv = 0;
+ unsigned int i;
+
+ for (i = 0; i != RTE_DIM(in); ++i)
+ if (rss_hf & in[i]) {
+ seen |= rss_hf & in[i];
+ conv |= out[i];
+ }
+ if ((conv & priv->hw_rss_sup) == conv) {
+ if (rss_hf == (uint64_t)-1) {
+ /* Include inner RSS by default if supported. */
+ conv |= priv->hw_rss_sup & IBV_RX_HASH_INNER;
+ return conv;
+ }
+ if (!(rss_hf & ~seen))
+ return conv;
+ }
+ rte_errno = ENOTSUP;
+ return (uint64_t)-1;
+}
+
/**
* Merge Ethernet pattern item into flow rule handle.
*
struct ibv_flow_spec_tcp_udp *udp;
const char *msg;
- if (!mask ||
+ if (mask &&
((uint16_t)(mask->hdr.src_port + 1) > UINT16_C(1) ||
(uint16_t)(mask->hdr.dst_port + 1) > UINT16_C(1))) {
msg = "mlx4 does not support matching partial UDP fields";
struct ibv_flow_spec_tcp_udp *tcp;
const char *msg;
- if (!mask ||
+ if (mask &&
((uint16_t)(mask->hdr.src_port + 1) > UINT16_C(1) ||
(uint16_t)(mask->hdr.dst_port + 1) > UINT16_C(1))) {
msg = "mlx4 does not support matching partial TCP fields";
for (action = actions; action->type; ++action) {
switch (action->type) {
const struct rte_flow_action_queue *queue;
+ const struct rte_flow_action_rss *rss;
+ const struct rte_eth_rss_conf *rss_conf;
+ unsigned int i;
case RTE_FLOW_ACTION_TYPE_VOID:
continue;
flow->drop = 1;
break;
case RTE_FLOW_ACTION_TYPE_QUEUE:
+ if (flow->rss)
+ break;
queue = action->conf;
- if (queue->index >= priv->dev->data->nb_rx_queues)
+ if (queue->index >= priv->dev->data->nb_rx_queues) {
+ msg = "queue target index beyond number of"
+ " configured Rx queues";
+ goto exit_action_not_supported;
+ }
+ flow->rss = mlx4_rss_get
+ (priv, 0, mlx4_rss_hash_key_default, 1,
+ &queue->index);
+ if (!flow->rss) {
+ msg = "not enough resources for additional"
+ " single-queue RSS context";
+ goto exit_action_not_supported;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_RSS:
+ if (flow->rss)
+ break;
+ rss = action->conf;
+ /* Default RSS configuration if none is provided. */
+ rss_conf =
+ rss->rss_conf ?
+ rss->rss_conf :
+ &(struct rte_eth_rss_conf){
+ .rss_key = mlx4_rss_hash_key_default,
+ .rss_key_len = MLX4_RSS_HASH_KEY_SIZE,
+ .rss_hf = -1,
+ };
+ /* Sanity checks. */
+ for (i = 0; i < rss->num; ++i)
+ if (rss->queue[i] >=
+ priv->dev->data->nb_rx_queues)
+ break;
+ if (i != rss->num) {
+ msg = "queue index target beyond number of"
+ " configured Rx queues";
goto exit_action_not_supported;
- flow->queue = 1;
- flow->queue_id = queue->index;
+ }
+ if (!rte_is_power_of_2(rss->num)) {
+ msg = "for RSS, mlx4 requires the number of"
+ " queues to be a power of two";
+ goto exit_action_not_supported;
+ }
+ if (rss_conf->rss_key_len !=
+ sizeof(flow->rss->key)) {
+ msg = "mlx4 supports exactly one RSS hash key"
+ " length: "
+ MLX4_STR_EXPAND(MLX4_RSS_HASH_KEY_SIZE);
+ goto exit_action_not_supported;
+ }
+ for (i = 1; i < rss->num; ++i)
+ if (rss->queue[i] - rss->queue[i - 1] != 1)
+ break;
+ if (i != rss->num) {
+ msg = "mlx4 requires RSS contexts to use"
+ " consecutive queue indices only";
+ goto exit_action_not_supported;
+ }
+ if (rss->queue[0] % rss->num) {
+ msg = "mlx4 requires the first queue of a RSS"
+ " context to be aligned on a multiple"
+ " of the context size";
+ goto exit_action_not_supported;
+ }
+ flow->rss = mlx4_rss_get
+ (priv,
+ mlx4_conv_rss_hf(priv, rss_conf->rss_hf),
+ rss_conf->rss_key, rss->num, rss->queue);
+ if (!flow->rss) {
+ msg = "either invalid parameters or not enough"
+ " resources for additional multi-queue"
+ " RSS context";
+ goto exit_action_not_supported;
+ }
break;
default:
goto exit_action_not_supported;
}
}
- if (!flow->queue && !flow->drop)
+ if (!flow->rss && !flow->drop)
return rte_flow_error_set
(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "no valid action");
/* Validation ends here. */
- if (!addr)
+ if (!addr) {
+ if (flow->rss)
+ mlx4_rss_put(flow->rss);
return 0;
+ }
if (flow == &temp) {
/* Allocate proper handle based on collected data. */
const struct mlx4_malloc_vec vec[] = {
*flow = (struct rte_flow){
.ibv_attr = temp.ibv_attr,
.ibv_attr_size = sizeof(*flow->ibv_attr),
+ .rss = temp.rss,
};
*flow->ibv_attr = (struct ibv_flow_attr){
.type = IBV_FLOW_ATTR_NORMAL,
item, msg ? msg : "item not supported");
exit_action_not_supported:
return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
- action, "action not supported");
+ action, msg ? msg : "action not supported");
}
/**
flow->ibv_flow = NULL;
if (flow->drop)
mlx4_drop_put(priv->drop);
+ else if (flow->rss)
+ mlx4_rss_detach(flow->rss);
return 0;
}
assert(flow->ibv_attr);
flow->ibv_flow = NULL;
if (flow->drop)
mlx4_drop_put(priv->drop);
+ else if (flow->rss)
+ mlx4_rss_detach(flow->rss);
}
err = EACCES;
msg = ("priority level "
" is reserved when not in isolated mode");
goto error;
}
- if (flow->queue) {
- struct rxq *rxq = NULL;
+ if (flow->rss) {
+ struct mlx4_rss *rss = flow->rss;
+ int missing = 0;
+ unsigned int i;
- if (flow->queue_id < priv->dev->data->nb_rx_queues)
- rxq = priv->dev->data->rx_queues[flow->queue_id];
+ /* Stop at the first nonexistent target queue. */
+ for (i = 0; i != rss->queues; ++i)
+ if (rss->queue_id[i] >=
+ priv->dev->data->nb_rx_queues ||
+ !priv->dev->data->rx_queues[rss->queue_id[i]]) {
+ missing = 1;
+ break;
+ }
if (flow->ibv_flow) {
- if (!rxq ^ !flow->drop)
+ if (missing ^ !flow->drop)
return 0;
/* Verbs flow needs updating. */
claim_zero(ibv_destroy_flow(flow->ibv_flow));
flow->ibv_flow = NULL;
if (flow->drop)
mlx4_drop_put(priv->drop);
+ else
+ mlx4_rss_detach(rss);
+ }
+ if (!missing) {
+ err = mlx4_rss_attach(rss);
+ if (err) {
+ err = -err;
+ msg = "cannot create indirection table or hash"
+ " QP to associate flow rule with";
+ goto error;
+ }
+ qp = rss->qp;
}
- if (rxq)
- qp = rxq->qp;
/* A missing target queue drops traffic implicitly. */
- flow->drop = !rxq;
+ flow->drop = missing;
}
if (flow->drop) {
mlx4_drop_get(priv);
return 0;
if (flow->drop)
mlx4_drop_put(priv->drop);
+ else if (flow->rss)
+ mlx4_rss_detach(flow->rss);
err = errno;
msg = "flow rule rejected by device";
error:
}
return flow;
}
+ if (flow->rss)
+ mlx4_rss_put(flow->rss);
rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
error->message);
rte_free(flow);
if (err)
return err;
LIST_REMOVE(flow, next);
+ if (flow->rss)
+ mlx4_rss_put(flow->rss);
rte_free(flow);
return 0;
}
* - MAC flow rules are generated from @p dev->data->mac_addrs
* (@p priv->mac array).
* - An additional flow rule for Ethernet broadcasts is also generated.
- * - All these are per-VLAN if @p dev->data->dev_conf.rxmode.hw_vlan_filter
+ * - All these are per-VLAN if @p DEV_RX_OFFLOAD_VLAN_FILTER
* is enabled and VLAN filters are configured.
*
* @param priv
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
+ /*
+ * Round number of queues down to their previous power of 2 to
+ * comply with RSS context limitations. Extra queues silently do not
+ * get RSS by default.
+ */
+ uint32_t queues =
+ rte_align32pow2(priv->dev->data->nb_rx_queues + 1) >> 1;
+ alignas(struct rte_flow_action_rss) uint8_t rss_conf_data
+ [offsetof(struct rte_flow_action_rss, queue) +
+ sizeof(((struct rte_flow_action_rss *)0)->queue[0]) * queues];
+ struct rte_flow_action_rss *rss_conf = (void *)rss_conf_data;
struct rte_flow_action actions[] = {
{
- .type = RTE_FLOW_ACTION_TYPE_QUEUE,
- .conf = &(struct rte_flow_action_queue){
- .index = 0,
- },
+ .type = RTE_FLOW_ACTION_TYPE_RSS,
+ .conf = rss_conf,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
};
struct ether_addr *rule_mac = ð_spec.dst;
rte_be16_t *rule_vlan =
- priv->dev->data->dev_conf.rxmode.hw_vlan_filter &&
+ (priv->dev->data->dev_conf.rxmode.offloads &
+ DEV_RX_OFFLOAD_VLAN_FILTER) &&
!priv->dev->data->promiscuous ?
&vlan_spec.tci :
NULL;
unsigned int i;
int err = 0;
+ /* Nothing to be done if there are no Rx queues. */
+ if (!queues)
+ goto error;
+ /* Prepare default RSS configuration. */
+ *rss_conf = (struct rte_flow_action_rss){
+ .rss_conf = NULL, /* Rely on default fallback settings. */
+ .num = queues,
+ };
+ for (i = 0; i != queues; ++i)
+ rss_conf->queue[i] = i;
/*
* Set up VLAN item if filtering is enabled and at least one VLAN
* filter is configured.
assert(flow->ibv_attr->type == IBV_FLOW_ATTR_NORMAL);
assert(flow->ibv_attr->num_of_specs == 1);
assert(eth->type == IBV_FLOW_SPEC_ETH);
+ assert(flow->rss);
if (rule_vlan &&
(eth->val.vlan_tag != *rule_vlan ||
eth->mask.vlan_tag != RTE_BE16(0x0fff)))
eth->val.src_mac[j] != UINT8_C(0x00) ||
eth->mask.src_mac[j] != UINT8_C(0x00))
break;
- if (j == sizeof(mac->addr_bytes))
- break;
+ if (j != sizeof(mac->addr_bytes))
+ continue;
+ if (flow->rss->queues != queues ||
+ memcmp(flow->rss->queue_id, rss_conf->queue,
+ queues * sizeof(flow->rss->queue_id[0])))
+ continue;
+ break;
}
if (!flow || !flow->internal) {
/* Not found, create a new flow rule. */
break;
}
}
+ if (flow && flow->internal) {
+ assert(flow->rss);
+ if (flow->rss->queues != queues ||
+ memcmp(flow->rss->queue_id, rss_conf->queue,
+ queues * sizeof(flow->rss->queue_id[0])))
+ flow = NULL;
+ }
if (!flow || !flow->internal) {
/* Not found, create a new flow rule. */
if (priv->dev->data->promiscuous) {
return ret;
}
/* Toggle the remaining flow rules . */
- for (flow = LIST_FIRST(&priv->flows);
- flow;
- flow = LIST_NEXT(flow, next)) {
+ LIST_FOREACH(flow, &priv->flows, next) {
ret = mlx4_flow_toggle(priv, flow, priv->started, error);
if (ret)
return ret;
while ((flow = LIST_FIRST(&priv->flows)))
mlx4_flow_destroy(priv->dev, flow, NULL);
+ assert(LIST_EMPTY(&priv->rss));
}
static const struct rte_flow_ops mlx4_flow_ops = {