/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2016 6WIND S.A.
- * Copyright 2016 Mellanox.
+ * Copyright 2016 Mellanox Technologies, Ltd
*/
#include <sys/queue.h>
+#include <stdint.h>
#include <string.h>
/* Verbs header. */
#pragma GCC diagnostic error "-Wpedantic"
#endif
+#include <rte_common.h>
+#include <rte_ether.h>
+#include <rte_eth_ctrl.h>
#include <rte_ethdev_driver.h>
#include <rte_flow.h>
#include <rte_flow_driver.h>
#include "mlx5_prm.h"
#include "mlx5_glue.h"
-/* Define minimal priority for control plane flows. */
-#define MLX5_CTRL_FLOW_PRIORITY 4
+/* Flow priority for control plane flows. */
+#define MLX5_CTRL_FLOW_PRIORITY 1
/* Internet Protocol versions. */
#define MLX5_IPV4 4
#define MLX5_IPV6 6
+#define MLX5_GRE 47
#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
struct ibv_flow_spec_counter_action {
/** Structure give to the conversion functions. */
struct mlx5_flow_data {
+ struct rte_eth_dev *dev; /** Ethernet device. */
struct mlx5_flow_parse *parser; /** Parser context. */
struct rte_flow_error *error; /** Error context. */
};
const void *default_mask,
struct mlx5_flow_data *data);
+static int
+mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data);
+
+static int
+mlx5_flow_create_gre(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data);
+
+static int
+mlx5_flow_create_mpls(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data);
+
struct mlx5_flow_parse;
static void
HASH_RXQ_UDPV6,
HASH_RXQ_IPV6,
HASH_RXQ_ETH,
+ HASH_RXQ_TUNNEL,
};
/* Initialization data for hash RX queue. */
IBV_RX_HASH_SRC_PORT_TCP |
IBV_RX_HASH_DST_PORT_TCP),
.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP,
- .flow_priority = 1,
+ .flow_priority = 0,
.ip_version = MLX5_IPV4,
},
[HASH_RXQ_UDPV4] = {
IBV_RX_HASH_SRC_PORT_UDP |
IBV_RX_HASH_DST_PORT_UDP),
.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP,
- .flow_priority = 1,
+ .flow_priority = 0,
.ip_version = MLX5_IPV4,
},
[HASH_RXQ_IPV4] = {
IBV_RX_HASH_DST_IPV4),
.dpdk_rss_hf = (ETH_RSS_IPV4 |
ETH_RSS_FRAG_IPV4),
- .flow_priority = 2,
+ .flow_priority = 1,
.ip_version = MLX5_IPV4,
},
[HASH_RXQ_TCPV6] = {
IBV_RX_HASH_SRC_PORT_TCP |
IBV_RX_HASH_DST_PORT_TCP),
.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP,
- .flow_priority = 1,
+ .flow_priority = 0,
.ip_version = MLX5_IPV6,
},
[HASH_RXQ_UDPV6] = {
IBV_RX_HASH_SRC_PORT_UDP |
IBV_RX_HASH_DST_PORT_UDP),
.dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP,
- .flow_priority = 1,
+ .flow_priority = 0,
.ip_version = MLX5_IPV6,
},
[HASH_RXQ_IPV6] = {
IBV_RX_HASH_DST_IPV6),
.dpdk_rss_hf = (ETH_RSS_IPV6 |
ETH_RSS_FRAG_IPV6),
- .flow_priority = 2,
+ .flow_priority = 1,
.ip_version = MLX5_IPV6,
},
[HASH_RXQ_ETH] = {
.hash_fields = 0,
.dpdk_rss_hf = 0,
- .flow_priority = 3,
+ .flow_priority = 2,
},
};
TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
uint32_t mark:1; /**< Set if the flow is marked. */
uint32_t drop:1; /**< Drop queue. */
- uint16_t queues_n; /**< Number of entries in queue[]. */
+ struct rte_flow_action_rss rss_conf; /**< RSS configuration */
uint16_t (*queues)[]; /**< Queues indexes to use. */
- struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
uint8_t rss_key[40]; /**< copy of the RSS key. */
+ uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
__VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
}
+#define IS_TUNNEL(type) ( \
+ (type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
+ (type) == RTE_FLOW_ITEM_TYPE_VXLAN_GPE || \
+ (type) == RTE_FLOW_ITEM_TYPE_GRE || \
+ (type) == RTE_FLOW_ITEM_TYPE_MPLS)
+
+const uint32_t flow_ptype[] = {
+ [RTE_FLOW_ITEM_TYPE_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
+ [RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = RTE_PTYPE_TUNNEL_VXLAN_GPE,
+ [RTE_FLOW_ITEM_TYPE_GRE] = RTE_PTYPE_TUNNEL_GRE,
+ [RTE_FLOW_ITEM_TYPE_MPLS] = RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
+};
+
+#define PTYPE_IDX(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
+
+const uint32_t ptype_ext[] = {
+ [PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] = RTE_PTYPE_TUNNEL_VXLAN |
+ RTE_PTYPE_L4_UDP,
+ [PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)] = RTE_PTYPE_TUNNEL_VXLAN_GPE |
+ RTE_PTYPE_L4_UDP,
+ [PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)] = RTE_PTYPE_TUNNEL_GRE,
+ [PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)] =
+ RTE_PTYPE_TUNNEL_MPLS_IN_GRE,
+ [PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)] =
+ RTE_PTYPE_TUNNEL_MPLS_IN_GRE | RTE_PTYPE_L4_UDP,
+};
+
/** Structure to generate a simple graph of layers supported by the NIC. */
struct mlx5_flow_items {
/** List of possible actions for these items. */
static const struct mlx5_flow_items mlx5_flow_items[] = {
[RTE_FLOW_ITEM_TYPE_END] = {
.items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
- RTE_FLOW_ITEM_TYPE_VXLAN),
+ RTE_FLOW_ITEM_TYPE_VXLAN,
+ RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+ RTE_FLOW_ITEM_TYPE_GRE),
},
[RTE_FLOW_ITEM_TYPE_ETH] = {
.items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
.actions = valid_actions,
.mask = &(const struct rte_flow_item_vlan){
.tci = -1,
+ .inner_type = -1,
},
.default_mask = &rte_flow_item_vlan_mask,
.mask_sz = sizeof(struct rte_flow_item_vlan),
},
[RTE_FLOW_ITEM_TYPE_IPV4] = {
.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
- RTE_FLOW_ITEM_TYPE_TCP),
+ RTE_FLOW_ITEM_TYPE_TCP,
+ RTE_FLOW_ITEM_TYPE_GRE),
.actions = valid_actions,
.mask = &(const struct rte_flow_item_ipv4){
.hdr = {
},
[RTE_FLOW_ITEM_TYPE_IPV6] = {
.items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
- RTE_FLOW_ITEM_TYPE_TCP),
+ RTE_FLOW_ITEM_TYPE_TCP,
+ RTE_FLOW_ITEM_TYPE_GRE),
.actions = valid_actions,
.mask = &(const struct rte_flow_item_ipv6){
.hdr = {
.dst_sz = sizeof(struct ibv_flow_spec_ipv6),
},
[RTE_FLOW_ITEM_TYPE_UDP] = {
- .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
+ .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN,
+ RTE_FLOW_ITEM_TYPE_VXLAN_GPE,
+ RTE_FLOW_ITEM_TYPE_MPLS),
.actions = valid_actions,
.mask = &(const struct rte_flow_item_udp){
.hdr = {
.convert = mlx5_flow_create_tcp,
.dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
},
+ [RTE_FLOW_ITEM_TYPE_GRE] = {
+ .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
+ RTE_FLOW_ITEM_TYPE_IPV4,
+ RTE_FLOW_ITEM_TYPE_IPV6,
+ RTE_FLOW_ITEM_TYPE_MPLS),
+ .actions = valid_actions,
+ .mask = &(const struct rte_flow_item_gre){
+ .protocol = -1,
+ },
+ .default_mask = &rte_flow_item_gre_mask,
+ .mask_sz = sizeof(struct rte_flow_item_gre),
+ .convert = mlx5_flow_create_gre,
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+ .dst_sz = sizeof(struct ibv_flow_spec_gre),
+#else
+ .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
+#endif
+ },
+ [RTE_FLOW_ITEM_TYPE_MPLS] = {
+ .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
+ RTE_FLOW_ITEM_TYPE_IPV4,
+ RTE_FLOW_ITEM_TYPE_IPV6),
+ .actions = valid_actions,
+ .mask = &(const struct rte_flow_item_mpls){
+ .label_tc_s = "\xff\xff\xf0",
+ },
+ .default_mask = &rte_flow_item_mpls_mask,
+ .mask_sz = sizeof(struct rte_flow_item_mpls),
+ .convert = mlx5_flow_create_mpls,
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+ .dst_sz = sizeof(struct ibv_flow_spec_mpls),
+#endif
+ },
[RTE_FLOW_ITEM_TYPE_VXLAN] = {
- .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
+ .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
+ RTE_FLOW_ITEM_TYPE_IPV4, /* For L3 VXLAN. */
+ RTE_FLOW_ITEM_TYPE_IPV6), /* For L3 VXLAN. */
.actions = valid_actions,
.mask = &(const struct rte_flow_item_vxlan){
.vni = "\xff\xff\xff",
.convert = mlx5_flow_create_vxlan,
.dst_sz = sizeof(struct ibv_flow_spec_tunnel),
},
+ [RTE_FLOW_ITEM_TYPE_VXLAN_GPE] = {
+ .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
+ RTE_FLOW_ITEM_TYPE_IPV4,
+ RTE_FLOW_ITEM_TYPE_IPV6),
+ .actions = valid_actions,
+ .mask = &(const struct rte_flow_item_vxlan_gpe){
+ .vni = "\xff\xff\xff",
+ },
+ .default_mask = &rte_flow_item_vxlan_gpe_mask,
+ .mask_sz = sizeof(struct rte_flow_item_vxlan_gpe),
+ .convert = mlx5_flow_create_vxlan_gpe,
+ .dst_sz = sizeof(struct ibv_flow_spec_tunnel),
+ },
};
/** Structure to pass to the conversion function. */
struct mlx5_flow_parse {
- uint32_t inner; /**< Set once VXLAN is encountered. */
+ uint32_t inner; /**< Verbs value, set once tunnel is encountered. */
uint32_t create:1;
/**< Whether resources should remain after a validate. */
uint32_t drop:1; /**< Target is a drop queue. */
uint32_t mark:1; /**< Mark is present in the flow. */
uint32_t count:1; /**< Count is present in the flow. */
uint32_t mark_id; /**< Mark identifier. */
+ struct rte_flow_action_rss rss_conf; /**< RSS configuration */
uint16_t queues[RTE_MAX_QUEUES_PER_PORT]; /**< Queues indexes to use. */
- uint16_t queues_n; /**< Number of entries in queue[]. */
- struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
uint8_t rss_key[40]; /**< copy of the RSS key. */
enum hash_rxq_type layer; /**< Last pattern layer detected. */
+ enum hash_rxq_type out_layer; /**< Last outer pattern layer detected. */
+ uint32_t tunnel; /**< Tunnel type of RTE_PTYPE_TUNNEL_XXX. */
struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
struct {
struct ibv_flow_attr *ibv_attr;
/**< Pointer to Verbs attributes. */
unsigned int offset;
/**< Current position or total size of the attribute. */
+ uint64_t hash_fields; /**< Verbs hash fields. */
} queue[RTE_DIM(hash_rxq_init)];
};
struct rte_flow_item_ipv4 ipv4;
struct rte_flow_item_ipv6 ipv6;
} l3;
+ union {
+ struct rte_flow_item_ipv4 ipv4;
+ struct rte_flow_item_ipv6 ipv6;
+ } l3_mask;
union {
struct rte_flow_item_udp udp;
struct rte_flow_item_tcp tcp;
} l4;
+ union {
+ struct rte_flow_item_udp udp;
+ struct rte_flow_item_tcp tcp;
+ } l4_mask;
struct rte_flow_action_queue queue;
};
};
/**
- * Check support for a given item.
+ * Check item is fully supported by the NIC matching capability.
*
* @param item[in]
* Item specification.
mlx5_flow_item_validate(const struct rte_flow_item *item,
const uint8_t *mask, unsigned int size)
{
- if (!item->spec && (item->mask || item->last)) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- if (item->spec && !item->mask) {
- unsigned int i;
- const uint8_t *spec = item->spec;
-
- for (i = 0; i < size; ++i)
- if ((spec[i] | mask[i]) != mask[i]) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- if (item->last && !item->mask) {
- unsigned int i;
- const uint8_t *spec = item->last;
-
- for (i = 0; i < size; ++i)
- if ((spec[i] | mask[i]) != mask[i]) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- if (item->mask) {
- unsigned int i;
- const uint8_t *spec = item->spec;
-
- for (i = 0; i < size; ++i)
- if ((spec[i] | mask[i]) != mask[i]) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- if (item->spec && item->last) {
- uint8_t spec[size];
- uint8_t last[size];
- const uint8_t *apply = mask;
- unsigned int i;
- int ret;
-
- if (item->mask)
- apply = item->mask;
- for (i = 0; i < size; ++i) {
- spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
- last[i] = ((const uint8_t *)item->last)[i] & apply[i];
- }
- ret = memcmp(spec, last, size);
- if (ret != 0) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- }
- return 0;
-}
+ unsigned int i;
+ const uint8_t *spec = item->spec;
+ const uint8_t *last = item->last;
+ const uint8_t *m = item->mask ? item->mask : mask;
-/**
- * Copy the RSS configuration from the user ones, of the rss_conf is null,
- * uses the driver one.
- *
- * @param parser
- * Internal parser structure.
- * @param rss_conf
- * User RSS configuration to save.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser,
- const struct rte_eth_rss_conf *rss_conf)
-{
+ if (!spec && (item->mask || last))
+ goto error;
+ if (!spec)
+ return 0;
/*
- * This function is also called at the beginning of
- * mlx5_flow_convert_actions() to initialize the parser with the
- * device default RSS configuration.
+ * Single-pass check to make sure that:
+ * - item->mask is supported, no bits are set outside mask.
+ * - Both masked item->spec and item->last are equal (no range
+ * supported).
*/
- if (rss_conf) {
- if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- if (rss_conf->rss_key_len != 40) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- if (rss_conf->rss_key_len && rss_conf->rss_key) {
- parser->rss_conf.rss_key_len = rss_conf->rss_key_len;
- memcpy(parser->rss_key, rss_conf->rss_key,
- rss_conf->rss_key_len);
- parser->rss_conf.rss_key = parser->rss_key;
- }
- parser->rss_conf.rss_hf = rss_conf->rss_hf;
+ for (i = 0; i < size; i++) {
+ if (!m[i])
+ continue;
+ if ((m[i] | mask[i]) != mask[i])
+ goto error;
+ if (last && ((spec[i] & m[i]) != (last[i] & m[i])))
+ goto error;
}
return 0;
+error:
+ rte_errno = ENOTSUP;
+ return -rte_errno;
}
/**
"egress is not supported");
return -rte_errno;
}
+ if (attr->transfer) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
+ NULL,
+ "transfer is not supported");
+ return -rte_errno;
+ }
if (!attr->ingress) {
rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
struct rte_flow_error *error,
struct mlx5_flow_parse *parser)
{
+ enum { FATE = 1, MARK = 2, COUNT = 4, };
+ uint32_t overlap = 0;
struct priv *priv = dev->data->dev_private;
- int ret;
- /*
- * Add default RSS configuration necessary for Verbs to create QP even
- * if no RSS is necessary.
- */
- ret = mlx5_flow_convert_rss_conf(parser,
- (const struct rte_eth_rss_conf *)
- &priv->rss_conf);
- if (ret)
- return ret;
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
continue;
} else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
+ if (overlap & FATE)
+ goto exit_action_overlap;
+ overlap |= FATE;
parser->drop = 1;
} else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
const struct rte_flow_action_queue *queue =
(const struct rte_flow_action_queue *)
actions->conf;
- uint16_t n;
- uint16_t found = 0;
+ if (overlap & FATE)
+ goto exit_action_overlap;
+ overlap |= FATE;
if (!queue || (queue->index > (priv->rxqs_n - 1)))
goto exit_action_not_supported;
- for (n = 0; n < parser->queues_n; ++n) {
- if (parser->queues[n] == queue->index) {
- found = 1;
- break;
- }
- }
- if (parser->queues_n > 1 && !found) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "queue action not in RSS queues");
- return -rte_errno;
- }
- if (!found) {
- parser->queues_n = 1;
- parser->queues[0] = queue->index;
- }
+ parser->queues[0] = queue->index;
+ parser->rss_conf = (struct rte_flow_action_rss){
+ .queue_num = 1,
+ .queue = parser->queues,
+ };
} else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
const struct rte_flow_action_rss *rss =
(const struct rte_flow_action_rss *)
actions->conf;
+ const uint8_t *rss_key;
+ uint32_t rss_key_len;
uint16_t n;
- if (!rss || !rss->num) {
+ if (overlap & FATE)
+ goto exit_action_overlap;
+ overlap |= FATE;
+ if (rss->func &&
+ rss->func != RTE_ETH_HASH_FUNCTION_TOEPLITZ) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "the only supported RSS hash"
+ " function is Toeplitz");
+ return -rte_errno;
+ }
+#ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
+ if (parser->rss_conf.level > 1) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "a nonzero RSS encapsulation"
+ " level is not supported");
+ return -rte_errno;
+ }
+#endif
+ if (parser->rss_conf.level > 2) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "RSS encapsulation level"
+ " > 1 is not supported");
+ return -rte_errno;
+ }
+ if (rss->types & MLX5_RSS_HF_MASK) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "unsupported RSS type"
+ " requested");
+ return -rte_errno;
+ }
+ if (rss->key_len) {
+ rss_key_len = rss->key_len;
+ rss_key = rss->key;
+ } else {
+ rss_key_len = rss_hash_default_key_len;
+ rss_key = rss_hash_default_key;
+ }
+ if (rss_key_len != RTE_DIM(parser->rss_key)) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "RSS hash key must be"
+ " exactly 40 bytes long");
+ return -rte_errno;
+ }
+ if (!rss->queue_num) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
actions,
"no valid queues");
return -rte_errno;
}
- if (parser->queues_n == 1) {
- uint16_t found = 0;
-
- assert(parser->queues_n);
- for (n = 0; n < rss->num; ++n) {
- if (parser->queues[0] ==
- rss->queue[n]) {
- found = 1;
- break;
- }
- }
- if (!found) {
- rte_flow_error_set(error, ENOTSUP,
+ if (rss->queue_num > RTE_DIM(parser->queues)) {
+ rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
actions,
- "queue action not in RSS"
- " queues");
- return -rte_errno;
- }
+ "too many queues for RSS"
+ " context");
+ return -rte_errno;
}
- for (n = 0; n < rss->num; ++n) {
+ for (n = 0; n < rss->queue_num; ++n) {
if (rss->queue[n] >= priv->rxqs_n) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
return -rte_errno;
}
}
- for (n = 0; n < rss->num; ++n)
- parser->queues[n] = rss->queue[n];
- parser->queues_n = rss->num;
- if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) {
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ACTION,
- actions,
- "wrong RSS configuration");
- return -rte_errno;
- }
+ parser->rss_conf = (struct rte_flow_action_rss){
+ .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+ .level = rss->level,
+ .types = rss->types,
+ .key_len = rss_key_len,
+ .queue_num = rss->queue_num,
+ .key = memcpy(parser->rss_key, rss_key,
+ sizeof(*rss_key) * rss_key_len),
+ .queue = memcpy(parser->queues, rss->queue,
+ sizeof(*rss->queue) *
+ rss->queue_num),
+ };
} else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
const struct rte_flow_action_mark *mark =
(const struct rte_flow_action_mark *)
actions->conf;
+ if (overlap & MARK)
+ goto exit_action_overlap;
+ overlap |= MARK;
if (!mark) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
parser->mark = 1;
parser->mark_id = mark->id;
} else if (actions->type == RTE_FLOW_ACTION_TYPE_FLAG) {
+ if (overlap & MARK)
+ goto exit_action_overlap;
+ overlap |= MARK;
parser->mark = 1;
} else if (actions->type == RTE_FLOW_ACTION_TYPE_COUNT &&
priv->config.flow_counter_en) {
+ if (overlap & COUNT)
+ goto exit_action_overlap;
+ overlap |= COUNT;
parser->count = 1;
} else {
goto exit_action_not_supported;
}
}
+ /* When fate is unknown, drop traffic. */
+ if (!(overlap & FATE))
+ parser->drop = 1;
if (parser->drop && parser->mark)
parser->mark = 0;
- if (!parser->queues_n && !parser->drop) {
+ if (!parser->rss_conf.queue_num && !parser->drop) {
rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
NULL, "no valid action");
return -rte_errno;
rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
actions, "action not supported");
return -rte_errno;
+exit_action_overlap:
+ rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
+ actions, "overlapping actions are not supported");
+ return -rte_errno;
}
/**
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-mlx5_flow_convert_items_validate(const struct rte_flow_item items[],
+mlx5_flow_convert_items_validate(struct rte_eth_dev *dev,
+ const struct rte_flow_item items[],
struct rte_flow_error *error,
struct mlx5_flow_parse *parser)
{
+ struct priv *priv = dev->data->dev_private;
const struct mlx5_flow_items *cur_item = mlx5_flow_items;
unsigned int i;
+ unsigned int last_voids = 0;
int ret = 0;
/* Initialise the offsets to start after verbs attribute. */
const struct mlx5_flow_items *token = NULL;
unsigned int n;
- if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
+ if (items->type == RTE_FLOW_ITEM_TYPE_VOID) {
+ last_voids++;
continue;
+ }
for (i = 0;
cur_item->items &&
cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
break;
}
}
- if (!token)
+ if (!token) {
+ ret = -ENOTSUP;
goto exit_item_not_supported;
+ }
cur_item = token;
ret = mlx5_flow_item_validate(items,
(const uint8_t *)cur_item->mask,
cur_item->mask_sz);
if (ret)
goto exit_item_not_supported;
- if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) {
- if (parser->inner) {
+ if (IS_TUNNEL(items->type)) {
+ if (parser->tunnel &&
+ !((items - last_voids - 1)->type ==
+ RTE_FLOW_ITEM_TYPE_GRE && items->type ==
+ RTE_FLOW_ITEM_TYPE_MPLS)) {
rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ITEM,
items,
- "cannot recognize multiple"
- " VXLAN encapsulations");
+ "Cannot recognize multiple"
+ " tunnel encapsulations.");
+ return -rte_errno;
+ }
+ if (items->type == RTE_FLOW_ITEM_TYPE_MPLS &&
+ !priv->config.mpls_en) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ items,
+ "MPLS not supported or"
+ " disabled in firmware"
+ " configuration.");
+ return -rte_errno;
+ }
+ if (!priv->config.tunnel_en &&
+ parser->rss_conf.level > 1) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ items,
+ "RSS on tunnel is not supported");
return -rte_errno;
}
parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = flow_ptype[items->type];
}
if (parser->drop) {
parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
for (n = 0; n != hash_rxq_init_n; ++n)
parser->queue[n].offset += cur_item->dst_sz;
}
+ last_voids = 0;
}
if (parser->drop) {
parser->queue[HASH_RXQ_ETH].offset +=
* Make inner packet matching with an higher priority from the non Inner
* matching.
*
+ * @param dev
+ * Pointer to Ethernet device.
* @param[in, out] parser
* Internal parser structure.
* @param attr
* User flow attribute.
*/
static void
-mlx5_flow_update_priority(struct mlx5_flow_parse *parser,
+mlx5_flow_update_priority(struct rte_eth_dev *dev,
+ struct mlx5_flow_parse *parser,
const struct rte_flow_attr *attr)
{
+ struct priv *priv = dev->data->dev_private;
unsigned int i;
+ uint16_t priority;
+ /* 8 priorities >= 16 priorities
+ * Control flow: 4-7 8-15
+ * User normal flow: 1-3 4-7
+ * User tunnel flow: 0-2 0-3
+ */
+ priority = attr->priority * MLX5_VERBS_FLOW_PRIO_8;
+ if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
+ priority /= 2;
+ /*
+ * Lower non-tunnel flow Verbs priority 1 if only support 8 Verbs
+ * priorities, lower 4 otherwise.
+ */
+ if (!parser->inner) {
+ if (priv->config.max_verbs_prio == MLX5_VERBS_FLOW_PRIO_8)
+ priority += 1;
+ else
+ priority += MLX5_VERBS_FLOW_PRIO_8 / 2;
+ }
if (parser->drop) {
- parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
- attr->priority +
- hash_rxq_init[HASH_RXQ_ETH].flow_priority;
+ parser->queue[HASH_RXQ_ETH].ibv_attr->priority = priority +
+ hash_rxq_init[HASH_RXQ_ETH].flow_priority;
return;
}
for (i = 0; i != hash_rxq_init_n; ++i) {
- if (parser->queue[i].ibv_attr) {
- parser->queue[i].ibv_attr->priority =
- attr->priority +
- hash_rxq_init[i].flow_priority -
- (parser->inner ? 1 : 0);
- }
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ parser->queue[i].ibv_attr->priority = priority +
+ hash_rxq_init[i].flow_priority;
}
}
static void
mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
{
- const unsigned int ipv4 =
- hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
- const enum hash_rxq_type hmin = ipv4 ? HASH_RXQ_TCPV4 : HASH_RXQ_TCPV6;
- const enum hash_rxq_type hmax = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
- const enum hash_rxq_type ohmin = ipv4 ? HASH_RXQ_TCPV6 : HASH_RXQ_TCPV4;
- const enum hash_rxq_type ohmax = ipv4 ? HASH_RXQ_IPV6 : HASH_RXQ_IPV4;
- const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6;
unsigned int i;
+ uint32_t inner = parser->inner;
- /* Remove any other flow not matching the pattern. */
- if (parser->queues_n == 1 && !parser->rss_conf.rss_hf) {
- for (i = 0; i != hash_rxq_init_n; ++i) {
- if (i == HASH_RXQ_ETH)
- continue;
- rte_free(parser->queue[i].ibv_attr);
- parser->queue[i].ibv_attr = NULL;
- }
+ /* Don't create extra flows for outer RSS. */
+ if (parser->tunnel && parser->rss_conf.level < 2)
return;
- }
- if (parser->layer == HASH_RXQ_ETH) {
- goto fill;
- } else {
- /*
- * This layer becomes useless as the pattern define under
- * layers.
- */
- rte_free(parser->queue[HASH_RXQ_ETH].ibv_attr);
- parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
- }
- /* Remove opposite kind of layer e.g. IPv6 if the pattern is IPv4. */
- for (i = ohmin; i != (ohmax + 1); ++i) {
- if (!parser->queue[i].ibv_attr)
- continue;
- rte_free(parser->queue[i].ibv_attr);
- parser->queue[i].ibv_attr = NULL;
- }
- /* Remove impossible flow according to the RSS configuration. */
- if (hash_rxq_init[parser->layer].dpdk_rss_hf &
- parser->rss_conf.rss_hf) {
- /* Remove any other flow. */
- for (i = hmin; i != (hmax + 1); ++i) {
- if ((i == parser->layer) ||
- (!parser->queue[i].ibv_attr))
- continue;
- rte_free(parser->queue[i].ibv_attr);
- parser->queue[i].ibv_attr = NULL;
- }
- } else if (!parser->queue[ip].ibv_attr) {
- /* no RSS possible with the current configuration. */
- parser->queues_n = 1;
- return;
- }
-fill:
/*
* Fill missing layers in verbs specifications, or compute the correct
* offset to allocate the memory space for the attributes and
struct ibv_flow_spec_ipv4_ext ipv4;
struct ibv_flow_spec_ipv6 ipv6;
struct ibv_flow_spec_tcp_udp udp_tcp;
+ struct ibv_flow_spec_eth eth;
} specs;
void *dst;
uint16_t size;
if (i == parser->layer)
continue;
- if (parser->layer == HASH_RXQ_ETH) {
+ if (parser->layer == HASH_RXQ_ETH ||
+ parser->layer == HASH_RXQ_TUNNEL) {
if (hash_rxq_init[i].ip_version == MLX5_IPV4) {
size = sizeof(struct ibv_flow_spec_ipv4_ext);
specs.ipv4 = (struct ibv_flow_spec_ipv4_ext){
- .type = IBV_FLOW_SPEC_IPV4_EXT,
+ .type = inner | IBV_FLOW_SPEC_IPV4_EXT,
.size = size,
};
} else {
size = sizeof(struct ibv_flow_spec_ipv6);
specs.ipv6 = (struct ibv_flow_spec_ipv6){
- .type = IBV_FLOW_SPEC_IPV6,
+ .type = inner | IBV_FLOW_SPEC_IPV6,
.size = size,
};
}
(i == HASH_RXQ_UDPV6) || (i == HASH_RXQ_TCPV6)) {
size = sizeof(struct ibv_flow_spec_tcp_udp);
specs.udp_tcp = (struct ibv_flow_spec_tcp_udp) {
- .type = ((i == HASH_RXQ_UDPV4 ||
+ .type = inner | ((i == HASH_RXQ_UDPV4 ||
i == HASH_RXQ_UDPV6) ?
IBV_FLOW_SPEC_UDP :
IBV_FLOW_SPEC_TCP),
}
}
+/**
+ * Update flows according to pattern and RSS hash fields.
+ *
+ * @param[in, out] parser
+ * Internal parser structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_convert_rss(struct mlx5_flow_parse *parser)
+{
+ unsigned int i;
+ enum hash_rxq_type start;
+ enum hash_rxq_type layer;
+ int outer = parser->tunnel && parser->rss_conf.level < 2;
+ uint64_t rss = parser->rss_conf.types;
+
+ /* Default to outer RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 1;
+ layer = outer ? parser->out_layer : parser->layer;
+ if (layer == HASH_RXQ_TUNNEL)
+ layer = HASH_RXQ_ETH;
+ if (outer) {
+ /* Only one hash type for outer RSS. */
+ if (rss && layer == HASH_RXQ_ETH) {
+ start = HASH_RXQ_TCPV4;
+ } else if (rss && layer != HASH_RXQ_ETH &&
+ !(rss & hash_rxq_init[layer].dpdk_rss_hf)) {
+ /* If RSS not match L4 pattern, try L3 RSS. */
+ if (layer < HASH_RXQ_IPV4)
+ layer = HASH_RXQ_IPV4;
+ else if (layer > HASH_RXQ_IPV4 && layer < HASH_RXQ_IPV6)
+ layer = HASH_RXQ_IPV6;
+ start = layer;
+ } else {
+ start = layer;
+ }
+ /* Scan first valid hash type. */
+ for (i = start; rss && i <= layer; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ if (hash_rxq_init[i].dpdk_rss_hf & rss)
+ break;
+ }
+ if (rss && i <= layer)
+ parser->queue[layer].hash_fields =
+ hash_rxq_init[i].hash_fields;
+ /* Trim unused hash types. */
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (parser->queue[i].ibv_attr && i != layer) {
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ }
+ }
+ } else {
+ /* Expand for inner or normal RSS. */
+ if (rss && (layer == HASH_RXQ_ETH || layer == HASH_RXQ_IPV4))
+ start = HASH_RXQ_TCPV4;
+ else if (rss && layer == HASH_RXQ_IPV6)
+ start = HASH_RXQ_TCPV6;
+ else
+ start = layer;
+ /* For L4 pattern, try L3 RSS if no L4 RSS. */
+ /* Trim unused hash types. */
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ if (i < start || i > layer) {
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ continue;
+ }
+ if (!rss)
+ continue;
+ if (hash_rxq_init[i].dpdk_rss_hf & rss) {
+ parser->queue[i].hash_fields =
+ hash_rxq_init[i].hash_fields;
+ } else if (i != layer) {
+ /* Remove unused RSS expansion. */
+ rte_free(parser->queue[i].ibv_attr);
+ parser->queue[i].ibv_attr = NULL;
+ } else if (layer < HASH_RXQ_IPV4 &&
+ (hash_rxq_init[HASH_RXQ_IPV4].dpdk_rss_hf &
+ rss)) {
+ /* Allow IPv4 RSS on L4 pattern. */
+ parser->queue[i].hash_fields =
+ hash_rxq_init[HASH_RXQ_IPV4]
+ .hash_fields;
+ } else if (i > HASH_RXQ_IPV4 && i < HASH_RXQ_IPV6 &&
+ (hash_rxq_init[HASH_RXQ_IPV6].dpdk_rss_hf &
+ rss)) {
+ /* Allow IPv4 RSS on L4 pattern. */
+ parser->queue[i].hash_fields =
+ hash_rxq_init[HASH_RXQ_IPV6]
+ .hash_fields;
+ }
+ }
+ }
+ return 0;
+}
+
/**
* Validate and convert a flow supported by the NIC.
*
ret = mlx5_flow_convert_actions(dev, actions, error, parser);
if (ret)
return ret;
- ret = mlx5_flow_convert_items_validate(items, error, parser);
+ ret = mlx5_flow_convert_items_validate(dev, items, error, parser);
if (ret)
return ret;
mlx5_flow_convert_finalise(parser);
for (i = 0; i != hash_rxq_init_n; ++i) {
unsigned int offset;
- if (!(parser->rss_conf.rss_hf &
- hash_rxq_init[i].dpdk_rss_hf) &&
- (i != HASH_RXQ_ETH))
- continue;
offset = parser->queue[i].offset;
parser->queue[i].ibv_attr =
mlx5_flow_convert_allocate(offset, error);
}
/* Third step. Conversion parse, fill the specifications. */
parser->inner = 0;
+ parser->tunnel = 0;
+ parser->layer = HASH_RXQ_ETH;
for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
struct mlx5_flow_data data = {
+ .dev = dev,
.parser = parser,
.error = error,
};
if (ret)
goto exit_free;
}
+ if (!parser->drop) {
+ /* RSS check, remove unused hash types. */
+ ret = mlx5_flow_convert_rss(parser);
+ if (ret)
+ goto exit_free;
+ /* Complete missing specification. */
+ mlx5_flow_convert_finalise(parser);
+ }
+ mlx5_flow_update_priority(dev, parser, attr);
if (parser->mark)
mlx5_flow_create_flag_mark(parser, parser->mark_id);
if (parser->count && parser->create) {
if (!parser->cs)
goto exit_count_error;
}
- /*
- * Last step. Complete missing specification to reach the RSS
- * configuration.
- */
- if (!parser->drop)
- mlx5_flow_convert_finalise(parser);
- mlx5_flow_update_priority(parser, attr);
exit_free:
/* Only verification is expected, all resources should be released. */
if (!parser->create) {
for (i = 0; i != hash_rxq_init_n; ++i) {
if (!parser->queue[i].ibv_attr)
continue;
- /* Specification must be the same l3 type or none. */
- if (parser->layer == HASH_RXQ_ETH ||
- (hash_rxq_init[parser->layer].ip_version ==
- hash_rxq_init[i].ip_version) ||
- (hash_rxq_init[i].ip_version == 0)) {
- dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
- parser->queue[i].offset);
- memcpy(dst, src, size);
- ++parser->queue[i].ibv_attr->num_of_specs;
- parser->queue[i].offset += size;
- }
+ dst = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset);
+ memcpy(dst, src, size);
+ ++parser->queue[i].ibv_attr->num_of_specs;
+ parser->queue[i].offset += size;
}
}
.size = eth_size,
};
- /* Don't update layer for the inner pattern. */
- if (!parser->inner)
- parser->layer = HASH_RXQ_ETH;
+ parser->layer = HASH_RXQ_ETH;
if (spec) {
unsigned int i;
struct mlx5_flow_parse *parser = data->parser;
struct ibv_flow_spec_eth *eth;
const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
+ const char *msg = "VLAN cannot be empty";
if (spec) {
unsigned int i;
*/
if (!eth->mask.vlan_tag)
goto error;
+ /* Outer TPID cannot be matched. */
+ if (eth->mask.ether_type) {
+ msg = "VLAN TPID matching is not supported";
+ goto error;
+ }
+ eth->val.ether_type = spec->inner_type;
+ eth->mask.ether_type = mask->inner_type;
+ eth->val.ether_type &= eth->mask.ether_type;
}
return 0;
}
error:
return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM,
- item, "VLAN cannot be empty");
+ item, msg);
}
/**
const void *default_mask,
struct mlx5_flow_data *data)
{
+ struct priv *priv = data->dev->data->dev_private;
const struct rte_flow_item_ipv4 *spec = item->spec;
const struct rte_flow_item_ipv4 *mask = item->mask;
struct mlx5_flow_parse *parser = data->parser;
.size = ipv4_size,
};
- /* Don't update layer for the inner pattern. */
- if (!parser->inner)
- parser->layer = HASH_RXQ_IPV4;
+ if (parser->layer == HASH_RXQ_TUNNEL &&
+ parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
+ !priv->config.l3_vxlan_en)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "L3 VXLAN not enabled by device"
+ " parameter and/or not configured"
+ " in firmware");
+ parser->layer = HASH_RXQ_IPV4;
if (spec) {
if (!mask)
mask = default_mask;
const void *default_mask,
struct mlx5_flow_data *data)
{
+ struct priv *priv = data->dev->data->dev_private;
const struct rte_flow_item_ipv6 *spec = item->spec;
const struct rte_flow_item_ipv6 *mask = item->mask;
struct mlx5_flow_parse *parser = data->parser;
.size = ipv6_size,
};
- /* Don't update layer for the inner pattern. */
- if (!parser->inner)
- parser->layer = HASH_RXQ_IPV6;
+ if (parser->layer == HASH_RXQ_TUNNEL &&
+ parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
+ !priv->config.l3_vxlan_en)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "L3 VXLAN not enabled by device"
+ " parameter and/or not configured"
+ " in firmware");
+ parser->layer = HASH_RXQ_IPV6;
if (spec) {
unsigned int i;
uint32_t vtc_flow_val;
.size = udp_size,
};
- /* Don't update layer for the inner pattern. */
- if (!parser->inner) {
- if (parser->layer == HASH_RXQ_IPV4)
- parser->layer = HASH_RXQ_UDPV4;
- else
- parser->layer = HASH_RXQ_UDPV6;
- }
+ if (parser->layer == HASH_RXQ_IPV4)
+ parser->layer = HASH_RXQ_UDPV4;
+ else
+ parser->layer = HASH_RXQ_UDPV6;
if (spec) {
if (!mask)
mask = default_mask;
}
/**
- * Convert TCP item to Verbs specification.
+ * Convert TCP item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_tcp(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ const struct rte_flow_item_tcp *spec = item->spec;
+ const struct rte_flow_item_tcp *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
+ struct ibv_flow_spec_tcp_udp tcp = {
+ .type = parser->inner | IBV_FLOW_SPEC_TCP,
+ .size = tcp_size,
+ };
+
+ if (parser->layer == HASH_RXQ_IPV4)
+ parser->layer = HASH_RXQ_TCPV4;
+ else
+ parser->layer = HASH_RXQ_TCPV6;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ tcp.val.dst_port = spec->hdr.dst_port;
+ tcp.val.src_port = spec->hdr.src_port;
+ tcp.mask.dst_port = mask->hdr.dst_port;
+ tcp.mask.src_port = mask->hdr.src_port;
+ /* Remove unwanted bits from values. */
+ tcp.val.src_port &= tcp.mask.src_port;
+ tcp.val.dst_port &= tcp.mask.dst_port;
+ }
+ mlx5_flow_create_copy(parser, &tcp, tcp_size);
+ return 0;
+}
+
+/**
+ * Convert VXLAN item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_vxlan(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ const struct rte_flow_item_vxlan *spec = item->spec;
+ const struct rte_flow_item_vxlan *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+ struct ibv_flow_spec_tunnel vxlan = {
+ .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ .size = size,
+ };
+ union vni {
+ uint32_t vlan_id;
+ uint8_t vni[4];
+ } id;
+
+ id.vni[0] = 0;
+ parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
+ parser->out_layer = parser->layer;
+ parser->layer = HASH_RXQ_TUNNEL;
+ /* Default VXLAN to outer RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 1;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ memcpy(&id.vni[1], spec->vni, 3);
+ vxlan.val.tunnel_id = id.vlan_id;
+ memcpy(&id.vni[1], mask->vni, 3);
+ vxlan.mask.tunnel_id = id.vlan_id;
+ /* Remove unwanted bits from values. */
+ vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+ }
+ /*
+ * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+ * layer is defined in the Verbs specification it is interpreted as
+ * wildcard and all packets will match this rule, if it follows a full
+ * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+ * before will also match this rule.
+ * To avoid such situation, VNI 0 is currently refused.
+ */
+ /* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
+ if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "VxLAN vni cannot be 0");
+ mlx5_flow_create_copy(parser, &vxlan, size);
+ return 0;
+}
+
+/**
+ * Convert VXLAN-GPE item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ struct priv *priv = data->dev->data->dev_private;
+ const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+ const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+ struct ibv_flow_spec_tunnel vxlan = {
+ .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ .size = size,
+ };
+ union vni {
+ uint32_t vlan_id;
+ uint8_t vni[4];
+ } id;
+
+ if (!priv->config.l3_vxlan_en)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "L3 VXLAN not enabled by device"
+ " parameter and/or not configured"
+ " in firmware");
+ id.vni[0] = 0;
+ parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
+ parser->out_layer = parser->layer;
+ parser->layer = HASH_RXQ_TUNNEL;
+ /* Default VXLAN-GPE to outer RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 1;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ memcpy(&id.vni[1], spec->vni, 3);
+ vxlan.val.tunnel_id = id.vlan_id;
+ memcpy(&id.vni[1], mask->vni, 3);
+ vxlan.mask.tunnel_id = id.vlan_id;
+ if (spec->protocol)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "VxLAN-GPE protocol not"
+ " supported");
+ /* Remove unwanted bits from values. */
+ vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+ }
+ /*
+ * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+ * layer is defined in the Verbs specification it is interpreted as
+ * wildcard and all packets will match this rule, if it follows a full
+ * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+ * before will also match this rule.
+ * To avoid such situation, VNI 0 is currently refused.
+ */
+ /* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
+ if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "VxLAN-GPE vni cannot be 0");
+ mlx5_flow_create_copy(parser, &vxlan, size);
+ return 0;
+}
+
+/**
+ * Convert GRE item to Verbs specification.
*
* @param item[in]
* Item specification.
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-mlx5_flow_create_tcp(const struct rte_flow_item *item,
+mlx5_flow_create_gre(const struct rte_flow_item *item,
const void *default_mask,
struct mlx5_flow_data *data)
{
- const struct rte_flow_item_tcp *spec = item->spec;
- const struct rte_flow_item_tcp *mask = item->mask;
struct mlx5_flow_parse *parser = data->parser;
- unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
- struct ibv_flow_spec_tcp_udp tcp = {
- .type = parser->inner | IBV_FLOW_SPEC_TCP,
- .size = tcp_size,
+#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
+ (void)default_mask;
+ unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+ struct ibv_flow_spec_tunnel tunnel = {
+ .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ .size = size,
+ };
+#else
+ const struct rte_flow_item_gre *spec = item->spec;
+ const struct rte_flow_item_gre *mask = item->mask;
+ unsigned int size = sizeof(struct ibv_flow_spec_gre);
+ struct ibv_flow_spec_gre tunnel = {
+ .type = parser->inner | IBV_FLOW_SPEC_GRE,
+ .size = size,
};
+#endif
+ struct ibv_flow_spec_ipv4_ext *ipv4;
+ struct ibv_flow_spec_ipv6 *ipv6;
+ unsigned int i;
- /* Don't update layer for the inner pattern. */
- if (!parser->inner) {
- if (parser->layer == HASH_RXQ_IPV4)
- parser->layer = HASH_RXQ_TCPV4;
- else
- parser->layer = HASH_RXQ_TCPV6;
- }
+ parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
+ parser->out_layer = parser->layer;
+ parser->layer = HASH_RXQ_TUNNEL;
+ /* Default GRE to inner RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 2;
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
if (spec) {
if (!mask)
mask = default_mask;
- tcp.val.dst_port = spec->hdr.dst_port;
- tcp.val.src_port = spec->hdr.src_port;
- tcp.mask.dst_port = mask->hdr.dst_port;
- tcp.mask.src_port = mask->hdr.src_port;
+ tunnel.val.c_ks_res0_ver = spec->c_rsvd0_ver;
+ tunnel.val.protocol = spec->protocol;
+ tunnel.mask.c_ks_res0_ver = mask->c_rsvd0_ver;
+ tunnel.mask.protocol = mask->protocol;
/* Remove unwanted bits from values. */
- tcp.val.src_port &= tcp.mask.src_port;
- tcp.val.dst_port &= tcp.mask.dst_port;
+ tunnel.val.c_ks_res0_ver &= tunnel.mask.c_ks_res0_ver;
+ tunnel.val.protocol &= tunnel.mask.protocol;
+ tunnel.val.key &= tunnel.mask.key;
}
- mlx5_flow_create_copy(parser, &tcp, tcp_size);
+#endif
+ /* Update encapsulation IP layer protocol. */
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ if (parser->out_layer == HASH_RXQ_IPV4) {
+ ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset -
+ sizeof(struct ibv_flow_spec_ipv4_ext));
+ if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
+ break;
+ ipv4->val.proto = MLX5_GRE;
+ ipv4->mask.proto = 0xff;
+ } else if (parser->out_layer == HASH_RXQ_IPV6) {
+ ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset -
+ sizeof(struct ibv_flow_spec_ipv6));
+ if (ipv6->mask.next_hdr &&
+ ipv6->val.next_hdr != MLX5_GRE)
+ break;
+ ipv6->val.next_hdr = MLX5_GRE;
+ ipv6->mask.next_hdr = 0xff;
+ }
+ }
+ if (i != hash_rxq_init_n)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "IP protocol of GRE must be 47");
+ mlx5_flow_create_copy(parser, &tunnel, size);
return 0;
}
/**
- * Convert VXLAN item to Verbs specification.
+ * Convert MPLS item to Verbs specification.
+ * MPLS tunnel types currently supported are MPLS-in-GRE and MPLS-in-UDP.
*
* @param item[in]
* Item specification.
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-mlx5_flow_create_vxlan(const struct rte_flow_item *item,
- const void *default_mask,
- struct mlx5_flow_data *data)
+mlx5_flow_create_mpls(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
{
- const struct rte_flow_item_vxlan *spec = item->spec;
- const struct rte_flow_item_vxlan *mask = item->mask;
+#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
+ (void)default_mask;
+ return rte_flow_error_set(data->error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "MPLS is not supported by driver");
+#else
+ const struct rte_flow_item_mpls *spec = item->spec;
+ const struct rte_flow_item_mpls *mask = item->mask;
struct mlx5_flow_parse *parser = data->parser;
- unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
- struct ibv_flow_spec_tunnel vxlan = {
- .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ unsigned int size = sizeof(struct ibv_flow_spec_mpls);
+ struct ibv_flow_spec_mpls mpls = {
+ .type = IBV_FLOW_SPEC_MPLS,
.size = size,
};
- union vni {
- uint32_t vlan_id;
- uint8_t vni[4];
- } id;
- id.vni[0] = 0;
parser->inner = IBV_FLOW_SPEC_INNER;
+ if (parser->layer == HASH_RXQ_UDPV4 ||
+ parser->layer == HASH_RXQ_UDPV6) {
+ parser->tunnel =
+ ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_UDP)];
+ parser->out_layer = parser->layer;
+ } else {
+ parser->tunnel =
+ ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_MPLS_IN_GRE)];
+ /* parser->out_layer stays as in GRE out_layer. */
+ }
+ parser->layer = HASH_RXQ_TUNNEL;
+ /*
+ * For MPLS-in-GRE, RSS level should have been set.
+ * For MPLS-in-UDP, use outer RSS.
+ */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 1;
if (spec) {
if (!mask)
mask = default_mask;
- memcpy(&id.vni[1], spec->vni, 3);
- vxlan.val.tunnel_id = id.vlan_id;
- memcpy(&id.vni[1], mask->vni, 3);
- vxlan.mask.tunnel_id = id.vlan_id;
+ /*
+ * The verbs label field includes the entire MPLS header:
+ * bits 0:19 - label value field.
+ * bits 20:22 - traffic class field.
+ * bits 23 - bottom of stack bit.
+ * bits 24:31 - ttl field.
+ */
+ mpls.val.label = *(const uint32_t *)spec;
+ mpls.mask.label = *(const uint32_t *)mask;
/* Remove unwanted bits from values. */
- vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+ mpls.val.label &= mpls.mask.label;
}
- /*
- * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
- * layer is defined in the Verbs specification it is interpreted as
- * wildcard and all packets will match this rule, if it follows a full
- * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
- * before will also match this rule.
- * To avoid such situation, VNI 0 is currently refused.
- */
- if (!vxlan.val.tunnel_id)
- return rte_flow_error_set(data->error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ITEM,
- item,
- "VxLAN vni cannot be 0");
- mlx5_flow_create_copy(parser, &vxlan, size);
+ mlx5_flow_create_copy(parser, &mpls, size);
return 0;
+#endif
}
/**
parser->queue[HASH_RXQ_ETH].ibv_attr;
if (parser->count)
flow->cs = parser->cs;
- if (!priv->dev->data->dev_started)
+ if (!dev->data->dev_started)
return 0;
parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
flow->frxq[HASH_RXQ_ETH].ibv_flow =
struct rte_flow *flow,
struct rte_flow_error *error)
{
- struct priv *priv = dev->data->dev_private;
unsigned int i;
for (i = 0; i != hash_rxq_init_n; ++i) {
- uint64_t hash_fields;
-
if (!parser->queue[i].ibv_attr)
continue;
flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
parser->queue[i].ibv_attr = NULL;
- hash_fields = hash_rxq_init[i].hash_fields;
- if (!priv->dev->data->dev_started)
+ flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
+ if (!dev->data->dev_started)
continue;
flow->frxq[i].hrxq =
mlx5_hrxq_get(dev,
- parser->rss_conf.rss_key,
- parser->rss_conf.rss_key_len,
- hash_fields,
- parser->queues,
- parser->queues_n);
+ parser->rss_conf.key,
+ parser->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ parser->rss_conf.queue,
+ parser->rss_conf.queue_num,
+ parser->tunnel,
+ parser->rss_conf.level);
if (flow->frxq[i].hrxq)
continue;
flow->frxq[i].hrxq =
mlx5_hrxq_new(dev,
- parser->rss_conf.rss_key,
- parser->rss_conf.rss_key_len,
- hash_fields,
- parser->queues,
- parser->queues_n);
+ parser->rss_conf.key,
+ parser->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ parser->rss_conf.queue,
+ parser->rss_conf.queue_num,
+ parser->tunnel,
+ parser->rss_conf.level);
if (!flow->frxq[i].hrxq) {
return rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_HANDLE,
return 0;
}
+/**
+ * RXQ update after flow rule creation.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param flow
+ * Pointer to the flow rule.
+ */
+static void
+mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int i;
+ unsigned int j;
+
+ if (!dev->data->dev_started)
+ return;
+ for (i = 0; i != flow->rss_conf.queue_num; ++i) {
+ struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
+ [(*flow->queues)[i]];
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+ uint8_t tunnel = PTYPE_IDX(flow->tunnel);
+
+ rxq_data->mark |= flow->mark;
+ if (!tunnel)
+ continue;
+ rxq_ctrl->tunnel_types[tunnel] += 1;
+ /* Clear tunnel type if more than one tunnel types set. */
+ for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
+ if (j == tunnel)
+ continue;
+ if (rxq_ctrl->tunnel_types[j] > 0) {
+ rxq_data->tunnel = 0;
+ break;
+ }
+ }
+ if (j == RTE_DIM(rxq_ctrl->tunnel_types))
+ rxq_data->tunnel = flow->tunnel;
+ }
+}
+
+/**
+ * Dump flow hash RX queue detail.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param flow
+ * Pointer to the rte_flow.
+ * @param hrxq_idx
+ * Hash RX queue index.
+ */
+static void
+mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
+ struct rte_flow *flow __rte_unused,
+ unsigned int hrxq_idx __rte_unused)
+{
+#ifndef NDEBUG
+ uintptr_t spec_ptr;
+ uint16_t j;
+ char buf[256];
+ uint8_t off;
+
+ spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
+ for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
+ j++) {
+ struct ibv_flow_spec *spec = (void *)spec_ptr;
+ off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
+ spec->hdr.size);
+ spec_ptr += spec->hdr.size;
+ }
+ DRV_LOG(DEBUG,
+ "port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
+ " hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
+ " flags:%x, comp_mask:%x specs:%s",
+ dev->data->port_id, (void *)flow, hrxq_idx,
+ (void *)flow->frxq[hrxq_idx].hrxq,
+ (void *)flow->frxq[hrxq_idx].hrxq->qp,
+ (void *)flow->frxq[hrxq_idx].hrxq->ind_table,
+ flow->frxq[hrxq_idx].hash_fields |
+ (flow->tunnel &&
+ flow->rss_conf.level > 1 ? (uint32_t)IBV_RX_HASH_INNER : 0),
+ flow->rss_conf.queue_num,
+ flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
+ flow->frxq[hrxq_idx].ibv_attr->size,
+ flow->frxq[hrxq_idx].ibv_attr->priority,
+ flow->frxq[hrxq_idx].ibv_attr->type,
+ flow->frxq[hrxq_idx].ibv_attr->flags,
+ flow->frxq[hrxq_idx].ibv_attr->comp_mask,
+ buf);
+#endif
+}
+
/**
* Complete flow rule creation.
*
struct rte_flow *flow,
struct rte_flow_error *error)
{
- struct priv *priv = dev->data->dev_private;
+ struct priv *priv __rte_unused = dev->data->dev_private;
int ret;
unsigned int i;
unsigned int flows_n = 0;
goto error;
if (parser->count)
flow->cs = parser->cs;
- if (!priv->dev->data->dev_started)
+ if (!dev->data->dev_started)
return 0;
for (i = 0; i != hash_rxq_init_n; ++i) {
if (!flow->frxq[i].hrxq)
flow->frxq[i].ibv_flow =
mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
flow->frxq[i].ibv_attr);
+ mlx5_flow_dump(dev, flow, i);
if (!flow->frxq[i].ibv_flow) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_HANDLE,
goto error;
}
++flows_n;
- DRV_LOG(DEBUG, "port %u %p type %d QP %p ibv_flow %p",
- dev->data->port_id,
- (void *)flow, i,
- (void *)flow->frxq[i].hrxq,
- (void *)flow->frxq[i].ibv_flow);
}
if (!flows_n) {
rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
NULL, "internal error in flow creation");
goto error;
}
- for (i = 0; i != parser->queues_n; ++i) {
- struct mlx5_rxq_data *q =
- (*priv->rxqs)[parser->queues[i]];
-
- q->mark |= parser->mark;
- }
+ mlx5_flow_create_update_rxqs(dev, flow);
return 0;
error:
ret = rte_errno; /* Save rte_errno before cleanup. */
if (ret)
goto exit;
flow = rte_calloc(__func__, 1,
- sizeof(*flow) + parser.queues_n * sizeof(uint16_t),
+ sizeof(*flow) +
+ parser.rss_conf.queue_num * sizeof(uint16_t),
0);
if (!flow) {
rte_flow_error_set(error, ENOMEM,
"cannot allocate flow memory");
return NULL;
}
- /* Copy queues configuration. */
+ /* Copy configuration. */
flow->queues = (uint16_t (*)[])(flow + 1);
- memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
- flow->queues_n = parser.queues_n;
+ flow->tunnel = parser.tunnel;
+ flow->rss_conf = (struct rte_flow_action_rss){
+ .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+ .level = 0,
+ .types = parser.rss_conf.types,
+ .key_len = parser.rss_conf.key_len,
+ .queue_num = parser.rss_conf.queue_num,
+ .key = memcpy(flow->rss_key, parser.rss_conf.key,
+ sizeof(*parser.rss_conf.key) *
+ parser.rss_conf.key_len),
+ .queue = memcpy(flow->queues, parser.rss_conf.queue,
+ sizeof(*parser.rss_conf.queue) *
+ parser.rss_conf.queue_num),
+ };
flow->mark = parser.mark;
- /* Copy RSS configuration. */
- flow->rss_conf = parser.rss_conf;
- flow->rss_conf.rss_key = flow->rss_key;
- memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
/* finalise the flow. */
if (parser.drop)
ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
struct priv *priv = dev->data->dev_private;
unsigned int i;
- if (flow->drop || !flow->mark)
+ if (flow->drop || !dev->data->dev_started)
goto free;
- for (i = 0; i != flow->queues_n; ++i) {
+ for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
+ /* Update queue tunnel type. */
+ struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
+ [(*flow->queues)[i]];
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+ uint8_t tunnel = PTYPE_IDX(flow->tunnel);
+
+ assert(rxq_ctrl->tunnel_types[tunnel] > 0);
+ rxq_ctrl->tunnel_types[tunnel] -= 1;
+ if (!rxq_ctrl->tunnel_types[tunnel]) {
+ /* Update tunnel type. */
+ uint8_t j;
+ uint8_t types = 0;
+ uint8_t last;
+
+ for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
+ if (rxq_ctrl->tunnel_types[j]) {
+ types += 1;
+ last = j;
+ }
+ /* Keep same if more than one tunnel types left. */
+ if (types == 1)
+ rxq_data->tunnel = ptype_ext[last];
+ else if (types == 0)
+ /* No tunnel type left. */
+ rxq_data->tunnel = 0;
+ }
+ }
+ for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
struct rte_flow *tmp;
int mark = 0;
{
struct priv *priv = dev->data->dev_private;
struct rte_flow *flow;
+ unsigned int i;
TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
- unsigned int i;
struct mlx5_ind_table_ibv *ind_tbl = NULL;
if (flow->drop) {
DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
(void *)flow);
}
+ /* Cleanup Rx queue tunnel info. */
+ for (i = 0; i != priv->rxqs_n; ++i) {
+ struct mlx5_rxq_data *q = (*priv->rxqs)[i];
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(q, struct mlx5_rxq_ctrl, rxq);
+
+ if (!q)
+ continue;
+ memset((void *)rxq_ctrl->tunnel_types, 0,
+ sizeof(rxq_ctrl->tunnel_types));
+ q->tunnel = 0;
+ }
}
/**
if (!flow->frxq[i].ibv_attr)
continue;
flow->frxq[i].hrxq =
- mlx5_hrxq_get(dev, flow->rss_conf.rss_key,
- flow->rss_conf.rss_key_len,
- hash_rxq_init[i].hash_fields,
- (*flow->queues),
- flow->queues_n);
+ mlx5_hrxq_get(dev, flow->rss_conf.key,
+ flow->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ flow->rss_conf.queue,
+ flow->rss_conf.queue_num,
+ flow->tunnel,
+ flow->rss_conf.level);
if (flow->frxq[i].hrxq)
goto flow_create;
flow->frxq[i].hrxq =
- mlx5_hrxq_new(dev, flow->rss_conf.rss_key,
- flow->rss_conf.rss_key_len,
- hash_rxq_init[i].hash_fields,
- (*flow->queues),
- flow->queues_n);
+ mlx5_hrxq_new(dev, flow->rss_conf.key,
+ flow->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ flow->rss_conf.queue,
+ flow->rss_conf.queue_num,
+ flow->tunnel,
+ flow->rss_conf.level);
if (!flow->frxq[i].hrxq) {
DRV_LOG(DEBUG,
- "port %u flow %p cannot be applied",
+ "port %u flow %p cannot create hash"
+ " rxq",
dev->data->port_id, (void *)flow);
rte_errno = EINVAL;
return -rte_errno;
}
flow_create:
+ mlx5_flow_dump(dev, flow, i);
flow->frxq[i].ibv_flow =
mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
flow->frxq[i].ibv_attr);
if (!flow->frxq[i].ibv_flow) {
DRV_LOG(DEBUG,
- "port %u flow %p cannot be applied",
- dev->data->port_id, (void *)flow);
+ "port %u flow %p type %u cannot be"
+ " applied",
+ dev->data->port_id, (void *)flow, i);
rte_errno = EINVAL;
return -rte_errno;
}
- DRV_LOG(DEBUG, "port %u flow %p applied",
- dev->data->port_id, (void *)flow);
}
- if (!flow->mark)
- continue;
- for (i = 0; i != flow->queues_n; ++i)
- (*priv->rxqs)[(*flow->queues)[i]]->mark = 1;
+ mlx5_flow_create_update_rxqs(dev, flow);
}
return 0;
}
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
+ uint16_t queue[priv->reta_idx_n];
+ struct rte_flow_action_rss action_rss = {
+ .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+ .level = 0,
+ .types = priv->rss_conf.rss_hf,
+ .key_len = priv->rss_conf.rss_key_len,
+ .queue_num = priv->reta_idx_n,
+ .key = priv->rss_conf.rss_key,
+ .queue = queue,
+ };
struct rte_flow_action actions[] = {
{
.type = RTE_FLOW_ACTION_TYPE_RSS,
+ .conf = &action_rss,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
struct rte_flow *flow;
struct rte_flow_error error;
unsigned int i;
- union {
- struct rte_flow_action_rss rss;
- struct {
- const struct rte_eth_rss_conf *rss_conf;
- uint16_t num;
- uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
- } local;
- } action_rss;
if (!priv->reta_idx_n) {
rte_errno = EINVAL;
return -rte_errno;
}
for (i = 0; i != priv->reta_idx_n; ++i)
- action_rss.local.queue[i] = (*priv->reta_idx)[i];
- action_rss.local.rss_conf = &priv->rss_conf;
- action_rss.local.num = priv->reta_idx_n;
- actions[0].conf = (const void *)&action_rss.rss;
+ queue[i] = (*priv->reta_idx)[i];
flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
actions, &error);
if (!flow)
int
mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
struct rte_flow *flow,
- enum rte_flow_action_type action __rte_unused,
+ const struct rte_flow_action *action __rte_unused,
void *data,
struct rte_flow_error *error)
{
}
priv->isolated = !!enable;
if (enable)
- priv->dev->dev_ops = &mlx5_dev_ops_isolate;
+ dev->dev_ops = &mlx5_dev_ops_isolate;
else
- priv->dev->dev_ops = &mlx5_dev_ops;
+ dev->dev_ops = &mlx5_dev_ops;
return 0;
}
{
struct priv *priv = dev->data->dev_private;
const struct rte_eth_fdir_input *input = &fdir_filter->input;
+ const struct rte_eth_fdir_masks *mask =
+ &dev->data->dev_conf.fdir_conf.mask;
/* Validate queue number. */
if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
return -rte_errno;
}
attributes->queue.index = fdir_filter->action.rx_queue;
+ /* Handle L3. */
switch (fdir_filter->input.flow_type) {
case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
+ case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
+ case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
attributes->l3.ipv4.hdr = (struct ipv4_hdr){
- .src_addr = input->flow.udp4_flow.ip.src_ip,
- .dst_addr = input->flow.udp4_flow.ip.dst_ip,
- .time_to_live = input->flow.udp4_flow.ip.ttl,
- .type_of_service = input->flow.udp4_flow.ip.tos,
- .next_proto_id = input->flow.udp4_flow.ip.proto,
+ .src_addr = input->flow.ip4_flow.src_ip,
+ .dst_addr = input->flow.ip4_flow.dst_ip,
+ .time_to_live = input->flow.ip4_flow.ttl,
+ .type_of_service = input->flow.ip4_flow.tos,
+ .next_proto_id = input->flow.ip4_flow.proto,
};
- attributes->l4.udp.hdr = (struct udp_hdr){
- .src_port = input->flow.udp4_flow.src_port,
- .dst_port = input->flow.udp4_flow.dst_port,
+ attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
+ .src_addr = mask->ipv4_mask.src_ip,
+ .dst_addr = mask->ipv4_mask.dst_ip,
+ .time_to_live = mask->ipv4_mask.ttl,
+ .type_of_service = mask->ipv4_mask.tos,
+ .next_proto_id = mask->ipv4_mask.proto,
};
attributes->items[1] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_IPV4,
.spec = &attributes->l3,
- .mask = &attributes->l3,
+ .mask = &attributes->l3_mask,
+ };
+ break;
+ case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
+ case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
+ case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
+ attributes->l3.ipv6.hdr = (struct ipv6_hdr){
+ .hop_limits = input->flow.ipv6_flow.hop_limits,
+ .proto = input->flow.ipv6_flow.proto,
+ };
+
+ memcpy(attributes->l3.ipv6.hdr.src_addr,
+ input->flow.ipv6_flow.src_ip,
+ RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
+ memcpy(attributes->l3.ipv6.hdr.dst_addr,
+ input->flow.ipv6_flow.dst_ip,
+ RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
+ memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
+ mask->ipv6_mask.src_ip,
+ RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
+ memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
+ mask->ipv6_mask.dst_ip,
+ RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
+ attributes->items[1] = (struct rte_flow_item){
+ .type = RTE_FLOW_ITEM_TYPE_IPV6,
+ .spec = &attributes->l3,
+ .mask = &attributes->l3_mask,
+ };
+ break;
+ default:
+ DRV_LOG(ERR, "port %u invalid flow type%d",
+ dev->data->port_id, fdir_filter->input.flow_type);
+ rte_errno = ENOTSUP;
+ return -rte_errno;
+ }
+ /* Handle L4. */
+ switch (fdir_filter->input.flow_type) {
+ case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
+ attributes->l4.udp.hdr = (struct udp_hdr){
+ .src_port = input->flow.udp4_flow.src_port,
+ .dst_port = input->flow.udp4_flow.dst_port,
+ };
+ attributes->l4_mask.udp.hdr = (struct udp_hdr){
+ .src_port = mask->src_port_mask,
+ .dst_port = mask->dst_port_mask,
};
attributes->items[2] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_UDP,
.spec = &attributes->l4,
- .mask = &attributes->l4,
+ .mask = &attributes->l4_mask,
};
break;
case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
- attributes->l3.ipv4.hdr = (struct ipv4_hdr){
- .src_addr = input->flow.tcp4_flow.ip.src_ip,
- .dst_addr = input->flow.tcp4_flow.ip.dst_ip,
- .time_to_live = input->flow.tcp4_flow.ip.ttl,
- .type_of_service = input->flow.tcp4_flow.ip.tos,
- .next_proto_id = input->flow.tcp4_flow.ip.proto,
- };
attributes->l4.tcp.hdr = (struct tcp_hdr){
.src_port = input->flow.tcp4_flow.src_port,
.dst_port = input->flow.tcp4_flow.dst_port,
};
- attributes->items[1] = (struct rte_flow_item){
- .type = RTE_FLOW_ITEM_TYPE_IPV4,
- .spec = &attributes->l3,
- .mask = &attributes->l3,
+ attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
+ .src_port = mask->src_port_mask,
+ .dst_port = mask->dst_port_mask,
};
attributes->items[2] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_TCP,
.spec = &attributes->l4,
- .mask = &attributes->l4,
- };
- break;
- case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
- attributes->l3.ipv4.hdr = (struct ipv4_hdr){
- .src_addr = input->flow.ip4_flow.src_ip,
- .dst_addr = input->flow.ip4_flow.dst_ip,
- .time_to_live = input->flow.ip4_flow.ttl,
- .type_of_service = input->flow.ip4_flow.tos,
- .next_proto_id = input->flow.ip4_flow.proto,
- };
- attributes->items[1] = (struct rte_flow_item){
- .type = RTE_FLOW_ITEM_TYPE_IPV4,
- .spec = &attributes->l3,
- .mask = &attributes->l3,
+ .mask = &attributes->l4_mask,
};
break;
case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
- attributes->l3.ipv6.hdr = (struct ipv6_hdr){
- .hop_limits = input->flow.udp6_flow.ip.hop_limits,
- .proto = input->flow.udp6_flow.ip.proto,
- };
- memcpy(attributes->l3.ipv6.hdr.src_addr,
- input->flow.udp6_flow.ip.src_ip,
- RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
- memcpy(attributes->l3.ipv6.hdr.dst_addr,
- input->flow.udp6_flow.ip.dst_ip,
- RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
attributes->l4.udp.hdr = (struct udp_hdr){
.src_port = input->flow.udp6_flow.src_port,
.dst_port = input->flow.udp6_flow.dst_port,
};
- attributes->items[1] = (struct rte_flow_item){
- .type = RTE_FLOW_ITEM_TYPE_IPV6,
- .spec = &attributes->l3,
- .mask = &attributes->l3,
+ attributes->l4_mask.udp.hdr = (struct udp_hdr){
+ .src_port = mask->src_port_mask,
+ .dst_port = mask->dst_port_mask,
};
attributes->items[2] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_UDP,
.spec = &attributes->l4,
- .mask = &attributes->l4,
+ .mask = &attributes->l4_mask,
};
break;
case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
- attributes->l3.ipv6.hdr = (struct ipv6_hdr){
- .hop_limits = input->flow.tcp6_flow.ip.hop_limits,
- .proto = input->flow.tcp6_flow.ip.proto,
- };
- memcpy(attributes->l3.ipv6.hdr.src_addr,
- input->flow.tcp6_flow.ip.src_ip,
- RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
- memcpy(attributes->l3.ipv6.hdr.dst_addr,
- input->flow.tcp6_flow.ip.dst_ip,
- RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
attributes->l4.tcp.hdr = (struct tcp_hdr){
.src_port = input->flow.tcp6_flow.src_port,
.dst_port = input->flow.tcp6_flow.dst_port,
};
- attributes->items[1] = (struct rte_flow_item){
- .type = RTE_FLOW_ITEM_TYPE_IPV6,
- .spec = &attributes->l3,
- .mask = &attributes->l3,
+ attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
+ .src_port = mask->src_port_mask,
+ .dst_port = mask->dst_port_mask,
};
attributes->items[2] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_TCP,
.spec = &attributes->l4,
- .mask = &attributes->l4,
+ .mask = &attributes->l4_mask,
};
break;
+ case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
- attributes->l3.ipv6.hdr = (struct ipv6_hdr){
- .hop_limits = input->flow.ipv6_flow.hop_limits,
- .proto = input->flow.ipv6_flow.proto,
- };
- memcpy(attributes->l3.ipv6.hdr.src_addr,
- input->flow.ipv6_flow.src_ip,
- RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
- memcpy(attributes->l3.ipv6.hdr.dst_addr,
- input->flow.ipv6_flow.dst_ip,
- RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
- attributes->items[1] = (struct rte_flow_item){
- .type = RTE_FLOW_ITEM_TYPE_IPV6,
- .spec = &attributes->l3,
- .mask = &attributes->l3,
- };
break;
default:
DRV_LOG(ERR, "port %u invalid flow type%d",
struct ibv_spec_header *flow_h;
void *flow_spec;
unsigned int specs_n;
+ unsigned int queue_id = parser.drop ? HASH_RXQ_ETH :
+ parser.layer;
- attr = parser.queue[HASH_RXQ_ETH].ibv_attr;
- flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr;
+ attr = parser.queue[queue_id].ibv_attr;
+ flow_attr = flow->frxq[queue_id].ibv_attr;
/* Compare first the attributes. */
- if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
+ if (!flow_attr ||
+ memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr)))
continue;
if (attr->num_of_specs == 0)
continue;
static void
mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
{
- struct priv *priv = dev->data->dev_private;
struct rte_eth_fdir_masks *mask =
- &priv->dev->data->dev_conf.fdir_conf.mask;
+ &dev->data->dev_conf.fdir_conf.mask;
- fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
+ fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
fdir_info->guarant_spc = 0;
rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
fdir_info->max_flexpayload = 0;
mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
void *arg)
{
- struct priv *priv = dev->data->dev_private;
enum rte_fdir_mode fdir_mode =
- priv->dev->data->dev_conf.fdir_conf.mode;
+ dev->data->dev_conf.fdir_conf.mode;
if (filter_op == RTE_ETH_FILTER_NOP)
return 0;
}
return 0;
}
+
+/**
+ * Detect number of Verbs flow priorities supported.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * number of supported Verbs flow priority.
+ */
+unsigned int
+mlx5_get_max_verbs_prio(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int verb_priorities = MLX5_VERBS_FLOW_PRIO_8;
+ struct {
+ struct ibv_flow_attr attr;
+ struct ibv_flow_spec_eth eth;
+ struct ibv_flow_spec_action_drop drop;
+ } flow_attr = {
+ .attr = {
+ .num_of_specs = 2,
+ },
+ .eth = {
+ .type = IBV_FLOW_SPEC_ETH,
+ .size = sizeof(struct ibv_flow_spec_eth),
+ },
+ .drop = {
+ .size = sizeof(struct ibv_flow_spec_action_drop),
+ .type = IBV_FLOW_SPEC_ACTION_DROP,
+ },
+ };
+ struct ibv_flow *flow;
+
+ do {
+ flow_attr.attr.priority = verb_priorities - 1;
+ flow = mlx5_glue->create_flow(priv->flow_drop_queue->qp,
+ &flow_attr.attr);
+ if (flow) {
+ claim_zero(mlx5_glue->destroy_flow(flow));
+ /* Try more priorities. */
+ verb_priorities *= 2;
+ } else {
+ /* Failed, restore last right number. */
+ verb_priorities /= 2;
+ break;
+ }
+ } while (1);
+ DRV_LOG(DEBUG, "port %u Verbs flow priorities: %d,"
+ " user flow priorities: %d",
+ dev->data->port_id, verb_priorities, MLX5_CTRL_FLOW_PRIORITY);
+ return verb_priorities;
+}