X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_verbs.c;h=176d867202288ce4eb5151d4dc3c781123f0203e;hb=d61138d4f0e2927cc1d0fef6d810fa7f5d1161a1;hp=ef4d7a36204bf74e7d05045e0e8bbcd3d0f08fe5;hpb=3ac3d8234b823126739475528cc23cf9ad2a8c17;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index ef4d7a3620..176d867202 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -8,19 +8,9 @@ #include #include -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif - #include #include -#include +#include #include #include #include @@ -28,15 +18,144 @@ #include #include +#include #include "mlx5_defs.h" #include "mlx5.h" #include "mlx5_flow.h" -#include "mlx5_rxtx.h" +#include "mlx5_rx.h" #define VERBS_SPEC_INNER(item_flags) \ (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0) +/* Map of Verbs to Flow priority with 8 Verbs priorities. */ +static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { + { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, +}; + +/* Map of Verbs to Flow priority with 16 Verbs priorities. */ +static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { + { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, + { 9, 10, 11 }, { 12, 13, 14 }, +}; + +/* Verbs specification header. */ +struct ibv_spec_header { + enum ibv_flow_spec_type type; + uint16_t size; +}; + +/** + * Discover the maximum number of priority available. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * number of supported flow priority on success, a negative errno + * value otherwise and rte_errno is set. + */ +int +mlx5_flow_discover_priorities(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct { + struct ibv_flow_attr attr; + struct ibv_flow_spec_eth eth; + struct ibv_flow_spec_action_drop drop; + } flow_attr = { + .attr = { + .num_of_specs = 2, + .port = (uint8_t)priv->dev_port, + }, + .eth = { + .type = IBV_FLOW_SPEC_ETH, + .size = sizeof(struct ibv_flow_spec_eth), + }, + .drop = { + .size = sizeof(struct ibv_flow_spec_action_drop), + .type = IBV_FLOW_SPEC_ACTION_DROP, + }, + }; + struct ibv_flow *flow; + struct mlx5_hrxq *drop = priv->drop_queue.hrxq; + uint16_t vprio[] = { 8, 16 }; + int i; + int priority = 0; + +#if defined(HAVE_MLX5DV_DR_DEVX_PORT) || defined(HAVE_MLX5DV_DR_DEVX_PORT_V35) + /* If DevX supported, driver must support 16 verbs flow priorities. */ + priority = RTE_DIM(priority_map_5); + goto out; +#endif + if (!drop->qp) { + rte_errno = ENOTSUP; + return -rte_errno; + } + for (i = 0; i != RTE_DIM(vprio); i++) { + flow_attr.attr.priority = vprio[i] - 1; + flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); + if (!flow) + break; + claim_zero(mlx5_glue->destroy_flow(flow)); + priority = vprio[i]; + } + switch (priority) { + case 8: + priority = RTE_DIM(priority_map_3); + break; + case 16: + priority = RTE_DIM(priority_map_5); + break; + default: + rte_errno = ENOTSUP; + DRV_LOG(ERR, + "port %u verbs maximum priority: %d expected 8/16", + dev->data->port_id, priority); + return -rte_errno; + } +#if defined(HAVE_MLX5DV_DR_DEVX_PORT) || defined(HAVE_MLX5DV_DR_DEVX_PORT_V35) +out: +#endif + DRV_LOG(INFO, "port %u supported flow priorities:" + " 0-%d for ingress or egress root table," + " 0-%d for non-root table or transfer root table.", + dev->data->port_id, priority - 2, + MLX5_NON_ROOT_FLOW_MAX_PRIO - 1); + return priority; +} + +/** + * Adjust flow priority based on the highest layer and the request priority. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] priority + * The rule base priority. + * @param[in] subpriority + * The priority based on the items. + * + * @return + * The new priority. + */ +uint32_t +mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, + uint32_t subpriority) +{ + uint32_t res = 0; + struct mlx5_priv *priv = dev->data->dev_private; + + switch (priv->config.flow_prio) { + case RTE_DIM(priority_map_3): + res = priority_map_3[priority][subpriority]; + break; + case RTE_DIM(priority_map_5): + res = priority_map_5[priority][subpriority]; + break; + } + return res; +} + /** * Get Verbs flow counter by index. * @@ -56,15 +175,15 @@ flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, struct mlx5_flow_counter_pool **ppool) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0); + struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; struct mlx5_flow_counter_pool *pool; - idx--; - pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL]; + idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1); + pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL]; MLX5_ASSERT(pool); if (ppool) *ppool = pool; - return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL]; + return MLX5_POOL_GET_CNT(pool, idx % MLX5_COUNTERS_PER_POOL); } /** @@ -83,45 +202,45 @@ flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, */ static int flow_verbs_counter_create(struct rte_eth_dev *dev, - struct mlx5_flow_counter_ext *counter) + struct mlx5_flow_counter *counter) { #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) struct mlx5_priv *priv = dev->data->dev_private; - struct ibv_context *ctx = priv->sh->ctx; + struct ibv_context *ctx = priv->sh->cdev->ctx; struct ibv_counter_set_init_attr init = { - .counter_set_id = counter->id}; + .counter_set_id = counter->shared_info.id}; - counter->cs = mlx5_glue->create_counter_set(ctx, &init); - if (!counter->cs) { + counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init); + if (!counter->dcs_when_free) { rte_errno = ENOTSUP; return -ENOTSUP; } return 0; #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) struct mlx5_priv *priv = dev->data->dev_private; - struct ibv_context *ctx = priv->sh->ctx; + struct ibv_context *ctx = priv->sh->cdev->ctx; struct ibv_counters_init_attr init = {0}; struct ibv_counter_attach_attr attach; int ret; memset(&attach, 0, sizeof(attach)); - counter->cs = mlx5_glue->create_counters(ctx, &init); - if (!counter->cs) { + counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init); + if (!counter->dcs_when_free) { rte_errno = ENOTSUP; return -ENOTSUP; } attach.counter_desc = IBV_COUNTER_PACKETS; attach.index = 0; - ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL); + ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL); if (!ret) { attach.counter_desc = IBV_COUNTER_BYTES; attach.index = 1; ret = mlx5_glue->attach_counters - (counter->cs, &attach, NULL); + (counter->dcs_when_free, &attach, NULL); } if (ret) { - claim_zero(mlx5_glue->destroy_counters(counter->cs)); - counter->cs = NULL; + claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free)); + counter->dcs_when_free = NULL; rte_errno = ret; return -ret; } @@ -139,8 +258,6 @@ flow_verbs_counter_create(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to the Ethernet device structure. - * @param[in] shared - * Indicate if this counter is shared with other flows. * @param[in] id * Counter identifier. * @@ -148,35 +265,22 @@ flow_verbs_counter_create(struct rte_eth_dev *dev, * Index to the counter, 0 otherwise and rte_errno is set. */ static uint32_t -flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) +flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t id __rte_unused) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0); + struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; struct mlx5_flow_counter_pool *pool = NULL; - struct mlx5_flow_counter_ext *cnt_ext = NULL; struct mlx5_flow_counter *cnt = NULL; - uint32_t n_valid = rte_atomic16_read(&cont->n_valid); - uint32_t pool_idx; + uint32_t n_valid = cmng->n_valid; + uint32_t pool_idx, cnt_idx; uint32_t i; int ret; - if (shared) { - for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { - pool = cont->pools[pool_idx]; - for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { - cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i); - if (cnt_ext->shared && cnt_ext->id == id) { - cnt_ext->ref_cnt++; - return MLX5_MAKE_CNT_IDX(pool_idx, i); - } - } - } - } for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { - pool = cont->pools[pool_idx]; + pool = cmng->pools[pool_idx]; if (!pool) continue; - cnt = TAILQ_FIRST(&pool->counters); + cnt = TAILQ_FIRST(&pool->counters[0]); if (cnt) break; } @@ -184,51 +288,49 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) struct mlx5_flow_counter_pool **pools; uint32_t size; - if (n_valid == cont->n) { + if (n_valid == cmng->n) { /* Resize the container pool array. */ size = sizeof(struct mlx5_flow_counter_pool *) * (n_valid + MLX5_CNT_CONTAINER_RESIZE); - pools = rte_zmalloc(__func__, size, 0); + pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0, + SOCKET_ID_ANY); if (!pools) return 0; if (n_valid) { - memcpy(pools, cont->pools, + memcpy(pools, cmng->pools, sizeof(struct mlx5_flow_counter_pool *) * n_valid); - rte_free(cont->pools); + mlx5_free(cmng->pools); } - cont->pools = pools; - cont->n += MLX5_CNT_CONTAINER_RESIZE; + cmng->pools = pools; + cmng->n += MLX5_CNT_CONTAINER_RESIZE; } /* Allocate memory for new pool*/ - size = sizeof(*pool) + sizeof(*cnt_ext) * - MLX5_COUNTERS_PER_POOL; - pool = rte_calloc(__func__, 1, size, 0); + size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL; + pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); if (!pool) return 0; for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { - cnt = &pool->counters_raw[i]; - TAILQ_INSERT_HEAD(&pool->counters, cnt, next); + cnt = MLX5_POOL_GET_CNT(pool, i); + TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); } - cnt = &pool->counters_raw[0]; - cont->pools[n_valid] = pool; + cnt = MLX5_POOL_GET_CNT(pool, 0); + cmng->pools[n_valid] = pool; pool_idx = n_valid; - rte_atomic16_add(&cont->n_valid, 1); - TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); + cmng->n_valid++; } - i = cnt - pool->counters_raw; - cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i); - cnt_ext->id = id; - cnt_ext->shared = shared; - cnt_ext->ref_cnt = 1; - cnt->hits = 0; - cnt->bytes = 0; + TAILQ_REMOVE(&pool->counters[0], cnt, next); + i = MLX5_CNT_ARRAY_IDX(pool, cnt); + cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i); /* Create counter with Verbs. */ - ret = flow_verbs_counter_create(dev, cnt_ext); + ret = flow_verbs_counter_create(dev, cnt); if (!ret) { - TAILQ_REMOVE(&pool->counters, cnt, next); - return MLX5_MAKE_CNT_IDX(pool_idx, i); + cnt->dcs_when_active = cnt->dcs_when_free; + cnt->hits = 0; + cnt->bytes = 0; + return cnt_idx; } + TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); /* Some error occurred in Verbs library. */ rte_errno = -ret; return 0; @@ -247,21 +349,16 @@ flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter) { struct mlx5_flow_counter_pool *pool; struct mlx5_flow_counter *cnt; - struct mlx5_flow_counter_ext *cnt_ext; - cnt = flow_verbs_counter_get_by_idx(dev, counter, - &pool); - cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); - if (--cnt_ext->ref_cnt == 0) { + cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool); #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) - claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs)); - cnt_ext->cs = NULL; + claim_zero(mlx5_glue->destroy_counter_set + ((struct ibv_counter_set *)cnt->dcs_when_active)); #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) - claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs)); - cnt_ext->cs = NULL; + claim_zero(mlx5_glue->destroy_counters + ((struct ibv_counters *)cnt->dcs_when_active)); #endif - TAILQ_INSERT_HEAD(&pool->counters, cnt, next); - } + TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); } /** @@ -281,13 +378,12 @@ flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, struct mlx5_flow_counter_pool *pool; struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx (dev, flow->counter, &pool); - struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT - (pool, cnt); struct rte_flow_query_count *qc = data; uint64_t counters[2] = {0, 0}; #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) struct ibv_query_counter_set_attr query_cs_attr = { - .cs = cnt_ext->cs, + .dcs_when_free = (struct ibv_counter_set *) + cnt->dcs_when_active, .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, }; struct ibv_counter_set_data query_out = { @@ -298,7 +394,7 @@ flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, &query_out); #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) int err = mlx5_glue->query_counters - (cnt_ext->cs, counters, + ((struct ibv_counters *)cnt->dcs_when_active, counters, RTE_DIM(counters), IBV_READ_COUNTERS_ATTR_PREFER_CACHED); #endif @@ -678,6 +774,28 @@ flow_verbs_translate_item_udp(struct mlx5_flow *dev_flow, udp.val.src_port &= udp.mask.src_port; udp.val.dst_port &= udp.mask.dst_port; } + item++; + while (item->type == RTE_FLOW_ITEM_TYPE_VOID) + item++; + if (!(udp.val.dst_port & udp.mask.dst_port)) { + switch ((item)->type) { + case RTE_FLOW_ITEM_TYPE_VXLAN: + udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN); + udp.mask.dst_port = 0xffff; + break; + case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: + udp.val.dst_port = htons(MLX5_UDP_PORT_VXLAN_GPE); + udp.mask.dst_port = 0xffff; + break; + case RTE_FLOW_ITEM_TYPE_MPLS: + udp.val.dst_port = htons(MLX5_UDP_PORT_MPLS); + udp.mask.dst_port = 0xffff; + break; + default: + break; + } + } + flow_verbs_spec_add(&dev_flow->verbs, &udp, size); } @@ -941,21 +1059,19 @@ flow_verbs_translate_action_drop * the input is valid and that there is space to insert the requested action * into the flow. * - * @param[in] dev_flow - * Pointer to mlx5_flow. + * @param[in] rss_desc + * Pointer to mlx5_flow_rss_desc. * @param[in] action * Action configuration. */ static void -flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow, +flow_verbs_translate_action_queue(struct mlx5_flow_rss_desc *rss_desc, const struct rte_flow_action *action) { const struct rte_flow_action_queue *queue = action->conf; - struct rte_flow *flow = dev_flow->flow; - if (flow->rss.queue) - (*flow->rss.queue)[0] = queue->index; - flow->rss.queue_num = 1; + rss_desc->queue[0] = queue->index; + rss_desc->queue_num = 1; } /** @@ -963,28 +1079,23 @@ flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow, * the input is valid and that there is space to insert the requested action * into the flow. * + * @param[in] rss_desc + * Pointer to mlx5_flow_rss_desc. * @param[in] action * Action configuration. - * @param[in, out] action_flags - * Pointer to the detected actions. - * @param[in] dev_flow - * Pointer to mlx5_flow. */ static void -flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow, +flow_verbs_translate_action_rss(struct mlx5_flow_rss_desc *rss_desc, const struct rte_flow_action *action) { const struct rte_flow_action_rss *rss = action->conf; const uint8_t *rss_key; - struct rte_flow *flow = dev_flow->flow; - if (flow->rss.queue) - memcpy((*flow->rss.queue), rss->queue, - rss->queue_num * sizeof(uint16_t)); - flow->rss.queue_num = rss->queue_num; + memcpy(rss_desc->queue, rss->queue, rss->queue_num * sizeof(uint16_t)); + rss_desc->queue_num = rss->queue_num; /* NULL RSS key indicates default RSS key. */ rss_key = !rss->key ? rss_hash_default_key : rss->key; - memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN); + memcpy(rss_desc->key, rss_key, MLX5_RSS_HASH_KEY_LEN); /* * rss->level and rss.types should be set in advance when expanding * items for RSS. @@ -1070,7 +1181,6 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) struct mlx5_flow_counter_pool *pool; struct mlx5_flow_counter *cnt = NULL; - struct mlx5_flow_counter_ext *cnt_ext; unsigned int size = sizeof(struct ibv_flow_spec_counter_action); struct ibv_flow_spec_counter_action counter = { .type = IBV_FLOW_SPEC_ACTION_COUNT, @@ -1079,8 +1189,7 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, #endif if (!flow->counter) { - flow->counter = flow_verbs_counter_new(dev, count->shared, - count->id); + flow->counter = flow_verbs_counter_new(dev, count->id); if (!flow->counter) return rte_flow_error_set(error, rte_errno, RTE_FLOW_ERROR_TYPE_ACTION, @@ -1090,13 +1199,12 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, } #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); - cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); - counter.counter_set_handle = cnt_ext->cs->handle; + counter.counter_set_handle = + ((struct ibv_counter_set *)cnt->dcs_when_active)->handle; flow_verbs_spec_add(&dev_flow->verbs, &counter, size); #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); - cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); - counter.counters = cnt_ext->cs; + counter.counters = (struct ibv_counters *)cnt->dcs_when_active; flow_verbs_spec_add(&dev_flow->verbs, &counter, size); #endif return 0; @@ -1115,6 +1223,8 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, * Pointer to the list of actions. * @param[in] external * This flow rule is created by request external to PMD. + * @param[in] hairpin + * Number of hairpin TX actions, 0 means classic flow. * @param[out] error * Pointer to the error structure. * @@ -1127,6 +1237,7 @@ flow_verbs_validate(struct rte_eth_dev *dev, const struct rte_flow_item items[], const struct rte_flow_action actions[], bool external __rte_unused, + int hairpin __rte_unused, struct rte_flow_error *error) { int ret; @@ -1135,6 +1246,8 @@ flow_verbs_validate(struct rte_eth_dev *dev, uint64_t last_item = 0; uint8_t next_protocol = 0xff; uint16_t ether_type = 0; + bool is_empty_vlan = false; + uint16_t udp_dport = 0; if (items == NULL) return -1; @@ -1150,7 +1263,7 @@ flow_verbs_validate(struct rte_eth_dev *dev, break; case RTE_FLOW_ITEM_TYPE_ETH: ret = mlx5_flow_validate_item_eth(items, item_flags, - error); + false, error); if (ret < 0) return ret; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L2 : @@ -1162,6 +1275,8 @@ flow_verbs_validate(struct rte_eth_dev *dev, ether_type &= ((const struct rte_flow_item_eth *) items->mask)->type; + if (ether_type == RTE_BE16(RTE_ETHER_TYPE_VLAN)) + is_empty_vlan = true; ether_type = rte_be_to_cpu_16(ether_type); } else { ether_type = 0; @@ -1187,12 +1302,14 @@ flow_verbs_validate(struct rte_eth_dev *dev, } else { ether_type = 0; } + is_empty_vlan = false; break; case RTE_FLOW_ITEM_TYPE_IPV4: - ret = mlx5_flow_validate_item_ipv4(items, item_flags, - last_item, - ether_type, NULL, - error); + ret = mlx5_flow_validate_item_ipv4 + (items, item_flags, + last_item, ether_type, NULL, + MLX5_ITEM_RANGE_NOT_ACCEPTED, + error); if (ret < 0) return ret; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : @@ -1238,6 +1355,15 @@ flow_verbs_validate(struct rte_eth_dev *dev, ret = mlx5_flow_validate_item_udp(items, item_flags, next_protocol, error); + const struct rte_flow_item_udp *spec = items->spec; + const struct rte_flow_item_udp *mask = items->mask; + if (!mask) + mask = &rte_flow_item_udp_mask; + if (spec != NULL) + udp_dport = rte_be_to_cpu_16 + (spec->hdr.dst_port & + mask->hdr.dst_port); + if (ret < 0) return ret; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : @@ -1255,8 +1381,9 @@ flow_verbs_validate(struct rte_eth_dev *dev, MLX5_FLOW_LAYER_OUTER_L4_TCP; break; case RTE_FLOW_ITEM_TYPE_VXLAN: - ret = mlx5_flow_validate_item_vxlan(items, item_flags, - error); + ret = mlx5_flow_validate_item_vxlan(dev, udp_dport, + items, item_flags, + attr, error); if (ret < 0) return ret; last_item = MLX5_FLOW_LAYER_VXLAN; @@ -1284,6 +1411,12 @@ flow_verbs_validate(struct rte_eth_dev *dev, return ret; last_item = MLX5_FLOW_LAYER_MPLS; break; + case RTE_FLOW_ITEM_TYPE_ICMP: + case RTE_FLOW_ITEM_TYPE_ICMP6: + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, + NULL, "ICMP/ICMP6 " + "item not supported"); default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, @@ -1291,6 +1424,10 @@ flow_verbs_validate(struct rte_eth_dev *dev, } item_flags |= last_item; } + if (is_empty_vlan) + return rte_flow_error_set(error, ENOTSUP, + RTE_FLOW_ERROR_TYPE_ITEM, NULL, + "VLAN matching without vid specification is not supported"); for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_VOID: @@ -1505,10 +1642,13 @@ flow_verbs_prepare(struct rte_eth_dev *dev, struct rte_flow_error *error) { size_t size = 0; + uint32_t handle_idx = 0; struct mlx5_flow *dev_flow; struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + MLX5_ASSERT(wks); size += flow_verbs_get_actions_size(actions); size += flow_verbs_get_items_size(items); if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) { @@ -1518,26 +1658,29 @@ flow_verbs_prepare(struct rte_eth_dev *dev, return NULL; } /* In case of corrupting the memory. */ - if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { + if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) { rte_flow_error_set(error, ENOSPC, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "not free temporary device flow"); return NULL; } - dev_handle = rte_calloc(__func__, 1, MLX5_FLOW_HANDLE_VERBS_SIZE, 0); + dev_handle = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + &handle_idx); if (!dev_handle) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "not enough memory to create flow handle"); return NULL; } - /* No multi-thread supporting. */ - dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++]; + MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows)); + dev_flow = &wks->flows[wks->flow_idx++]; dev_flow->handle = dev_handle; + dev_flow->handle_idx = handle_idx; /* Memcpy is used, only size needs to be cleared to 0. */ dev_flow->verbs.size = 0; dev_flow->verbs.attr.num_of_specs = 0; dev_flow->ingress = attr->ingress; + dev_flow->hash_fields = 0; /* Need to set transfer attribute: not supported in Verbs mode. */ return dev_flow; } @@ -1574,8 +1717,12 @@ flow_verbs_translate(struct rte_eth_dev *dev, uint64_t priority = attr->priority; uint32_t subpriority = 0; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); + struct mlx5_flow_rss_desc *rss_desc; - if (priority == MLX5_FLOW_PRIO_RSVD) + MLX5_ASSERT(wks); + rss_desc = &wks->rss_desc; + if (priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR) priority = priv->config.flow_prio - 1; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { int ret; @@ -1586,22 +1733,27 @@ flow_verbs_translate(struct rte_eth_dev *dev, case RTE_FLOW_ACTION_TYPE_FLAG: flow_verbs_translate_action_flag(dev_flow, actions); action_flags |= MLX5_FLOW_ACTION_FLAG; + dev_flow->handle->mark = 1; break; case RTE_FLOW_ACTION_TYPE_MARK: flow_verbs_translate_action_mark(dev_flow, actions); action_flags |= MLX5_FLOW_ACTION_MARK; + dev_flow->handle->mark = 1; break; case RTE_FLOW_ACTION_TYPE_DROP: flow_verbs_translate_action_drop(dev_flow, actions); action_flags |= MLX5_FLOW_ACTION_DROP; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP; break; case RTE_FLOW_ACTION_TYPE_QUEUE: - flow_verbs_translate_action_queue(dev_flow, actions); + flow_verbs_translate_action_queue(rss_desc, actions); action_flags |= MLX5_FLOW_ACTION_QUEUE; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; break; case RTE_FLOW_ACTION_TYPE_RSS: - flow_verbs_translate_action_rss(dev_flow, actions); + flow_verbs_translate_action_rss(rss_desc, actions); action_flags |= MLX5_FLOW_ACTION_RSS; + dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE; break; case RTE_FLOW_ACTION_TYPE_COUNT: ret = flow_verbs_translate_action_count(dev_flow, @@ -1618,7 +1770,7 @@ flow_verbs_translate(struct rte_eth_dev *dev, "action not supported"); } } - dev_flow->handle->act_flags = action_flags; + dev_flow->act_flags = action_flags; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL); @@ -1647,7 +1799,7 @@ flow_verbs_translate(struct rte_eth_dev *dev, subpriority = MLX5_PRIORITY_MAP_L3; dev_flow->hash_fields |= mlx5_flow_hashfields_adjust - (dev_flow, tunnel, + (rss_desc, tunnel, MLX5_IPV4_LAYER_TYPES, MLX5_IPV4_IBV_RX_HASH); item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : @@ -1659,7 +1811,7 @@ flow_verbs_translate(struct rte_eth_dev *dev, subpriority = MLX5_PRIORITY_MAP_L3; dev_flow->hash_fields |= mlx5_flow_hashfields_adjust - (dev_flow, tunnel, + (rss_desc, tunnel, MLX5_IPV6_LAYER_TYPES, MLX5_IPV6_IBV_RX_HASH); item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 : @@ -1669,9 +1821,10 @@ flow_verbs_translate(struct rte_eth_dev *dev, flow_verbs_translate_item_tcp(dev_flow, items, item_flags); subpriority = MLX5_PRIORITY_MAP_L4; - dev_flow->hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, ETH_RSS_TCP, + if (dev_flow->hash_fields != 0) + dev_flow->hash_fields |= + mlx5_flow_hashfields_adjust + (rss_desc, tunnel, RTE_ETH_RSS_TCP, (IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP)); item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP : @@ -1681,9 +1834,10 @@ flow_verbs_translate(struct rte_eth_dev *dev, flow_verbs_translate_item_udp(dev_flow, items, item_flags); subpriority = MLX5_PRIORITY_MAP_L4; - dev_flow->hash_fields |= - mlx5_flow_hashfields_adjust - (dev_flow, tunnel, ETH_RSS_UDP, + if (dev_flow->hash_fields != 0) + dev_flow->hash_fields |= + mlx5_flow_hashfields_adjust + (rss_desc, tunnel, RTE_ETH_RSS_UDP, (IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP)); item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP : @@ -1692,39 +1846,38 @@ flow_verbs_translate(struct rte_eth_dev *dev, case RTE_FLOW_ITEM_TYPE_VXLAN: flow_verbs_translate_item_vxlan(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_VXLAN; break; case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: flow_verbs_translate_item_vxlan_gpe(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE; break; case RTE_FLOW_ITEM_TYPE_GRE: flow_verbs_translate_item_gre(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_GRE; break; case RTE_FLOW_ITEM_TYPE_MPLS: flow_verbs_translate_item_mpls(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_MPLS; break; default: return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, - NULL, - "item not supported"); + NULL, "item not supported"); } } dev_flow->handle->layers = item_flags; /* Other members of attr will be ignored. */ dev_flow->verbs.attr.priority = mlx5_flow_adjust_priority(dev, priority, subpriority); - dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port; + dev_flow->verbs.attr.port = (uint8_t)priv->dev_port; return 0; } @@ -1739,21 +1892,23 @@ flow_verbs_translate(struct rte_eth_dev *dev, static void flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_handle *handle; + uint32_t handle_idx; if (!flow) return; - LIST_FOREACH(handle, &flow->dev_handles, next) { - if (handle->ib_flow) { - claim_zero(mlx5_glue->destroy_flow(handle->ib_flow)); - handle->ib_flow = NULL; + SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, + handle_idx, handle, next) { + if (handle->drv_flow) { + claim_zero(mlx5_glue->destroy_flow(handle->drv_flow)); + handle->drv_flow = NULL; } - if (handle->hrxq) { - if (handle->act_flags & MLX5_FLOW_ACTION_DROP) - mlx5_hrxq_drop_release(dev); - else - mlx5_hrxq_release(dev, handle->hrxq); - handle->hrxq = NULL; + /* hrxq is union, don't touch it only the flag is set. */ + if (handle->rix_hrxq && + handle->fate_action == MLX5_FLOW_FATE_QUEUE) { + mlx5_hrxq_release(dev, handle->rix_hrxq); + handle->rix_hrxq = 0; } if (handle->vf_vlan.tag && handle->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); @@ -1771,15 +1926,22 @@ flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) static void flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_handle *handle; if (!flow) return; flow_verbs_remove(dev, flow); - while (!LIST_EMPTY(&flow->dev_handles)) { - handle = LIST_FIRST(&flow->dev_handles); - LIST_REMOVE(handle, next); - rte_free(handle); + while (flow->dev_handles) { + uint32_t tmp_idx = flow->dev_handles; + + handle = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + tmp_idx); + if (!handle) + return; + flow->dev_handles = handle->next.next; + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], + tmp_idx); } if (flow->counter) { flow_verbs_counter_release(dev, flow->counter); @@ -1807,38 +1969,32 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_handle *handle; struct mlx5_flow *dev_flow; + struct mlx5_hrxq *hrxq; + uint32_t dev_handles; int err; int idx; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); - for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) { - dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; + MLX5_ASSERT(wks); + for (idx = wks->flow_idx - 1; idx >= 0; idx--) { + dev_flow = &wks->flows[idx]; handle = dev_flow->handle; - if (handle->act_flags & MLX5_FLOW_ACTION_DROP) { - handle->hrxq = mlx5_hrxq_drop_new(dev); - if (!handle->hrxq) { - rte_flow_error_set - (error, errno, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot get drop hash queue"); - goto error; - } + if (handle->fate_action == MLX5_FLOW_FATE_DROP) { + MLX5_ASSERT(priv->drop_queue.hrxq); + hrxq = priv->drop_queue.hrxq; } else { - struct mlx5_hrxq *hrxq; - - MLX5_ASSERT(flow->rss.queue); - hrxq = mlx5_hrxq_get(dev, flow->rss.key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - (*flow->rss.queue), - flow->rss.queue_num); - if (!hrxq) - hrxq = mlx5_hrxq_new(dev, flow->rss.key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - (*flow->rss.queue), - flow->rss.queue_num, - !!(handle->layers & - MLX5_FLOW_LAYER_TUNNEL)); + uint32_t hrxq_idx; + struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc; + + MLX5_ASSERT(rss_desc->queue_num); + rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc->hash_fields = dev_flow->hash_fields; + rss_desc->tunnel = !!(handle->layers & + MLX5_FLOW_LAYER_TUNNEL); + rss_desc->shared_rss = 0; + hrxq_idx = mlx5_hrxq_get(dev, rss_desc); + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], + hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -1846,11 +2002,12 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, "cannot get hash queue"); goto error; } - handle->hrxq = hrxq; + handle->rix_hrxq = hrxq_idx; } - handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp, - &dev_flow->verbs.attr); - if (!handle->ib_flow) { + MLX5_ASSERT(hrxq); + handle->drv_flow = mlx5_glue->create_flow + (hrxq->qp, &dev_flow->verbs.attr); + if (!handle->drv_flow) { rte_flow_error_set(error, errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -1871,13 +2028,13 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, return 0; error: err = rte_errno; /* Save rte_errno before cleanup. */ - LIST_FOREACH(handle, &flow->dev_handles, next) { - if (handle->hrxq) { - if (handle->act_flags & MLX5_FLOW_ACTION_DROP) - mlx5_hrxq_drop_release(dev); - else - mlx5_hrxq_release(dev, handle->hrxq); - handle->hrxq = NULL; + SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, + dev_handles, handle, next) { + /* hrxq is union, don't touch it only the flag is set. */ + if (handle->rix_hrxq && + handle->fate_action == MLX5_FLOW_FATE_QUEUE) { + mlx5_hrxq_release(dev, handle->rix_hrxq); + handle->rix_hrxq = 0; } if (handle->vf_vlan.tag && handle->vf_vlan.created) mlx5_vlan_vmwa_release(dev, &handle->vf_vlan); @@ -1918,6 +2075,17 @@ flow_verbs_query(struct rte_eth_dev *dev, return ret; } +static int +flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains, + uint32_t flags) +{ + RTE_SET_USED(dev); + RTE_SET_USED(domains); + RTE_SET_USED(flags); + + return 0; +} + const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = { .validate = flow_verbs_validate, .prepare = flow_verbs_prepare, @@ -1926,4 +2094,5 @@ const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = { .remove = flow_verbs_remove, .destroy = flow_verbs_destroy, .query = flow_verbs_query, + .sync_domain = flow_verbs_sync_domain, };