X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_verbs.c;h=9cc4410667b6dd1a4651f7532c335d11a136a42c;hb=872ba77bf7a20ee8e321f67463ab38ca4c429bb2;hp=781c97fd2eaf0259c7c42f2e2609058a4d6d3a25;hpb=341c894104983f5608671e8fb2be19ade03ed0e9;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index 781c97fd2e..9cc4410667 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -8,16 +8,6 @@ #include #include -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif - #include #include #include @@ -28,6 +18,7 @@ #include #include +#include #include "mlx5_defs.h" #include "mlx5.h" @@ -37,6 +28,118 @@ #define VERBS_SPEC_INNER(item_flags) \ (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0) +/* Map of Verbs to Flow priority with 8 Verbs priorities. */ +static const uint32_t priority_map_3[][MLX5_PRIORITY_MAP_MAX] = { + { 0, 1, 2 }, { 2, 3, 4 }, { 5, 6, 7 }, +}; + +/* Map of Verbs to Flow priority with 16 Verbs priorities. */ +static const uint32_t priority_map_5[][MLX5_PRIORITY_MAP_MAX] = { + { 0, 1, 2 }, { 3, 4, 5 }, { 6, 7, 8 }, + { 9, 10, 11 }, { 12, 13, 14 }, +}; + +/** + * Discover the maximum number of priority available. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * number of supported flow priority on success, a negative errno + * value otherwise and rte_errno is set. + */ +int +mlx5_flow_discover_priorities(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct { + struct ibv_flow_attr attr; + struct ibv_flow_spec_eth eth; + struct ibv_flow_spec_action_drop drop; + } flow_attr = { + .attr = { + .num_of_specs = 2, + .port = (uint8_t)priv->dev_port, + }, + .eth = { + .type = IBV_FLOW_SPEC_ETH, + .size = sizeof(struct ibv_flow_spec_eth), + }, + .drop = { + .size = sizeof(struct ibv_flow_spec_action_drop), + .type = IBV_FLOW_SPEC_ACTION_DROP, + }, + }; + struct ibv_flow *flow; + struct mlx5_hrxq *drop = mlx5_drop_action_create(dev); + uint16_t vprio[] = { 8, 16 }; + int i; + int priority = 0; + + if (!drop) { + rte_errno = ENOTSUP; + return -rte_errno; + } + for (i = 0; i != RTE_DIM(vprio); i++) { + flow_attr.attr.priority = vprio[i] - 1; + flow = mlx5_glue->create_flow(drop->qp, &flow_attr.attr); + if (!flow) + break; + claim_zero(mlx5_glue->destroy_flow(flow)); + priority = vprio[i]; + } + mlx5_drop_action_destroy(dev); + switch (priority) { + case 8: + priority = RTE_DIM(priority_map_3); + break; + case 16: + priority = RTE_DIM(priority_map_5); + break; + default: + rte_errno = ENOTSUP; + DRV_LOG(ERR, + "port %u verbs maximum priority: %d expected 8/16", + dev->data->port_id, priority); + return -rte_errno; + } + DRV_LOG(INFO, "port %u flow maximum priority: %d", + dev->data->port_id, priority); + return priority; +} + +/** + * Adjust flow priority based on the highest layer and the request priority. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] priority + * The rule base priority. + * @param[in] subpriority + * The priority based on the items. + * + * @return + * The new priority. + */ +uint32_t +mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority, + uint32_t subpriority) +{ + uint32_t res = 0; + struct mlx5_priv *priv = dev->data->dev_private; + + switch (priv->config.flow_prio) { + case RTE_DIM(priority_map_3): + res = priority_map_3[priority][subpriority]; + break; + case RTE_DIM(priority_map_5): + res = priority_map_5[priority][subpriority]; + break; + } + return res; +} + /** * Get Verbs flow counter by index. * @@ -56,11 +159,11 @@ flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, struct mlx5_flow_counter_pool **ppool) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0); + struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; struct mlx5_flow_counter_pool *pool; - idx--; - pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL]; + idx = (idx - 1) & (MLX5_CNT_SHARED_OFFSET - 1); + pool = cmng->pools[idx / MLX5_COUNTERS_PER_POOL]; MLX5_ASSERT(pool); if (ppool) *ppool = pool; @@ -83,16 +186,16 @@ flow_verbs_counter_get_by_idx(struct rte_eth_dev *dev, */ static int flow_verbs_counter_create(struct rte_eth_dev *dev, - struct mlx5_flow_counter_ext *counter) + struct mlx5_flow_counter *counter) { #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) struct mlx5_priv *priv = dev->data->dev_private; struct ibv_context *ctx = priv->sh->ctx; struct ibv_counter_set_init_attr init = { - .counter_set_id = counter->id}; + .counter_set_id = counter->shared_info.id}; - counter->cs = mlx5_glue->create_counter_set(ctx, &init); - if (!counter->cs) { + counter->dcs_when_free = mlx5_glue->create_counter_set(ctx, &init); + if (!counter->dcs_when_free) { rte_errno = ENOTSUP; return -ENOTSUP; } @@ -105,23 +208,23 @@ flow_verbs_counter_create(struct rte_eth_dev *dev, int ret; memset(&attach, 0, sizeof(attach)); - counter->cs = mlx5_glue->create_counters(ctx, &init); - if (!counter->cs) { + counter->dcs_when_free = mlx5_glue->create_counters(ctx, &init); + if (!counter->dcs_when_free) { rte_errno = ENOTSUP; return -ENOTSUP; } attach.counter_desc = IBV_COUNTER_PACKETS; attach.index = 0; - ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL); + ret = mlx5_glue->attach_counters(counter->dcs_when_free, &attach, NULL); if (!ret) { attach.counter_desc = IBV_COUNTER_BYTES; attach.index = 1; ret = mlx5_glue->attach_counters - (counter->cs, &attach, NULL); + (counter->dcs_when_free, &attach, NULL); } if (ret) { - claim_zero(mlx5_glue->destroy_counters(counter->cs)); - counter->cs = NULL; + claim_zero(mlx5_glue->destroy_counters(counter->dcs_when_free)); + counter->dcs_when_free = NULL; rte_errno = ret; return -ret; } @@ -151,29 +254,20 @@ static uint32_t flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, 0, 0); + struct mlx5_flow_counter_mng *cmng = &priv->sh->cmng; struct mlx5_flow_counter_pool *pool = NULL; - struct mlx5_flow_counter_ext *cnt_ext = NULL; struct mlx5_flow_counter *cnt = NULL; - uint32_t n_valid = rte_atomic16_read(&cont->n_valid); - uint32_t pool_idx; + union mlx5_l3t_data data; + uint32_t n_valid = cmng->n_valid; + uint32_t pool_idx, cnt_idx; uint32_t i; int ret; - if (shared) { - for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { - pool = cont->pools[pool_idx]; - for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { - cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i); - if (cnt_ext->shared && cnt_ext->id == id) { - cnt_ext->ref_cnt++; - return MLX5_MAKE_CNT_IDX(pool_idx, i); - } - } - } - } + if (shared && !mlx5_l3t_get_entry(priv->sh->cnt_id_tbl, id, &data) && + data.dword) + return data.dword; for (pool_idx = 0; pool_idx < n_valid; ++pool_idx) { - pool = cont->pools[pool_idx]; + pool = cmng->pools[pool_idx]; if (!pool) continue; cnt = TAILQ_FIRST(&pool->counters[0]); @@ -184,52 +278,56 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) struct mlx5_flow_counter_pool **pools; uint32_t size; - if (n_valid == cont->n) { + if (n_valid == cmng->n) { /* Resize the container pool array. */ size = sizeof(struct mlx5_flow_counter_pool *) * (n_valid + MLX5_CNT_CONTAINER_RESIZE); - pools = rte_zmalloc(__func__, size, 0); + pools = mlx5_malloc(MLX5_MEM_ZERO, size, 0, + SOCKET_ID_ANY); if (!pools) return 0; if (n_valid) { - memcpy(pools, cont->pools, + memcpy(pools, cmng->pools, sizeof(struct mlx5_flow_counter_pool *) * n_valid); - rte_free(cont->pools); + mlx5_free(cmng->pools); } - cont->pools = pools; - cont->n += MLX5_CNT_CONTAINER_RESIZE; + cmng->pools = pools; + cmng->n += MLX5_CNT_CONTAINER_RESIZE; } /* Allocate memory for new pool*/ - size = sizeof(*pool) + (sizeof(*cnt_ext) + sizeof(*cnt)) * - MLX5_COUNTERS_PER_POOL; - pool = rte_calloc(__func__, 1, size, 0); + size = sizeof(*pool) + sizeof(*cnt) * MLX5_COUNTERS_PER_POOL; + pool = mlx5_malloc(MLX5_MEM_ZERO, size, 0, SOCKET_ID_ANY); if (!pool) return 0; - pool->type |= CNT_POOL_TYPE_EXT; for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) { cnt = MLX5_POOL_GET_CNT(pool, i); TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); } cnt = MLX5_POOL_GET_CNT(pool, 0); - cont->pools[n_valid] = pool; + cmng->pools[n_valid] = pool; pool_idx = n_valid; - rte_atomic16_add(&cont->n_valid, 1); - TAILQ_INSERT_HEAD(&cont->pool_list, pool, next); + cmng->n_valid++; } + TAILQ_REMOVE(&pool->counters[0], cnt, next); i = MLX5_CNT_ARRAY_IDX(pool, cnt); - cnt_ext = MLX5_GET_POOL_CNT_EXT(pool, i); - cnt_ext->id = id; - cnt_ext->shared = shared; - cnt_ext->ref_cnt = 1; - cnt->hits = 0; - cnt->bytes = 0; + cnt_idx = MLX5_MAKE_CNT_IDX(pool_idx, i); + if (shared) { + data.dword = cnt_idx; + if (mlx5_l3t_set_entry(priv->sh->cnt_id_tbl, id, &data)) + return 0; + cnt->shared_info.id = id; + cnt_idx |= MLX5_CNT_SHARED_OFFSET; + } /* Create counter with Verbs. */ - ret = flow_verbs_counter_create(dev, cnt_ext); + ret = flow_verbs_counter_create(dev, cnt); if (!ret) { - TAILQ_REMOVE(&pool->counters[0], cnt, next); - return MLX5_MAKE_CNT_IDX(pool_idx, i); + cnt->dcs_when_active = cnt->dcs_when_free; + cnt->hits = 0; + cnt->bytes = 0; + return cnt_idx; } + TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); /* Some error occurred in Verbs library. */ rte_errno = -ret; return 0; @@ -246,23 +344,22 @@ flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id) static void flow_verbs_counter_release(struct rte_eth_dev *dev, uint32_t counter) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_counter_pool *pool; struct mlx5_flow_counter *cnt; - struct mlx5_flow_counter_ext *cnt_ext; - cnt = flow_verbs_counter_get_by_idx(dev, counter, - &pool); - cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); - if (--cnt_ext->ref_cnt == 0) { + cnt = flow_verbs_counter_get_by_idx(dev, counter, &pool); + if (IS_SHARED_CNT(counter) && + mlx5_l3t_clear_entry(priv->sh->cnt_id_tbl, cnt->shared_info.id)) + return; #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) - claim_zero(mlx5_glue->destroy_counter_set(cnt_ext->cs)); - cnt_ext->cs = NULL; + claim_zero(mlx5_glue->destroy_counter_set + ((struct ibv_counter_set *)cnt->dcs_when_active)); #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) - claim_zero(mlx5_glue->destroy_counters(cnt_ext->cs)); - cnt_ext->cs = NULL; + claim_zero(mlx5_glue->destroy_counters + ((struct ibv_counters *)cnt->dcs_when_active)); #endif - TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); - } + TAILQ_INSERT_HEAD(&pool->counters[0], cnt, next); } /** @@ -282,13 +379,12 @@ flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, struct mlx5_flow_counter_pool *pool; struct mlx5_flow_counter *cnt = flow_verbs_counter_get_by_idx (dev, flow->counter, &pool); - struct mlx5_flow_counter_ext *cnt_ext = MLX5_CNT_TO_CNT_EXT - (pool, cnt); struct rte_flow_query_count *qc = data; uint64_t counters[2] = {0, 0}; #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) struct ibv_query_counter_set_attr query_cs_attr = { - .cs = cnt_ext->cs, + .dcs_when_free = (struct ibv_counter_set *) + cnt->dcs_when_active, .query_flags = IBV_COUNTER_SET_FORCE_UPDATE, }; struct ibv_counter_set_data query_out = { @@ -299,7 +395,7 @@ flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused, &query_out); #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) int err = mlx5_glue->query_counters - (cnt_ext->cs, counters, + ((struct ibv_counters *)cnt->dcs_when_active, counters, RTE_DIM(counters), IBV_READ_COUNTERS_ATTR_PREFER_CACHED); #endif @@ -1086,7 +1182,6 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) struct mlx5_flow_counter_pool *pool; struct mlx5_flow_counter *cnt = NULL; - struct mlx5_flow_counter_ext *cnt_ext; unsigned int size = sizeof(struct ibv_flow_spec_counter_action); struct ibv_flow_spec_counter_action counter = { .type = IBV_FLOW_SPEC_ACTION_COUNT, @@ -1106,13 +1201,12 @@ flow_verbs_translate_action_count(struct mlx5_flow *dev_flow, } #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); - cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); - counter.counter_set_handle = cnt_ext->cs->handle; + counter.counter_set_handle = + ((struct ibv_counter_set *)cnt->dcs_when_active)->handle; flow_verbs_spec_add(&dev_flow->verbs, &counter, size); #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) cnt = flow_verbs_counter_get_by_idx(dev, flow->counter, &pool); - cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt); - counter.counters = cnt_ext->cs; + counter.counters = (struct ibv_counters *)cnt->dcs_when_active; flow_verbs_spec_add(&dev_flow->verbs, &counter, size); #endif return 0; @@ -1208,10 +1302,11 @@ flow_verbs_validate(struct rte_eth_dev *dev, } break; case RTE_FLOW_ITEM_TYPE_IPV4: - ret = mlx5_flow_validate_item_ipv4(items, item_flags, - last_item, - ether_type, NULL, - error); + ret = mlx5_flow_validate_item_ipv4 + (items, item_flags, + last_item, ether_type, NULL, + MLX5_ITEM_RANGE_NOT_ACCEPTED, + error); if (ret < 0) return ret; last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 : @@ -1723,25 +1818,25 @@ flow_verbs_translate(struct rte_eth_dev *dev, case RTE_FLOW_ITEM_TYPE_VXLAN: flow_verbs_translate_item_vxlan(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_VXLAN; break; case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: flow_verbs_translate_item_vxlan_gpe(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_VXLAN_GPE; break; case RTE_FLOW_ITEM_TYPE_GRE: flow_verbs_translate_item_gre(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_GRE; break; case RTE_FLOW_ITEM_TYPE_MPLS: flow_verbs_translate_item_mpls(dev_flow, items, item_flags); - subpriority = MLX5_PRIORITY_MAP_L2; + subpriority = MLX5_TUNNEL_PRIO_GET(rss_desc); item_flags |= MLX5_FLOW_LAYER_MPLS; break; default: @@ -1785,7 +1880,7 @@ flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow) /* hrxq is union, don't touch it only the flag is set. */ if (handle->rix_hrxq) { if (handle->fate_action == MLX5_FLOW_FATE_DROP) { - mlx5_hrxq_drop_release(dev); + mlx5_drop_action_destroy(dev); handle->rix_hrxq = 0; } else if (handle->fate_action == MLX5_FLOW_FATE_QUEUE) { @@ -1861,7 +1956,7 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx]; handle = dev_flow->handle; if (handle->fate_action == MLX5_FLOW_FATE_DROP) { - hrxq = mlx5_hrxq_drop_new(dev); + hrxq = mlx5_drop_action_create(dev); if (!hrxq) { rte_flow_error_set (error, errno, @@ -1877,20 +1972,22 @@ flow_verbs_apply(struct rte_eth_dev *dev, struct rte_flow *flow, MLX5_ASSERT(rss_desc->queue_num); hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, - rss_desc->queue_num); + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + rss_desc->queue, + rss_desc->queue_num); if (!hrxq_idx) - hrxq_idx = mlx5_hrxq_new(dev, rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, - rss_desc->queue_num, - !!(handle->layers & - MLX5_FLOW_LAYER_TUNNEL)); + hrxq_idx = mlx5_hrxq_new + (dev, rss_desc->key, + MLX5_RSS_HASH_KEY_LEN, + dev_flow->hash_fields, + rss_desc->queue, + rss_desc->queue_num, + !!(handle->layers & + MLX5_FLOW_LAYER_TUNNEL), + false); hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], - hrxq_idx); + hrxq_idx); if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -1929,7 +2026,7 @@ error: /* hrxq is union, don't touch it only the flag is set. */ if (handle->rix_hrxq) { if (handle->fate_action == MLX5_FLOW_FATE_DROP) { - mlx5_hrxq_drop_release(dev); + mlx5_drop_action_destroy(dev); handle->rix_hrxq = 0; } else if (handle->fate_action == MLX5_FLOW_FATE_QUEUE) {