X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5.c;h=8b7629ba6f9fb892a3c12ab3fba02031a85022ea;hb=5bc38358b544766ac69ab7ab7c9a2edcbb98b171;hp=7e13b38ee003984eff7b8fa33771f77a43a0de0a;hpb=2d2cef5d4ff4464c91a8ebc53c03455edeae29b5;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 7e13b38ee0..8b7629ba6f 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -28,7 +27,6 @@ #include #include #include -#include #include #include "mlx5_defs.h" @@ -43,6 +41,11 @@ #include "mlx5_flow_os.h" #include "rte_pmd_mlx5.h" +#define MLX5_ETH_DRIVER_NAME mlx5_eth + +/* Driver type key for new device global syntax. */ +#define MLX5_DRIVER_KEY "driver" + /* Device parameter to enable RX completion queue compression. */ #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en" @@ -175,6 +178,12 @@ /* Decap will be used or not. */ #define MLX5_DECAP_EN "decap_en" +/* Device parameter to configure allow or prevent duplicate rules pattern. */ +#define MLX5_ALLOW_DUPLICATE_PATTERN "allow_duplicate_pattern" + +/* Device parameter to configure implicit registration of mempool memory. */ +#define MLX5_MR_MEMPOOL_REG_EN "mr_mempool_reg_en" + /* Shared memory between primary and secondary processes. */ struct mlx5_shared_data *mlx5_shared_data; @@ -214,7 +223,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { .grow_trunk = 3, .grow_shift = 2, .need_lock = 1, - .release_mem_en = 1, + .release_mem_en = 0, + .per_core_cache = (1 << 16), .malloc = mlx5_malloc, .free = mlx5_free, .type = "mlx5_tag_ipool", @@ -322,7 +332,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { .grow_trunk = 3, .grow_shift = 2, .need_lock = 1, - .release_mem_en = 1, + .release_mem_en = 0, + .per_core_cache = 1 << 19, .malloc = mlx5_malloc, .free = mlx5_free, .type = "mlx5_flow_handle_ipool", @@ -352,13 +363,27 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { .free = mlx5_free, .type = "mlx5_shared_action_rss", }, + [MLX5_IPOOL_MTR_POLICY] = { + /** + * The ipool index should grow continually from small to big, + * for policy idx, so not set grow_trunk to avoid policy index + * not jump continually. + */ + .size = sizeof(struct mlx5_flow_meter_sub_policy), + .trunk_size = 64, + .need_lock = 1, + .release_mem_en = 1, + .malloc = mlx5_malloc, + .free = mlx5_free, + .type = "mlx5_meter_policy_ipool", + }, }; #define MLX5_FLOW_MIN_ID_POOL_SIZE 512 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16 -#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096 +#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024 /** * Decide whether representor ID is a HPF(host PF) port on BF2. @@ -380,6 +405,24 @@ mlx5_is_hpf(struct rte_eth_dev *dev) MLX5_REPRESENTOR_REPR(-1) == repr; } +/** + * Decide whether representor ID is a SF port representor. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * Non-zero if HPF, otherwise 0. + */ +bool +mlx5_is_sf_repr(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); + + return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_SF; +} + /** * Initialize the ASO aging management structure. * @@ -571,27 +614,23 @@ mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) * Pointer to mlx5_dev_ctx_shared object to free */ int -mlx5_aso_flow_mtrs_mng_init(struct mlx5_priv *priv) +mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) { - if (!priv->mtr_idx_tbl) { - priv->mtr_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); - if (!priv->mtr_idx_tbl) { - DRV_LOG(ERR, "fail to create meter lookup table."); - rte_errno = ENOMEM; - return -ENOMEM; - } - } - if (!priv->sh->mtrmng) { - priv->sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, - sizeof(*priv->sh->mtrmng), + if (!sh->mtrmng) { + sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*sh->mtrmng), RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); - if (!priv->sh->mtrmng) { - DRV_LOG(ERR, "mlx5_aso_mtr_pools_mng allocation was failed."); + if (!sh->mtrmng) { + DRV_LOG(ERR, + "meter management allocation was failed."); rte_errno = ENOMEM; return -ENOMEM; } - rte_spinlock_init(&priv->sh->mtrmng->mtrsl); - LIST_INIT(&priv->sh->mtrmng->meters); + if (sh->meter_aso_en) { + rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl); + LIST_INIT(&sh->mtrmng->pools_mng.meters); + } + sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID; } return 0; } @@ -607,31 +646,34 @@ static void mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) { struct mlx5_aso_mtr_pool *mtr_pool; - struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng; + struct mlx5_flow_mtr_mng *mtrmng = sh->mtrmng; uint32_t idx; #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO struct mlx5_aso_mtr *aso_mtr; int i; #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ - mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); - idx = mtrmng->n_valid; - while (idx--) { - mtr_pool = mtrmng->pools[idx]; + if (sh->meter_aso_en) { + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); + idx = mtrmng->pools_mng.n_valid; + while (idx--) { + mtr_pool = mtrmng->pools_mng.pools[idx]; #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO - for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { - aso_mtr = &mtr_pool->mtrs[i]; - if (aso_mtr->fm.meter_action) - claim_zero(mlx5_glue->destroy_flow_action - (aso_mtr->fm.meter_action)); - } + for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { + aso_mtr = &mtr_pool->mtrs[i]; + if (aso_mtr->fm.meter_action) + claim_zero + (mlx5_glue->destroy_flow_action + (aso_mtr->fm.meter_action)); + } #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */ - claim_zero(mlx5_devx_cmd_destroy + claim_zero(mlx5_devx_cmd_destroy (mtr_pool->devx_obj)); - mtrmng->n_valid--; - mlx5_free(mtr_pool); + mtrmng->pools_mng.n_valid--; + mlx5_free(mtr_pool); + } + mlx5_free(sh->mtrmng->pools_mng.pools); } - mlx5_free(sh->mtrmng->pools); mlx5_free(sh->mtrmng); sh->mtrmng = NULL; } @@ -647,12 +689,104 @@ mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) age_info = &sh->port[i].age_info; if (!MLX5_AGE_GET(age_info, MLX5_AGE_EVENT_NEW)) continue; - if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) + MLX5_AGE_UNSET(age_info, MLX5_AGE_EVENT_NEW); + if (MLX5_AGE_GET(age_info, MLX5_AGE_TRIGGER)) { + MLX5_AGE_UNSET(age_info, MLX5_AGE_TRIGGER); rte_eth_dev_callback_process (&rte_eth_devices[sh->port[i].devx_ih_port_id], RTE_ETH_EVENT_FLOW_AGED, NULL); - age_info->flags = 0; + } + } +} + +/* + * Initialize the ASO connection tracking structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh) +{ + int err; + + if (sh->ct_mng) + return 0; + sh->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->ct_mng), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + if (!sh->ct_mng) { + DRV_LOG(ERR, "ASO CT management allocation failed."); + rte_errno = ENOMEM; + return -rte_errno; + } + err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_CONNECTION_TRACKING); + if (err) { + mlx5_free(sh->ct_mng); + /* rte_errno should be extracted from the failure. */ + rte_errno = EINVAL; + return -rte_errno; + } + rte_spinlock_init(&sh->ct_mng->ct_sl); + rte_rwlock_init(&sh->ct_mng->resize_rwl); + LIST_INIT(&sh->ct_mng->free_cts); + return 0; +} + +/* + * Close and release all the resources of the + * ASO connection tracking management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free. + */ +static void +mlx5_flow_aso_ct_mng_close(struct mlx5_dev_ctx_shared *sh) +{ + struct mlx5_aso_ct_pools_mng *mng = sh->ct_mng; + struct mlx5_aso_ct_pool *ct_pool; + struct mlx5_aso_ct_action *ct; + uint32_t idx; + uint32_t val; + uint32_t cnt; + int i; + + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_CONNECTION_TRACKING); + idx = mng->next; + while (idx--) { + cnt = 0; + ct_pool = mng->pools[idx]; + for (i = 0; i < MLX5_ASO_CT_ACTIONS_PER_POOL; i++) { + ct = &ct_pool->actions[i]; + val = __atomic_fetch_sub(&ct->refcnt, 1, + __ATOMIC_RELAXED); + MLX5_ASSERT(val == 1); + if (val > 1) + cnt++; +#ifdef HAVE_MLX5_DR_ACTION_ASO_CT + if (ct->dr_action_orig) + claim_zero(mlx5_glue->destroy_flow_action + (ct->dr_action_orig)); + if (ct->dr_action_rply) + claim_zero(mlx5_glue->destroy_flow_action + (ct->dr_action_rply)); +#endif + } + claim_zero(mlx5_devx_cmd_destroy(ct_pool->devx_obj)); + if (cnt) { + DRV_LOG(DEBUG, "%u ASO CT objects are being used in the pool %u", + cnt, i); + } + mlx5_free(ct_pool); + /* in case of failure. */ + mng->next--; } + mlx5_free(mng->pools); + mlx5_free(mng); + /* Management structure must be cleared to 0s during allocation. */ + sh->ct_mng = NULL; } /** @@ -660,7 +794,7 @@ mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) * * @param[in] sh * Pointer to mlx5_dev_ctx_shared object. - * @param[in] sh + * @param[in] config * Pointer to user dev config. */ static void @@ -685,12 +819,17 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh, MLX5_FLOW_HANDLE_VERBS_SIZE; break; } - if (config->reclaim_mode) + if (config->reclaim_mode) { cfg.release_mem_en = 1; + cfg.per_core_cache = 0; + } else { + cfg.release_mem_en = 0; + } sh->ipool[i] = mlx5_ipool_create(&cfg); } } + /** * Release the flow resources' indexed mempool. * @@ -704,6 +843,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh) for (i = 0; i < MLX5_IPOOL_MAX; ++i) mlx5_ipool_destroy(sh->ipool[i]); + for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i) + if (sh->mdh_ipools[i]) + mlx5_ipool_destroy(sh->mdh_ipools[i]); } /* @@ -811,6 +953,36 @@ mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) prf->obj = NULL; } +uint32_t +mlx5_get_supported_sw_parsing_offloads(const struct mlx5_hca_attr *attr) +{ + uint32_t sw_parsing_offloads = 0; + + if (attr->swp) { + sw_parsing_offloads |= MLX5_SW_PARSING_CAP; + if (attr->swp_csum) + sw_parsing_offloads |= MLX5_SW_PARSING_CSUM_CAP; + + if (attr->swp_lso) + sw_parsing_offloads |= MLX5_SW_PARSING_TSO_CAP; + } + return sw_parsing_offloads; +} + +uint32_t +mlx5_get_supported_tunneling_offloads(const struct mlx5_hca_attr *attr) +{ + uint32_t tn_offloads = 0; + + if (attr->tunnel_stateless_vxlan) + tn_offloads |= MLX5_TUNNELED_OFFLOADS_VXLAN_CAP; + if (attr->tunnel_stateless_gre) + tn_offloads |= MLX5_TUNNELED_OFFLOADS_GRE_CAP; + if (attr->tunnel_stateless_geneve_rx) + tn_offloads |= MLX5_TUNNELED_OFFLOADS_GENEVE_CAP; + return tn_offloads; +} + /* * Allocate Rx and Tx UARs in robust fashion. * This routine handles the following UAR allocation issues: @@ -826,7 +998,7 @@ mlx5_flex_parser_ecpri_release(struct rte_eth_dev *dev) */ static int mlx5_alloc_rxtx_uars(struct mlx5_dev_ctx_shared *sh, - const struct mlx5_dev_config *config) + const struct mlx5_common_dev_config *config) { uint32_t uar_mapping, retry; int err = 0; @@ -949,6 +1121,141 @@ exit: return err; } +/** + * Unregister the mempool from the protection domain. + * + * @param sh + * Pointer to the device shared context. + * @param mp + * Mempool being unregistered. + */ +static void +mlx5_dev_ctx_shared_mempool_unregister(struct mlx5_dev_ctx_shared *sh, + struct rte_mempool *mp) +{ + struct mlx5_mp_id mp_id; + + mlx5_mp_id_init(&mp_id, 0); + if (mlx5_mr_mempool_unregister(&sh->share_cache, mp, &mp_id) < 0) + DRV_LOG(WARNING, "Failed to unregister mempool %s for PD %p: %s", + mp->name, sh->pd, rte_strerror(rte_errno)); +} + +/** + * rte_mempool_walk() callback to register mempools + * for the protection domain. + * + * @param mp + * The mempool being walked. + * @param arg + * Pointer to the device shared context. + */ +static void +mlx5_dev_ctx_shared_mempool_register_cb(struct rte_mempool *mp, void *arg) +{ + struct mlx5_dev_ctx_shared *sh = arg; + struct mlx5_mp_id mp_id; + int ret; + + mlx5_mp_id_init(&mp_id, 0); + ret = mlx5_mr_mempool_register(&sh->share_cache, sh->pd, mp, &mp_id); + if (ret < 0 && rte_errno != EEXIST) + DRV_LOG(ERR, "Failed to register existing mempool %s for PD %p: %s", + mp->name, sh->pd, rte_strerror(rte_errno)); +} + +/** + * rte_mempool_walk() callback to unregister mempools + * from the protection domain. + * + * @param mp + * The mempool being walked. + * @param arg + * Pointer to the device shared context. + */ +static void +mlx5_dev_ctx_shared_mempool_unregister_cb(struct rte_mempool *mp, void *arg) +{ + mlx5_dev_ctx_shared_mempool_unregister + ((struct mlx5_dev_ctx_shared *)arg, mp); +} + +/** + * Mempool life cycle callback for Ethernet devices. + * + * @param event + * Mempool life cycle event. + * @param mp + * Associated mempool. + * @param arg + * Pointer to a device shared context. + */ +static void +mlx5_dev_ctx_shared_mempool_event_cb(enum rte_mempool_event event, + struct rte_mempool *mp, void *arg) +{ + struct mlx5_dev_ctx_shared *sh = arg; + struct mlx5_mp_id mp_id; + + switch (event) { + case RTE_MEMPOOL_EVENT_READY: + mlx5_mp_id_init(&mp_id, 0); + if (mlx5_mr_mempool_register(&sh->share_cache, sh->pd, mp, + &mp_id) < 0) + DRV_LOG(ERR, "Failed to register new mempool %s for PD %p: %s", + mp->name, sh->pd, rte_strerror(rte_errno)); + break; + case RTE_MEMPOOL_EVENT_DESTROY: + mlx5_dev_ctx_shared_mempool_unregister(sh, mp); + break; + } +} + +/** + * Callback used when implicit mempool registration is disabled + * in order to track Rx mempool destruction. + * + * @param event + * Mempool life cycle event. + * @param mp + * An Rx mempool registered explicitly when the port is started. + * @param arg + * Pointer to a device shared context. + */ +static void +mlx5_dev_ctx_shared_rx_mempool_event_cb(enum rte_mempool_event event, + struct rte_mempool *mp, void *arg) +{ + struct mlx5_dev_ctx_shared *sh = arg; + + if (event == RTE_MEMPOOL_EVENT_DESTROY) + mlx5_dev_ctx_shared_mempool_unregister(sh, mp); +} + +int +mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_ctx_shared *sh = priv->sh; + int ret; + + /* Check if we only need to track Rx mempool destruction. */ + if (!sh->cdev->config.mr_mempool_reg_en) { + ret = rte_mempool_event_callback_register + (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); + return ret == 0 || rte_errno == EEXIST ? 0 : ret; + } + /* Callback for this shared context may be already registered. */ + ret = rte_mempool_event_callback_register + (mlx5_dev_ctx_shared_mempool_event_cb, sh); + if (ret != 0 && rte_errno != EEXIST) + return ret; + /* Register mempools only once for this shared context. */ + if (ret == 0) + rte_mempool_walk(mlx5_dev_ctx_shared_mempool_register_cb, sh); + return 0; +} + /** * Allocate shared device context. If there is multiport device the * master and representors will share this context, if there is single @@ -985,7 +1292,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, /* Search for IB context by device name. */ LIST_FOREACH(sh, &mlx5_dev_ctx_list, next) { if (!strcmp(sh->ibdev_name, - mlx5_os_get_dev_device_name(spawn->phys_dev))) { + mlx5_os_get_ctx_device_name(spawn->ctx))) { sh->refcnt++; goto exit; } @@ -1002,11 +1309,13 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, rte_errno = ENOMEM; goto exit; } + pthread_mutex_init(&sh->txpp.mutex, NULL); + sh->numa_node = spawn->cdev->dev->numa_node; + sh->cdev = spawn->cdev; + sh->devx = sh->cdev->config.devx; + sh->ctx = spawn->ctx; if (spawn->bond_info) sh->bond = *spawn->bond_info; - err = mlx5_os_open_device(spawn, config, sh); - if (!sh->ctx) - goto error; err = mlx5_os_get_dev_attr(sh->ctx, &sh->device_attr); if (err) { DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed"); @@ -1014,6 +1323,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, } sh->refcnt = 1; sh->max_port = spawn->max_port; + sh->reclaim_mode = config->reclaim_mode; strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx), sizeof(sh->ibdev_name) - 1); strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx), @@ -1052,7 +1362,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, err = ENOMEM; goto error; } - err = mlx5_alloc_rxtx_uars(sh, config); + err = mlx5_alloc_rxtx_uars(sh, &sh->cdev->config); if (err) goto error; MLX5_ASSERT(sh->tx_uar); @@ -1078,7 +1388,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, */ err = mlx5_mr_btree_init(&sh->share_cache.cache, MLX5_MR_BTREE_CACHE_N * 2, - spawn->pci_dev->device.numa_node); + sh->numa_node); if (err) { err = rte_errno; goto error; @@ -1086,11 +1396,6 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, mlx5_os_set_reg_mr_cb(&sh->share_cache.reg_mr_cb, &sh->share_cache.dereg_mr_cb); mlx5_os_dev_shared_handler_install(sh); - sh->cnt_id_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); - if (!sh->cnt_id_tbl) { - err = rte_errno; - goto error; - } if (LIST_EMPTY(&mlx5_dev_ctx_list)) { err = mlx5_flow_os_init_workspace_once(); if (err) @@ -1114,8 +1419,8 @@ error: pthread_mutex_destroy(&sh->txpp.mutex); pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex); MLX5_ASSERT(sh); - if (sh->cnt_id_tbl) - mlx5_l3t_destroy(sh->cnt_id_tbl); + if (sh->share_cache.cache.table) + mlx5_mr_btree_free(&sh->share_cache.cache); if (sh->tis) claim_zero(mlx5_devx_cmd_destroy(sh->tis)); if (sh->td) @@ -1144,6 +1449,8 @@ error: void mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) { + int ret; + pthread_mutex_lock(&mlx5_dev_ctx_list_mutex); #ifdef RTE_LIBRTE_MLX5_DEBUG /* Check the object presence in the list. */ @@ -1164,6 +1471,15 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); if (--sh->refcnt) goto exit; + /* Stop watching for mempool events and unregister all mempools. */ + ret = rte_mempool_event_callback_unregister + (mlx5_dev_ctx_shared_mempool_event_cb, sh); + if (ret < 0 && rte_errno == ENOENT) + ret = rte_mempool_event_callback_unregister + (mlx5_dev_ctx_shared_rx_mempool_event_cb, sh); + if (ret == 0) + rte_mempool_walk(mlx5_dev_ctx_shared_mempool_unregister_cb, + sh); /* Remove from memory callback device list. */ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock); LIST_REMOVE(sh, mem_event_cb); @@ -1189,10 +1505,6 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) mlx5_aso_flow_mtrs_mng_close(sh); mlx5_flow_ipool_destroy(sh); mlx5_os_dev_shared_handler_uninstall(sh); - if (sh->cnt_id_tbl) { - mlx5_l3t_destroy(sh->cnt_id_tbl); - sh->cnt_id_tbl = NULL; - } if (sh->tx_uar) { mlx5_glue->devx_free_uar(sh->tx_uar); sh->tx_uar = NULL; @@ -1229,6 +1541,7 @@ mlx5_free_table_hash_list(struct mlx5_priv *priv) if (!sh->flow_tbls) return; mlx5_hlist_destroy(sh->flow_tbls); + sh->flow_tbls = NULL; } /** @@ -1248,20 +1561,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) /* Tables are only used in DV and DR modes. */ #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) struct mlx5_dev_ctx_shared *sh = priv->sh; - char s[MLX5_HLIST_NAMESIZE]; + char s[MLX5_NAME_SIZE]; MLX5_ASSERT(sh); snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, - 0, 0, flow_dv_tbl_create_cb, + false, true, sh, + flow_dv_tbl_create_cb, flow_dv_tbl_match_cb, - flow_dv_tbl_remove_cb); + flow_dv_tbl_remove_cb, + flow_dv_tbl_clone_cb, + flow_dv_tbl_clone_free_cb); if (!sh->flow_tbls) { DRV_LOG(ERR, "flow tables with hash creation failed."); err = ENOMEM; return err; } - sh->flow_tbls->ctx = sh; #ifndef HAVE_MLX5DV_DR struct rte_flow_error error; struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; @@ -1348,6 +1663,7 @@ mlx5_proc_priv_init(struct rte_eth_dev *dev) struct mlx5_proc_priv *ppriv; size_t ppriv_size; + mlx5_proc_priv_uninit(dev); /* * UAR register table follows the process private structure. BlueFlame * registers for Tx queues are stored in the table. @@ -1420,7 +1736,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) * If all the flows are already flushed in the device stop stage, * then this will return directly without any action. */ - mlx5_flow_list_flush(dev, &priv->flows, true); + mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); mlx5_action_handle_flush(dev); mlx5_flow_meter_flush(dev, NULL); /* Prevent crashes when queues are still in use. */ @@ -1439,6 +1755,11 @@ mlx5_dev_close(struct rte_eth_dev *dev) priv->rxqs_n = 0; priv->rxqs = NULL; } + if (priv->representor) { + /* Each representor has a dedicated interrupts handler */ + mlx5_free(dev->intr_handle); + dev->intr_handle = NULL; + } if (priv->txqs != NULL) { /* XXX race condition if mlx5_tx_burst() is still running. */ rte_delay_us_sleep(1000); @@ -1457,6 +1778,8 @@ mlx5_dev_close(struct rte_eth_dev *dev) if (priv->mreg_cp_tbl) mlx5_hlist_destroy(priv->mreg_cp_tbl); mlx5_mprq_free_mp(dev); + if (priv->sh->ct_mng) + mlx5_flow_aso_ct_mng_close(priv->sh); mlx5_os_free_shared_dr(priv); if (priv->rss_conf.rss_key != NULL) mlx5_free(priv->rss_conf.rss_key); @@ -1498,7 +1821,8 @@ mlx5_dev_close(struct rte_eth_dev *dev) if (ret) DRV_LOG(WARNING, "port %u some flows still remain", dev->data->port_id); - mlx5_cache_list_destroy(&priv->hrxqs); + if (priv->hrxqs) + mlx5_list_destroy(priv->hrxqs); /* * Free the shared context in last turn, because the cleanup * routines above may use some shared fields, like @@ -1510,7 +1834,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) unsigned int c = 0; uint16_t port_id; - MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { + MLX5_ETH_FOREACH_DEV(port_id, dev->device) { struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private; @@ -1600,6 +1924,7 @@ const struct eth_dev_ops mlx5_dev_ops = { .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, + .get_monitor_addr = mlx5_get_monitor_addr, }; /* Available operations from secondary process. */ @@ -1611,6 +1936,7 @@ const struct eth_dev_ops mlx5_dev_sec_ops = { .xstats_get_names = mlx5_xstats_get_names, .fw_version_get = mlx5_fw_version_get, .dev_infos_get = mlx5_dev_infos_get, + .representor_info_get = mlx5_representor_info_get, .read_clock = mlx5_txpp_read_clock, .rx_queue_start = mlx5_rx_queue_start, .rx_queue_stop = mlx5_rx_queue_stop, @@ -1644,6 +1970,7 @@ const struct eth_dev_ops mlx5_dev_ops_isolate = { .xstats_get_names = mlx5_xstats_get_names, .fw_version_get = mlx5_fw_version_get, .dev_infos_get = mlx5_dev_infos_get, + .representor_info_get = mlx5_representor_info_get, .read_clock = mlx5_txpp_read_clock, .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, .vlan_filter_set = mlx5_vlan_filter_set, @@ -1684,6 +2011,7 @@ const struct eth_dev_ops mlx5_dev_ops_isolate = { .hairpin_queue_peer_update = mlx5_hairpin_queue_peer_update, .hairpin_queue_peer_bind = mlx5_hairpin_queue_peer_bind, .hairpin_queue_peer_unbind = mlx5_hairpin_queue_peer_unbind, + .get_monitor_addr = mlx5_get_monitor_addr, }; /** @@ -1707,7 +2035,10 @@ mlx5_args_check(const char *key, const char *val, void *opaque) signed long tmp; /* No-op, port representors are processed in mlx5_dev_spawn(). */ - if (!strcmp(MLX5_REPRESENTOR, key)) + if (!strcmp(MLX5_DRIVER_KEY, key) || !strcmp(MLX5_REPRESENTOR, key) || + !strcmp(MLX5_SYS_MEM_EN, key) || !strcmp(MLX5_TX_DB_NC, key) || + !strcmp(MLX5_MR_MEMPOOL_REG_EN, key) || + !strcmp(MLX5_MR_EXT_MEMSEG_EN, key)) return 0; errno = 0; tmp = strtol(val, NULL, 0); @@ -1760,16 +2091,6 @@ mlx5_args_check(const char *key, const char *val, void *opaque) DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); } else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) { config->mps = !!tmp; - } else if (strcmp(MLX5_TX_DB_NC, key) == 0) { - if (tmp != MLX5_TXDB_CACHED && - tmp != MLX5_TXDB_NCACHED && - tmp != MLX5_TXDB_HEURISTIC) { - DRV_LOG(ERR, "invalid Tx doorbell " - "mapping parameter"); - rte_errno = EINVAL; - return -rte_errno; - } - config->dbnc = tmp; } else if (strcmp(MLX5_TXQ_MPW_HDR_DSEG_EN, key) == 0) { DRV_LOG(WARNING, "%s: deprecated parameter, ignored", key); } else if (strcmp(MLX5_TXQ_MAX_INLINE_LEN, key) == 0) { @@ -1813,13 +2134,11 @@ mlx5_args_check(const char *key, const char *val, void *opaque) config->dv_miss_info = 1; } else if (strcmp(MLX5_LACP_BY_USER, key) == 0) { config->lacp_by_user = !!tmp; - } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) { - config->mr_ext_memseg_en = !!tmp; } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) { config->max_dump_files_num = tmp; } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) { config->lro.timeout = tmp; - } else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) { + } else if (strcmp(RTE_DEVARGS_KEY_CLASS, key) == 0) { DRV_LOG(DEBUG, "class argument is %s.", val); } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) { config->log_hp_size = tmp; @@ -1832,10 +2151,10 @@ mlx5_args_check(const char *key, const char *val, void *opaque) return -rte_errno; } config->reclaim_mode = tmp; - } else if (strcmp(MLX5_SYS_MEM_EN, key) == 0) { - config->sys_mem_en = !!tmp; } else if (strcmp(MLX5_DECAP_EN, key) == 0) { config->decap_en = !!tmp; + } else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) { + config->allow_duplicate_pattern = !!tmp; } else { DRV_LOG(WARNING, "%s: unknown parameter", key); rte_errno = EINVAL; @@ -1859,6 +2178,7 @@ int mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) { const char **params = (const char *[]){ + MLX5_DRIVER_KEY, MLX5_RXQ_CQE_COMP_EN, MLX5_RXQ_PKT_PAD_EN, MLX5_RX_MPRQ_EN, @@ -1890,11 +2210,13 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs) MLX5_REPRESENTOR, MLX5_MAX_DUMP_FILES_NUM, MLX5_LRO_TIMEOUT_USEC, - MLX5_CLASS_ARG_NAME, + RTE_DEVARGS_KEY_CLASS, MLX5_HP_BUF_SIZE, MLX5_RECLAIM_MEM, MLX5_SYS_MEM_EN, MLX5_DECAP_EN, + MLX5_ALLOW_DUPLICATE_PATTERN, + MLX5_MR_MEMPOOL_REG_EN, NULL, }; struct rte_kvargs *kvlist; @@ -1947,18 +2269,20 @@ mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, { if (config->txq_inline_min != MLX5_ARG_UNSET) { /* Application defines size of inlined data explicitly. */ - switch (spawn->pci_dev->id.device_id) { - case PCI_DEVICE_ID_MELLANOX_CONNECTX4: - case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: - if (config->txq_inline_min < - (int)MLX5_INLINE_HSIZE_L2) { - DRV_LOG(DEBUG, - "txq_inline_mix aligned to minimal" - " ConnectX-4 required value %d", - (int)MLX5_INLINE_HSIZE_L2); - config->txq_inline_min = MLX5_INLINE_HSIZE_L2; + if (spawn->pci_dev != NULL) { + switch (spawn->pci_dev->id.device_id) { + case PCI_DEVICE_ID_MELLANOX_CONNECTX4: + case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: + if (config->txq_inline_min < + (int)MLX5_INLINE_HSIZE_L2) { + DRV_LOG(DEBUG, + "txq_inline_mix aligned to minimal ConnectX-4 required value %d", + (int)MLX5_INLINE_HSIZE_L2); + config->txq_inline_min = + MLX5_INLINE_HSIZE_L2; + } + break; } - break; } goto exit; } @@ -2012,6 +2336,10 @@ mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, } } } + if (spawn->pci_dev == NULL) { + config->txq_inline_min = MLX5_INLINE_HSIZE_NONE; + goto exit; + } /* * We get here if we are unable to deduce * inline data size with DevX. Try PCI ID @@ -2135,7 +2463,8 @@ rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n) */ int mlx5_dev_check_sibling_config(struct mlx5_priv *priv, - struct mlx5_dev_config *config) + struct mlx5_dev_config *config, + struct rte_device *dpdk_dev) { struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_dev_config *sh_conf = NULL; @@ -2146,7 +2475,7 @@ mlx5_dev_check_sibling_config(struct mlx5_priv *priv, if (sh->refcnt == 1) return 0; /* Find the device with shared context. */ - MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { + MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) { struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private; @@ -2177,28 +2506,31 @@ mlx5_dev_check_sibling_config(struct mlx5_priv *priv, * * @param[in] port_id * port_id to start looking for device. - * @param[in] pci_dev - * Pointer to the hint PCI device. When device is being probed + * @param[in] odev + * Pointer to the hint device. When device is being probed * the its siblings (master and preceding representors might * not have assigned driver yet (because the mlx5_os_pci_probe() - * is not completed yet, for this case match on hint PCI + * is not completed yet, for this case match on hint * device may be used to detect sibling device. * * @return * port_id of found device, RTE_MAX_ETHPORT if not found. */ uint16_t -mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev) +mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev) { while (port_id < RTE_MAX_ETHPORTS) { struct rte_eth_dev *dev = &rte_eth_devices[port_id]; if (dev->state != RTE_ETH_DEV_UNUSED && dev->device && - (dev->device == &pci_dev->device || + (dev->device == odev || (dev->device->driver && dev->device->driver->name && - !strcmp(dev->device->driver->name, MLX5_PCI_DRIVER_NAME)))) + ((strcmp(dev->device->driver->name, + MLX5_PCI_DRIVER_NAME) == 0) || + (strcmp(dev->device->driver->name, + MLX5_AUXILIARY_DRIVER_NAME) == 0))))) break; port_id++; } @@ -2208,23 +2540,23 @@ mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev) } /** - * DPDK callback to remove a PCI device. + * Callback to remove a device. * - * This function removes all Ethernet devices belong to a given PCI device. + * This function removes all Ethernet devices belong to a given device. * - * @param[in] pci_dev - * Pointer to the PCI device. + * @param[in] cdev + * Pointer to the generic device. * * @return * 0 on success, the function cannot fail. */ -static int -mlx5_pci_remove(struct rte_pci_device *pci_dev) +int +mlx5_net_remove(struct mlx5_common_device *cdev) { uint16_t port_id; int ret = 0; - RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) { + RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) { /* * mlx5_dev_close() is not registered to secondary process, * call the close function explicitly for secondary process. @@ -2315,23 +2647,21 @@ static const struct rte_pci_id mlx5_pci_id_map[] = { } }; -static struct mlx5_pci_driver mlx5_driver = { - .driver_class = MLX5_CLASS_NET, - .pci_driver = { - .driver = { - .name = MLX5_PCI_DRIVER_NAME, - }, - .id_table = mlx5_pci_id_map, - .probe = mlx5_os_pci_probe, - .remove = mlx5_pci_remove, - .dma_map = mlx5_dma_map, - .dma_unmap = mlx5_dma_unmap, - .drv_flags = PCI_DRV_FLAGS, - }, +static struct mlx5_class_driver mlx5_net_driver = { + .drv_class = MLX5_CLASS_ETH, + .name = RTE_STR(MLX5_ETH_DRIVER_NAME), + .id_table = mlx5_pci_id_map, + .probe = mlx5_os_net_probe, + .remove = mlx5_net_remove, + .dma_map = mlx5_net_dma_map, + .dma_unmap = mlx5_net_dma_unmap, + .probe_again = 1, + .intr_lsc = 1, + .intr_rmv = 1, }; /* Initialize driver log type. */ -RTE_LOG_REGISTER(mlx5_logtype, pmd.net.mlx5, NOTICE) +RTE_LOG_REGISTER_DEFAULT(mlx5_logtype, NOTICE) /** * Driver initialization routine. @@ -2345,9 +2675,9 @@ RTE_INIT(rte_mlx5_pmd_init) mlx5_set_cksum_table(); mlx5_set_swp_types_table(); if (mlx5_glue) - mlx5_pci_driver_register(&mlx5_driver); + mlx5_class_driver_register(&mlx5_net_driver); } -RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__); -RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map); -RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib"); +RTE_PMD_EXPORT_NAME(MLX5_ETH_DRIVER_NAME, __COUNTER__); +RTE_PMD_REGISTER_PCI_TABLE(MLX5_ETH_DRIVER_NAME, mlx5_pci_id_map); +RTE_PMD_REGISTER_KMOD_DEP(MLX5_ETH_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");