- } else if (strcmp(MLX5_L3_VXLAN_EN, key) == 0) {
- config->l3_vxlan_en = !!tmp;
- } else if (strcmp(MLX5_VF_NL_EN, key) == 0) {
- config->vf_nl_en = !!tmp;
- } else if (strcmp(MLX5_DV_ESW_EN, key) == 0) {
- config->dv_esw_en = !!tmp;
- } else if (strcmp(MLX5_DV_FLOW_EN, key) == 0) {
- config->dv_flow_en = !!tmp;
- } else if (strcmp(MLX5_DV_XMETA_EN, key) == 0) {
- if (tmp != MLX5_XMETA_MODE_LEGACY &&
- tmp != MLX5_XMETA_MODE_META16 &&
- tmp != MLX5_XMETA_MODE_META32) {
- DRV_LOG(ERR, "invalid extensive "
- "metadata parameter");
- rte_errno = EINVAL;
- return -rte_errno;
- }
- config->dv_xmeta_en = tmp;
- } else if (strcmp(MLX5_MR_EXT_MEMSEG_EN, key) == 0) {
- config->mr_ext_memseg_en = !!tmp;
- } else if (strcmp(MLX5_MAX_DUMP_FILES_NUM, key) == 0) {
- config->max_dump_files_num = tmp;
- } else if (strcmp(MLX5_LRO_TIMEOUT_USEC, key) == 0) {
- config->lro.timeout = tmp;
- } else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
- DRV_LOG(DEBUG, "class argument is %s.", val);
- } else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
- config->log_hp_size = tmp;
- } else {
- DRV_LOG(WARNING, "%s: unknown parameter", key);
- rte_errno = EINVAL;
- return -rte_errno;
- }
- return 0;
-}
-
-/**
- * Parse device parameters.
- *
- * @param config
- * Pointer to device configuration structure.
- * @param devargs
- * Device arguments structure.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
-{
- const char **params = (const char *[]){
- MLX5_RXQ_CQE_COMP_EN,
- MLX5_RXQ_CQE_PAD_EN,
- MLX5_RXQ_PKT_PAD_EN,
- MLX5_RX_MPRQ_EN,
- MLX5_RX_MPRQ_LOG_STRIDE_NUM,
- MLX5_RX_MPRQ_LOG_STRIDE_SIZE,
- MLX5_RX_MPRQ_MAX_MEMCPY_LEN,
- MLX5_RXQS_MIN_MPRQ,
- MLX5_TXQ_INLINE,
- MLX5_TXQ_INLINE_MIN,
- MLX5_TXQ_INLINE_MAX,
- MLX5_TXQ_INLINE_MPW,
- MLX5_TXQS_MIN_INLINE,
- MLX5_TXQS_MAX_VEC,
- MLX5_TXQ_MPW_EN,
- MLX5_TXQ_MPW_HDR_DSEG_EN,
- MLX5_TXQ_MAX_INLINE_LEN,
- MLX5_TX_DB_NC,
- MLX5_TX_VEC_EN,
- MLX5_RX_VEC_EN,
- MLX5_L3_VXLAN_EN,
- MLX5_VF_NL_EN,
- MLX5_DV_ESW_EN,
- MLX5_DV_FLOW_EN,
- MLX5_DV_XMETA_EN,
- MLX5_MR_EXT_MEMSEG_EN,
- MLX5_REPRESENTOR,
- MLX5_MAX_DUMP_FILES_NUM,
- MLX5_LRO_TIMEOUT_USEC,
- MLX5_CLASS_ARG_NAME,
- MLX5_HP_BUF_SIZE,
- NULL,
- };
- struct rte_kvargs *kvlist;
- int ret = 0;
- int i;
-
- if (devargs == NULL)
- return 0;
- /* Following UGLY cast is done to pass checkpatch. */
- kvlist = rte_kvargs_parse(devargs->args, params);
- if (kvlist == NULL) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- /* Process parameters. */
- for (i = 0; (params[i] != NULL); ++i) {
- if (rte_kvargs_count(kvlist, params[i])) {
- ret = rte_kvargs_process(kvlist, params[i],
- mlx5_args_check, config);
- if (ret) {
- rte_errno = EINVAL;
- rte_kvargs_free(kvlist);
- return -rte_errno;
- }
- }
- }
- rte_kvargs_free(kvlist);
- return 0;
-}
-
-static struct rte_pci_driver mlx5_driver;
-
-/**
- * PMD global initialization.
- *
- * Independent from individual device, this function initializes global
- * per-PMD data structures distinguishing primary and secondary processes.
- * Hence, each initialization is called once per a process.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_init_once(void)
-{
- struct mlx5_shared_data *sd;
- struct mlx5_local_data *ld = &mlx5_local_data;
- int ret = 0;
-
- if (mlx5_init_shared_data())
- return -rte_errno;
- sd = mlx5_shared_data;
- MLX5_ASSERT(sd);
- rte_spinlock_lock(&sd->lock);
- switch (rte_eal_process_type()) {
- case RTE_PROC_PRIMARY:
- if (sd->init_done)
- break;
- LIST_INIT(&sd->mem_event_cb_list);
- rte_rwlock_init(&sd->mem_event_rwlock);
- rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
- mlx5_mr_mem_event_cb, NULL);
- ret = mlx5_mp_init_primary(MLX5_MP_NAME,
- mlx5_mp_primary_handle);
- if (ret)
- goto out;
- sd->init_done = true;
- break;
- case RTE_PROC_SECONDARY:
- if (ld->init_done)
- break;
- ret = mlx5_mp_init_secondary(MLX5_MP_NAME,
- mlx5_mp_secondary_handle);
- if (ret)
- goto out;
- ++sd->secondary_cnt;
- ld->init_done = true;
- break;
- default:
- break;
- }
-out:
- rte_spinlock_unlock(&sd->lock);
- return ret;
-}
-
-/**
- * Configures the minimal amount of data to inline into WQE
- * while sending packets.
- *
- * - the txq_inline_min has the maximal priority, if this
- * key is specified in devargs
- * - if DevX is enabled the inline mode is queried from the
- * device (HCA attributes and NIC vport context if needed).
- * - otherwise L2 mode (18 bytes) is assumed for ConnectX-4/4 Lx
- * and none (0 bytes) for other NICs
- *
- * @param spawn
- * Verbs device parameters (name, port, switch_info) to spawn.
- * @param config
- * Device configuration parameters.
- */
-static void
-mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
- struct mlx5_dev_config *config)
-{
- if (config->txq_inline_min != MLX5_ARG_UNSET) {
- /* Application defines size of inlined data explicitly. */
- switch (spawn->pci_dev->id.device_id) {
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
- if (config->txq_inline_min <
- (int)MLX5_INLINE_HSIZE_L2) {
- DRV_LOG(DEBUG,
- "txq_inline_mix aligned to minimal"
- " ConnectX-4 required value %d",
- (int)MLX5_INLINE_HSIZE_L2);
- config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
- }
- break;
- }
- goto exit;
- }
- if (config->hca_attr.eth_net_offloads) {
- /* We have DevX enabled, inline mode queried successfully. */
- switch (config->hca_attr.wqe_inline_mode) {
- case MLX5_CAP_INLINE_MODE_L2:
- /* outer L2 header must be inlined. */
- config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
- goto exit;
- case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
- /* No inline data are required by NIC. */
- config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
- config->hw_vlan_insert =
- config->hca_attr.wqe_vlan_insert;
- DRV_LOG(DEBUG, "Tx VLAN insertion is supported");
- goto exit;
- case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
- /* inline mode is defined by NIC vport context. */
- if (!config->hca_attr.eth_virt)
- break;
- switch (config->hca_attr.vport_inline_mode) {
- case MLX5_INLINE_MODE_NONE:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_NONE;
- goto exit;
- case MLX5_INLINE_MODE_L2:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_L2;
- goto exit;
- case MLX5_INLINE_MODE_IP:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_L3;
- goto exit;
- case MLX5_INLINE_MODE_TCP_UDP:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_L4;
- goto exit;
- case MLX5_INLINE_MODE_INNER_L2:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_INNER_L2;
- goto exit;
- case MLX5_INLINE_MODE_INNER_IP:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_INNER_L3;
- goto exit;
- case MLX5_INLINE_MODE_INNER_TCP_UDP:
- config->txq_inline_min =
- MLX5_INLINE_HSIZE_INNER_L4;
- goto exit;
- }
- }
- }
- /*
- * We get here if we are unable to deduce
- * inline data size with DevX. Try PCI ID
- * to determine old NICs.
- */
- switch (spawn->pci_dev->id.device_id) {
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4LX:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
- config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
- config->hw_vlan_insert = 0;
- break;
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
- /*
- * These NICs support VLAN insertion from WQE and
- * report the wqe_vlan_insert flag. But there is the bug
- * and PFC control may be broken, so disable feature.
- */
- config->hw_vlan_insert = 0;
- config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
- break;
- default:
- config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
- break;
- }
-exit:
- DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min);
-}
-
-/**
- * Configures the metadata mask fields in the shared context.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- */
-static void
-mlx5_set_metadata_mask(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_ibv_shared *sh = priv->sh;
- uint32_t meta, mark, reg_c0;
-
- reg_c0 = ~priv->vport_meta_mask;
- switch (priv->config.dv_xmeta_en) {
- case MLX5_XMETA_MODE_LEGACY:
- meta = UINT32_MAX;
- mark = MLX5_FLOW_MARK_MASK;
- break;
- case MLX5_XMETA_MODE_META16:
- meta = reg_c0 >> rte_bsf32(reg_c0);
- mark = MLX5_FLOW_MARK_MASK;
- break;
- case MLX5_XMETA_MODE_META32:
- meta = UINT32_MAX;
- mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK;
- break;
- default:
- meta = 0;
- mark = 0;
- MLX5_ASSERT(false);
- break;
- }
- if (sh->dv_mark_mask && sh->dv_mark_mask != mark)
- DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X",
- sh->dv_mark_mask, mark);
- else
- sh->dv_mark_mask = mark;
- if (sh->dv_meta_mask && sh->dv_meta_mask != meta)
- DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X",
- sh->dv_meta_mask, meta);
- else
- sh->dv_meta_mask = meta;
- if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0)
- DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X",
- sh->dv_meta_mask, reg_c0);
- else
- sh->dv_regc0_mask = reg_c0;
- DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en);
- DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask);
- DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask);
- DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
-}
-
-/**
- * Allocate page of door-bells and register it using DevX API.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- *
- * @return
- * Pointer to new page on success, NULL otherwise.
- */
-static struct mlx5_devx_dbr_page *
-mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page;
-
- /* Allocate space for door-bell page and management data. */
- page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
- RTE_CACHE_LINE_SIZE, dev->device->numa_node);
- if (!page) {
- DRV_LOG(ERR, "port %u cannot allocate dbr page",
- dev->data->port_id);
- return NULL;
- }
- /* Register allocated memory. */
- page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
- MLX5_DBR_PAGE_SIZE, 0);
- if (!page->umem) {
- DRV_LOG(ERR, "port %u cannot umem reg dbr page",
- dev->data->port_id);
- rte_free(page);
- return NULL;
- }
- return page;
-}
-
-/**
- * Find the next available door-bell, allocate new page if needed.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- * @param [out] dbr_page
- * Door-bell page containing the page data.
- *
- * @return
- * Door-bell address offset on success, a negative error value otherwise.
- */
-int64_t
-mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page = NULL;
- uint32_t i, j;
-
- LIST_FOREACH(page, &priv->dbrpgs, next)
- if (page->dbr_count < MLX5_DBR_PER_PAGE)
- break;
- if (!page) { /* No page with free door-bell exists. */
- page = mlx5_alloc_dbr_page(dev);
- if (!page) /* Failed to allocate new page. */
- return (-1);
- LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
- }
- /* Loop to find bitmap part with clear bit. */
- for (i = 0;
- i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
- i++)
- ; /* Empty. */
- /* Find the first clear bit. */
- MLX5_ASSERT(i < MLX5_DBR_BITMAP_SIZE);
- j = rte_bsf64(~page->dbr_bitmap[i]);
- page->dbr_bitmap[i] |= (1 << j);
- page->dbr_count++;
- *dbr_page = page;
- return (((i * 64) + j) * sizeof(uint64_t));
-}
-
-/**
- * Release a door-bell record.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- * @param [in] umem_id
- * UMEM ID of page containing the door-bell record to release.
- * @param [in] offset
- * Offset of door-bell record in page.
- *
- * @return
- * 0 on success, a negative error value otherwise.
- */
-int32_t
-mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page = NULL;
- int ret = 0;
-
- LIST_FOREACH(page, &priv->dbrpgs, next)
- /* Find the page this address belongs to. */
- if (page->umem->umem_id == umem_id)
- break;
- if (!page)
- return -EINVAL;
- page->dbr_count--;
- if (!page->dbr_count) {
- /* Page not used, free it and remove from list. */
- LIST_REMOVE(page, next);
- if (page->umem)
- ret = -mlx5_glue->devx_umem_dereg(page->umem);
- rte_free(page);
- } else {
- /* Mark in bitmap that this door-bell is not in use. */
- offset /= MLX5_DBR_SIZE;
- int i = offset / 64;
- int j = offset % 64;
-
- page->dbr_bitmap[i] &= ~(1 << j);
- }
- return ret;
-}
-
-int
-rte_pmd_mlx5_get_dyn_flag_names(char *names[], unsigned int n)
-{
- static const char *const dynf_names[] = {
- RTE_PMD_MLX5_FINE_GRANULARITY_INLINE,
- RTE_MBUF_DYNFLAG_METADATA_NAME
- };
- unsigned int i;
-
- if (n < RTE_DIM(dynf_names))
- return -ENOMEM;
- for (i = 0; i < RTE_DIM(dynf_names); i++) {
- if (names[i] == NULL)
- return -EINVAL;
- strcpy(names[i], dynf_names[i]);
- }
- return RTE_DIM(dynf_names);
-}
-
-/**
- * Check sibling device configurations.
- *
- * Sibling devices sharing the Infiniband device context
- * should have compatible configurations. This regards
- * representors and bonding slaves.
- *
- * @param priv
- * Private device descriptor.
- * @param config
- * Configuration of the device is going to be created.
- *
- * @return
- * 0 on success, EINVAL otherwise
- */
-static int
-mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
- struct mlx5_dev_config *config)
-{
- struct mlx5_ibv_shared *sh = priv->sh;
- struct mlx5_dev_config *sh_conf = NULL;
- uint16_t port_id;
-
- MLX5_ASSERT(sh);
- /* Nothing to compare for the single/first device. */
- if (sh->refcnt == 1)
- return 0;
- /* Find the device with shared context. */
- MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (opriv && opriv != priv && opriv->sh == sh) {
- sh_conf = &opriv->config;
- break;
- }
- }
- if (!sh_conf)
- return 0;
- if (sh_conf->dv_flow_en ^ config->dv_flow_en) {
- DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch"
- " for shared %s context", sh->ibdev_name);
- rte_errno = EINVAL;
- return rte_errno;
- }
- if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) {
- DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch"
- " for shared %s context", sh->ibdev_name);
- rte_errno = EINVAL;
- return rte_errno;