- case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
- config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
- config->hw_vlan_insert = 0;
- break;
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5EX:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
- /*
- * These NICs support VLAN insertion from WQE and
- * report the wqe_vlan_insert flag. But there is the bug
- * and PFC control may be broken, so disable feature.
- */
- config->hw_vlan_insert = 0;
- config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
- break;
- default:
- config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
- break;
- }
-exit:
- DRV_LOG(DEBUG, "min tx inline configured: %d", config->txq_inline_min);
-}
-
-/**
- * Configures the metadata mask fields in the shared context.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- */
-static void
-mlx5_set_metadata_mask(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_ibv_shared *sh = priv->sh;
- uint32_t meta, mark, reg_c0;
-
- reg_c0 = ~priv->vport_meta_mask;
- switch (priv->config.dv_xmeta_en) {
- case MLX5_XMETA_MODE_LEGACY:
- meta = UINT32_MAX;
- mark = MLX5_FLOW_MARK_MASK;
- break;
- case MLX5_XMETA_MODE_META16:
- meta = reg_c0 >> rte_bsf32(reg_c0);
- mark = MLX5_FLOW_MARK_MASK;
- break;
- case MLX5_XMETA_MODE_META32:
- meta = UINT32_MAX;
- mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK;
- break;
- default:
- meta = 0;
- mark = 0;
- assert(false);
- break;
- }
- if (sh->dv_mark_mask && sh->dv_mark_mask != mark)
- DRV_LOG(WARNING, "metadata MARK mask mismatche %08X:%08X",
- sh->dv_mark_mask, mark);
- else
- sh->dv_mark_mask = mark;
- if (sh->dv_meta_mask && sh->dv_meta_mask != meta)
- DRV_LOG(WARNING, "metadata META mask mismatche %08X:%08X",
- sh->dv_meta_mask, meta);
- else
- sh->dv_meta_mask = meta;
- if (sh->dv_regc0_mask && sh->dv_regc0_mask != reg_c0)
- DRV_LOG(WARNING, "metadata reg_c0 mask mismatche %08X:%08X",
- sh->dv_meta_mask, reg_c0);
- else
- sh->dv_regc0_mask = reg_c0;
- DRV_LOG(DEBUG, "metadata mode %u", priv->config.dv_xmeta_en);
- DRV_LOG(DEBUG, "metadata MARK mask %08X", sh->dv_mark_mask);
- DRV_LOG(DEBUG, "metadata META mask %08X", sh->dv_meta_mask);
- DRV_LOG(DEBUG, "metadata reg_c0 mask %08X", sh->dv_regc0_mask);
-}
-
-/**
- * Allocate page of door-bells and register it using DevX API.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- *
- * @return
- * Pointer to new page on success, NULL otherwise.
- */
-static struct mlx5_devx_dbr_page *
-mlx5_alloc_dbr_page(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page;
-
- /* Allocate space for door-bell page and management data. */
- page = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_devx_dbr_page),
- RTE_CACHE_LINE_SIZE, dev->device->numa_node);
- if (!page) {
- DRV_LOG(ERR, "port %u cannot allocate dbr page",
- dev->data->port_id);
- return NULL;
- }
- /* Register allocated memory. */
- page->umem = mlx5_glue->devx_umem_reg(priv->sh->ctx, page->dbrs,
- MLX5_DBR_PAGE_SIZE, 0);
- if (!page->umem) {
- DRV_LOG(ERR, "port %u cannot umem reg dbr page",
- dev->data->port_id);
- rte_free(page);
- return NULL;
- }
- return page;
-}
-
-/**
- * Find the next available door-bell, allocate new page if needed.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- * @param [out] dbr_page
- * Door-bell page containing the page data.
- *
- * @return
- * Door-bell address offset on success, a negative error value otherwise.
- */
-int64_t
-mlx5_get_dbr(struct rte_eth_dev *dev, struct mlx5_devx_dbr_page **dbr_page)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page = NULL;
- uint32_t i, j;
-
- LIST_FOREACH(page, &priv->dbrpgs, next)
- if (page->dbr_count < MLX5_DBR_PER_PAGE)
- break;
- if (!page) { /* No page with free door-bell exists. */
- page = mlx5_alloc_dbr_page(dev);
- if (!page) /* Failed to allocate new page. */
- return (-1);
- LIST_INSERT_HEAD(&priv->dbrpgs, page, next);
- }
- /* Loop to find bitmap part with clear bit. */
- for (i = 0;
- i < MLX5_DBR_BITMAP_SIZE && page->dbr_bitmap[i] == UINT64_MAX;
- i++)
- ; /* Empty. */
- /* Find the first clear bit. */
- j = rte_bsf64(~page->dbr_bitmap[i]);
- assert(i < (MLX5_DBR_PER_PAGE / 64));
- page->dbr_bitmap[i] |= (1 << j);
- page->dbr_count++;
- *dbr_page = page;
- return (((i * 64) + j) * sizeof(uint64_t));
-}
-
-/**
- * Release a door-bell record.
- *
- * @param [in] dev
- * Pointer to Ethernet device.
- * @param [in] umem_id
- * UMEM ID of page containing the door-bell record to release.
- * @param [in] offset
- * Offset of door-bell record in page.
- *
- * @return
- * 0 on success, a negative error value otherwise.
- */
-int32_t
-mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id, uint64_t offset)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_devx_dbr_page *page = NULL;
- int ret = 0;
-
- LIST_FOREACH(page, &priv->dbrpgs, next)
- /* Find the page this address belongs to. */
- if (page->umem->umem_id == umem_id)
- break;
- if (!page)
- return -EINVAL;
- page->dbr_count--;
- if (!page->dbr_count) {
- /* Page not used, free it and remove from list. */
- LIST_REMOVE(page, next);
- if (page->umem)
- ret = -mlx5_glue->devx_umem_dereg(page->umem);
- rte_free(page);
- } else {
- /* Mark in bitmap that this door-bell is not in use. */
- offset /= MLX5_DBR_SIZE;
- int i = offset / 64;
- int j = offset % 64;
-
- page->dbr_bitmap[i] &= ~(1 << j);
- }
- return ret;
-}
-
-/**
- * Check sibling device configurations.
- *
- * Sibling devices sharing the Infiniband device context
- * should have compatible configurations. This regards
- * representors and bonding slaves.
- *
- * @param priv
- * Private device descriptor.
- * @param config
- * Configuration of the device is going to be created.
- *
- * @return
- * 0 on success, EINVAL otherwise
- */
-static int
-mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
- struct mlx5_dev_config *config)
-{
- struct mlx5_ibv_shared *sh = priv->sh;
- struct mlx5_dev_config *sh_conf = NULL;
- uint16_t port_id;
-
- assert(sh);
- /* Nothing to compare for the single/first device. */
- if (sh->refcnt == 1)
- return 0;
- /* Find the device with shared context. */
- MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (opriv && opriv != priv && opriv->sh == sh) {
- sh_conf = &opriv->config;
- break;
- }
- }
- if (!sh_conf)
- return 0;
- if (sh_conf->dv_flow_en ^ config->dv_flow_en) {
- DRV_LOG(ERR, "\"dv_flow_en\" configuration mismatch"
- " for shared %s context", sh->ibdev_name);
- rte_errno = EINVAL;
- return rte_errno;
- }
- if (sh_conf->dv_xmeta_en ^ config->dv_xmeta_en) {
- DRV_LOG(ERR, "\"dv_xmeta_en\" configuration mismatch"
- " for shared %s context", sh->ibdev_name);
- rte_errno = EINVAL;
- return rte_errno;
- }
- return 0;
-}
-/**
- * Spawn an Ethernet device from Verbs information.
- *
- * @param dpdk_dev
- * Backing DPDK device.
- * @param spawn
- * Verbs device parameters (name, port, switch_info) to spawn.
- * @param config
- * Device configuration parameters.
- *
- * @return
- * A valid Ethernet device object on success, NULL otherwise and rte_errno
- * is set. The following errors are defined:
- *
- * EBUSY: device is not supposed to be spawned.
- * EEXIST: device is already spawned
- */
-static struct rte_eth_dev *
-mlx5_dev_spawn(struct rte_device *dpdk_dev,
- struct mlx5_dev_spawn_data *spawn,
- struct mlx5_dev_config config)
-{
- const struct mlx5_switch_info *switch_info = &spawn->info;
- struct mlx5_ibv_shared *sh = NULL;
- struct ibv_port_attr port_attr;
- struct mlx5dv_context dv_attr = { .comp_mask = 0 };
- struct rte_eth_dev *eth_dev = NULL;
- struct mlx5_priv *priv = NULL;
- int err = 0;
- unsigned int hw_padding = 0;
- unsigned int mps;
- unsigned int cqe_comp;
- unsigned int cqe_pad = 0;
- unsigned int tunnel_en = 0;
- unsigned int mpls_en = 0;
- unsigned int swp = 0;
- unsigned int mprq = 0;
- unsigned int mprq_min_stride_size_n = 0;
- unsigned int mprq_max_stride_size_n = 0;
- unsigned int mprq_min_stride_num_n = 0;
- unsigned int mprq_max_stride_num_n = 0;
- struct rte_ether_addr mac;
- char name[RTE_ETH_NAME_MAX_LEN];
- int own_domain_id = 0;
- uint16_t port_id;
- unsigned int i;
-#ifdef HAVE_MLX5DV_DR_DEVX_PORT
- struct mlx5dv_devx_port devx_port = { .comp_mask = 0 };
-#endif
-
- /* Determine if this port representor is supposed to be spawned. */
- if (switch_info->representor && dpdk_dev->devargs) {
- struct rte_eth_devargs eth_da;
-
- err = rte_eth_devargs_parse(dpdk_dev->devargs->args, ð_da);
- if (err) {
- rte_errno = -err;
- DRV_LOG(ERR, "failed to process device arguments: %s",
- strerror(rte_errno));
- return NULL;
- }
- for (i = 0; i < eth_da.nb_representor_ports; ++i)
- if (eth_da.representor_ports[i] ==
- (uint16_t)switch_info->port_name)
- break;
- if (i == eth_da.nb_representor_ports) {
- rte_errno = EBUSY;
- return NULL;
- }
- }
- /* Build device name. */
- if (spawn->pf_bond < 0) {
- /* Single device. */
- if (!switch_info->representor)
- strlcpy(name, dpdk_dev->name, sizeof(name));
- else
- snprintf(name, sizeof(name), "%s_representor_%u",
- dpdk_dev->name, switch_info->port_name);
- } else {
- /* Bonding device. */
- if (!switch_info->representor)
- snprintf(name, sizeof(name), "%s_%s",
- dpdk_dev->name, spawn->ibv_dev->name);
- else
- snprintf(name, sizeof(name), "%s_%s_representor_%u",
- dpdk_dev->name, spawn->ibv_dev->name,
- switch_info->port_name);
- }
- /* check if the device is already spawned */
- if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) {
- rte_errno = EEXIST;
- return NULL;
- }
- DRV_LOG(DEBUG, "naming Ethernet device \"%s\"", name);
- if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
- eth_dev = rte_eth_dev_attach_secondary(name);
- if (eth_dev == NULL) {
- DRV_LOG(ERR, "can not attach rte ethdev");
- rte_errno = ENOMEM;
- return NULL;
- }
- eth_dev->device = dpdk_dev;
- eth_dev->dev_ops = &mlx5_dev_sec_ops;
- err = mlx5_proc_priv_init(eth_dev);
- if (err)
- return NULL;
- /* Receive command fd from primary process */
- err = mlx5_mp_req_verbs_cmd_fd(eth_dev);
- if (err < 0)
- return NULL;
- /* Remap UAR for Tx queues. */
- err = mlx5_tx_uar_init_secondary(eth_dev, err);
- if (err)
- return NULL;
- /*
- * Ethdev pointer is still required as input since
- * the primary device is not accessible from the
- * secondary process.
- */
- eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev);
- eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev);
- return eth_dev;
- }
- /*
- * Some parameters ("tx_db_nc" in particularly) are needed in
- * advance to create dv/verbs device context. We proceed the
- * devargs here to get ones, and later proceed devargs again
- * to override some hardware settings.
- */
- err = mlx5_args(&config, dpdk_dev->devargs);
- if (err) {
- err = rte_errno;
- DRV_LOG(ERR, "failed to process device arguments: %s",
- strerror(rte_errno));
- goto error;
- }
- sh = mlx5_alloc_shared_ibctx(spawn, &config);
- if (!sh)
- return NULL;
- config.devx = sh->devx;
-#ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
- config.dest_tir = 1;
-#endif
-#ifdef HAVE_IBV_MLX5_MOD_SWP
- dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
-#endif
- /*
- * Multi-packet send is supported by ConnectX-4 Lx PF as well
- * as all ConnectX-5 devices.
- */
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
-#endif
-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
- dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
-#endif
- mlx5_glue->dv_query_device(sh->ctx, &dv_attr);
- if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
- if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
- DRV_LOG(DEBUG, "enhanced MPW is supported");
- mps = MLX5_MPW_ENHANCED;
- } else {
- DRV_LOG(DEBUG, "MPW is supported");
- mps = MLX5_MPW;
- }
- } else {
- DRV_LOG(DEBUG, "MPW isn't supported");
- mps = MLX5_MPW_DISABLED;
- }
-#ifdef HAVE_IBV_MLX5_MOD_SWP
- if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
- swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
- DRV_LOG(DEBUG, "SWP support: %u", swp);
-#endif
- config.swp = !!swp;
-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
- if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
- struct mlx5dv_striding_rq_caps mprq_caps =
- dv_attr.striding_rq_caps;
-
- DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
- mprq_caps.min_single_stride_log_num_of_bytes);
- DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d",
- mprq_caps.max_single_stride_log_num_of_bytes);
- DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d",
- mprq_caps.min_single_wqe_log_num_of_strides);
- DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d",
- mprq_caps.max_single_wqe_log_num_of_strides);
- DRV_LOG(DEBUG, "\tsupported_qpts: %d",
- mprq_caps.supported_qpts);
- DRV_LOG(DEBUG, "device supports Multi-Packet RQ");
- mprq = 1;
- mprq_min_stride_size_n =
- mprq_caps.min_single_stride_log_num_of_bytes;
- mprq_max_stride_size_n =
- mprq_caps.max_single_stride_log_num_of_bytes;
- mprq_min_stride_num_n =
- mprq_caps.min_single_wqe_log_num_of_strides;
- mprq_max_stride_num_n =
- mprq_caps.max_single_wqe_log_num_of_strides;
- config.mprq.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
- mprq_min_stride_num_n);
- }
-#endif
- if (RTE_CACHE_LINE_SIZE == 128 &&
- !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
- cqe_comp = 0;
- else
- cqe_comp = 1;
- config.cqe_comp = cqe_comp;
-#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
- /* Whether device supports 128B Rx CQE padding. */
- cqe_pad = RTE_CACHE_LINE_SIZE == 128 &&
- (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD);
-#endif
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
- tunnel_en = ((dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
- (dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE) &&
- (dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE));
- }
- DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
- tunnel_en ? "" : "not ");
-#else
- DRV_LOG(WARNING,
- "tunnel offloading disabled due to old OFED/rdma-core version");
-#endif
- config.tunnel_en = tunnel_en;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
- mpls_en = ((dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
- (dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
- DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
- mpls_en ? "" : "not ");
-#else
- DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
- " old OFED/rdma-core version or firmware configuration");
-#endif
- config.mpls_en = mpls_en;
- /* Check port status. */
- err = mlx5_glue->query_port(sh->ctx, spawn->ibv_port, &port_attr);
- if (err) {
- DRV_LOG(ERR, "port query failed: %s", strerror(err));
- goto error;
- }
- if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
- DRV_LOG(ERR, "port is not configured in Ethernet mode");
- err = EINVAL;
- goto error;
- }
- if (port_attr.state != IBV_PORT_ACTIVE)
- DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
- mlx5_glue->port_state_str(port_attr.state),
- port_attr.state);
- /* Allocate private eth device data. */
- priv = rte_zmalloc("ethdev private structure",
- sizeof(*priv),
- RTE_CACHE_LINE_SIZE);
- if (priv == NULL) {
- DRV_LOG(ERR, "priv allocation failure");
- err = ENOMEM;
- goto error;
- }
- priv->sh = sh;
- priv->ibv_port = spawn->ibv_port;
- priv->pci_dev = spawn->pci_dev;
- priv->mtu = RTE_ETHER_MTU;
-#ifndef RTE_ARCH_64
- /* Initialize UAR access locks for 32bit implementations. */
- rte_spinlock_init(&priv->uar_lock_cq);
- for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
- rte_spinlock_init(&priv->uar_lock[i]);
-#endif
- /* Some internal functions rely on Netlink sockets, open them now. */
- priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
- priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE);
- priv->nl_sn = 0;
- priv->representor = !!switch_info->representor;
- priv->master = !!switch_info->master;
- priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
- priv->vport_meta_tag = 0;
- priv->vport_meta_mask = 0;
- priv->pf_bond = spawn->pf_bond;
-#ifdef HAVE_MLX5DV_DR_DEVX_PORT
- /*
- * The DevX port query API is implemented. E-Switch may use
- * either vport or reg_c[0] metadata register to match on
- * vport index. The engaged part of metadata register is
- * defined by mask.
- */
- if (switch_info->representor || switch_info->master) {
- devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT |
- MLX5DV_DEVX_PORT_MATCH_REG_C_0;
- err = mlx5_glue->devx_port_query(sh->ctx, spawn->ibv_port,
- &devx_port);
- if (err) {
- DRV_LOG(WARNING,
- "can't query devx port %d on device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- devx_port.comp_mask = 0;
- }
- }
- if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
- priv->vport_meta_tag = devx_port.reg_c_0.value;
- priv->vport_meta_mask = devx_port.reg_c_0.mask;
- if (!priv->vport_meta_mask) {
- DRV_LOG(ERR, "vport zero mask for port %d"
- " on bonding device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- err = ENOTSUP;
- goto error;
- }
- if (priv->vport_meta_tag & ~priv->vport_meta_mask) {
- DRV_LOG(ERR, "invalid vport tag for port %d"
- " on bonding device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- err = ENOTSUP;
- goto error;
- }
- }
- if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
- priv->vport_id = devx_port.vport_num;
- } else if (spawn->pf_bond >= 0) {
- DRV_LOG(ERR, "can't deduce vport index for port %d"
- " on bonding device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- err = ENOTSUP;
- goto error;
- } else {
- /* Suppose vport index in compatible way. */
- priv->vport_id = switch_info->representor ?
- switch_info->port_name + 1 : -1;
- }
-#else
- /*
- * Kernel/rdma_core support single E-Switch per PF configurations
- * only and vport_id field contains the vport index for
- * associated VF, which is deduced from representor port name.
- * For example, let's have the IB device port 10, it has
- * attached network device eth0, which has port name attribute
- * pf0vf2, we can deduce the VF number as 2, and set vport index
- * as 3 (2+1). This assigning schema should be changed if the
- * multiple E-Switch instances per PF configurations or/and PCI
- * subfunctions are added.
- */
- priv->vport_id = switch_info->representor ?
- switch_info->port_name + 1 : -1;
-#endif
- /* representor_id field keeps the unmodified VF index. */
- priv->representor_id = switch_info->representor ?
- switch_info->port_name : -1;
- /*
- * Look for sibling devices in order to reuse their switch domain
- * if any, otherwise allocate one.
- */
- MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
- const struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id ==
- RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
- continue;
- priv->domain_id = opriv->domain_id;
- break;
- }
- if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
- err = rte_eth_switch_domain_alloc(&priv->domain_id);
- if (err) {
- err = rte_errno;
- DRV_LOG(ERR, "unable to allocate switch domain: %s",
- strerror(rte_errno));
- goto error;
- }
- own_domain_id = 1;
- }
- /* Override some values set by hardware configuration. */
- mlx5_args(&config, dpdk_dev->devargs);
- err = mlx5_dev_check_sibling_config(priv, &config);
- if (err)
- goto error;
- config.hw_csum = !!(sh->device_attr.device_cap_flags_ex &
- IBV_DEVICE_RAW_IP_CSUM);
- DRV_LOG(DEBUG, "checksum offloading is %ssupported",
- (config.hw_csum ? "" : "not "));
-#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
- !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
- DRV_LOG(DEBUG, "counters are not supported");
-#endif
-#if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR)
- if (config.dv_flow_en) {
- DRV_LOG(WARNING, "DV flow is not supported");
- config.dv_flow_en = 0;
- }
-#endif
- config.ind_table_max_size =
- sh->device_attr.rss_caps.max_rwq_indirection_table_size;
- /*
- * Remove this check once DPDK supports larger/variable
- * indirection tables.
- */
- if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
- config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
- DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
- config.ind_table_max_size);
- config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps &
- IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
- DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
- (config.hw_vlan_strip ? "" : "not "));
- config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps &
- IBV_RAW_PACKET_CAP_SCATTER_FCS);
- DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
- (config.hw_fcs_strip ? "" : "not "));
-#if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
- hw_padding = !!sh->device_attr.rx_pad_end_addr_align;
-#elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
- hw_padding = !!(sh->device_attr.device_cap_flags_ex &
- IBV_DEVICE_PCI_WRITE_END_PADDING);
-#endif
- if (config.hw_padding && !hw_padding) {
- DRV_LOG(DEBUG, "Rx end alignment padding isn't supported");
- config.hw_padding = 0;
- } else if (config.hw_padding) {
- DRV_LOG(DEBUG, "Rx end alignment padding is enabled");
- }
- config.tso = (sh->device_attr.tso_caps.max_tso > 0 &&
- (sh->device_attr.tso_caps.supported_qpts &
- (1 << IBV_QPT_RAW_PACKET)));
- if (config.tso)
- config.tso_max_payload_sz = sh->device_attr.tso_caps.max_tso;
- /*
- * MPW is disabled by default, while the Enhanced MPW is enabled
- * by default.
- */
- if (config.mps == MLX5_ARG_UNSET)
- config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED :
- MLX5_MPW_DISABLED;
- else
- config.mps = config.mps ? mps : MLX5_MPW_DISABLED;
- DRV_LOG(INFO, "%sMPS is %s",
- config.mps == MLX5_MPW_ENHANCED ? "enhanced " :
- config.mps == MLX5_MPW ? "legacy " : "",
- config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
- if (config.cqe_comp && !cqe_comp) {
- DRV_LOG(WARNING, "Rx CQE compression isn't supported");
- config.cqe_comp = 0;
- }
- if (config.cqe_pad && !cqe_pad) {
- DRV_LOG(WARNING, "Rx CQE padding isn't supported");
- config.cqe_pad = 0;
- } else if (config.cqe_pad) {
- DRV_LOG(INFO, "Rx CQE padding is enabled");
- }
- if (config.devx) {
- priv->counter_fallback = 0;
- err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr);
- if (err) {
- err = -err;
- goto error;
- }
- if (!config.hca_attr.flow_counters_dump)
- priv->counter_fallback = 1;
-#ifndef HAVE_IBV_DEVX_ASYNC
- priv->counter_fallback = 1;
-#endif
- if (priv->counter_fallback)
- DRV_LOG(INFO, "Use fall-back DV counter management");
- /* Check for LRO support. */
- if (config.dest_tir && config.hca_attr.lro_cap &&
- config.dv_flow_en) {
- /* TBD check tunnel lro caps. */
- config.lro.supported = config.hca_attr.lro_cap;
- DRV_LOG(DEBUG, "Device supports LRO");
- /*
- * If LRO timeout is not configured by application,
- * use the minimal supported value.
- */
- if (!config.lro.timeout)
- config.lro.timeout =
- config.hca_attr.lro_timer_supported_periods[0];
- DRV_LOG(DEBUG, "LRO session timeout set to %d usec",
- config.lro.timeout);
- }
-#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
- if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup &&
- config.dv_flow_en) {
- uint8_t reg_c_mask =
- config.hca_attr.qos.flow_meter_reg_c_ids;
- /*
- * Meter needs two REG_C's for color match and pre-sfx
- * flow match. Here get the REG_C for color match.
- * REG_C_0 and REG_C_1 is reserved for metadata feature.
- */
- reg_c_mask &= 0xfc;
- if (__builtin_popcount(reg_c_mask) < 1) {
- priv->mtr_en = 0;
- DRV_LOG(WARNING, "No available register for"
- " meter.");
- } else {
- priv->mtr_color_reg = ffs(reg_c_mask) - 1 +
- REG_C_0;
- priv->mtr_en = 1;
- priv->mtr_reg_share =
- config.hca_attr.qos.flow_meter_reg_share;
- DRV_LOG(DEBUG, "The REG_C meter uses is %d",
- priv->mtr_color_reg);
- }
- }
-#endif
- }
- if (config.mprq.enabled && mprq) {
- if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
- config.mprq.stride_num_n < mprq_min_stride_num_n) {
- config.mprq.stride_num_n =
- RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
- mprq_min_stride_num_n);
- DRV_LOG(WARNING,
- "the number of strides"
- " for Multi-Packet RQ is out of range,"
- " setting default value (%u)",
- 1 << config.mprq.stride_num_n);
- }
- config.mprq.min_stride_size_n = mprq_min_stride_size_n;
- config.mprq.max_stride_size_n = mprq_max_stride_size_n;
- } else if (config.mprq.enabled && !mprq) {
- DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
- config.mprq.enabled = 0;
- }
- if (config.max_dump_files_num == 0)
- config.max_dump_files_num = 128;
- eth_dev = rte_eth_dev_allocate(name);
- if (eth_dev == NULL) {
- DRV_LOG(ERR, "can not allocate rte ethdev");
- err = ENOMEM;
- goto error;
- }
- /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
- eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
- if (priv->representor) {
- eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
- eth_dev->data->representor_id = priv->representor_id;
- }
- /*
- * Store associated network device interface index. This index
- * is permanent throughout the lifetime of device. So, we may store
- * the ifindex here and use the cached value further.
- */
- assert(spawn->ifindex);
- priv->if_index = spawn->ifindex;
- eth_dev->data->dev_private = priv;
- priv->dev_data = eth_dev->data;
- eth_dev->data->mac_addrs = priv->mac;
- eth_dev->device = dpdk_dev;
- /* Configure the first MAC address by default. */
- if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
- DRV_LOG(ERR,
- "port %u cannot get MAC address, is mlx5_en"
- " loaded? (errno: %s)",
- eth_dev->data->port_id, strerror(rte_errno));
- err = ENODEV;
- goto error;
- }
- DRV_LOG(INFO,
- "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
- eth_dev->data->port_id,
- mac.addr_bytes[0], mac.addr_bytes[1],
- mac.addr_bytes[2], mac.addr_bytes[3],
- mac.addr_bytes[4], mac.addr_bytes[5]);
-#ifndef NDEBUG
- {
- char ifname[IF_NAMESIZE];
-
- if (mlx5_get_ifname(eth_dev, &ifname) == 0)
- DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
- eth_dev->data->port_id, ifname);
- else
- DRV_LOG(DEBUG, "port %u ifname is unknown",
- eth_dev->data->port_id);
- }
-#endif
- /* Get actual MTU if possible. */
- err = mlx5_get_mtu(eth_dev, &priv->mtu);
- if (err) {
- err = rte_errno;
- goto error;
- }
- DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
- priv->mtu);
- /* Initialize burst functions to prevent crashes before link-up. */
- eth_dev->rx_pkt_burst = removed_rx_burst;
- eth_dev->tx_pkt_burst = removed_tx_burst;
- eth_dev->dev_ops = &mlx5_dev_ops;
- /* Register MAC address. */
- claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
- if (config.vf && config.vf_nl_en)
- mlx5_nl_mac_addr_sync(eth_dev);
- TAILQ_INIT(&priv->flows);
- TAILQ_INIT(&priv->ctrl_flows);
- TAILQ_INIT(&priv->flow_meters);
- TAILQ_INIT(&priv->flow_meter_profiles);
- /* Hint libmlx5 to use PMD allocator for data plane resources */
- struct mlx5dv_ctx_allocators alctr = {
- .alloc = &mlx5_alloc_verbs_buf,
- .free = &mlx5_free_verbs_buf,
- .data = priv,
- };
- mlx5_glue->dv_set_context_attr(sh->ctx,
- MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
- (void *)((uintptr_t)&alctr));
- /* Bring Ethernet device up. */
- DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
- eth_dev->data->port_id);
- mlx5_set_link_up(eth_dev);
- /*
- * Even though the interrupt handler is not installed yet,
- * interrupts will still trigger on the async_fd from
- * Verbs context returned by ibv_open_device().
- */
- mlx5_link_update(eth_dev, 0);
-#ifdef HAVE_MLX5DV_DR_ESWITCH
- if (!(config.hca_attr.eswitch_manager && config.dv_flow_en &&
- (switch_info->representor || switch_info->master)))
- config.dv_esw_en = 0;
-#else
- config.dv_esw_en = 0;
-#endif
- /* Detect minimal data bytes to inline. */
- mlx5_set_min_inline(spawn, &config);
- /* Store device configuration on private structure. */
- priv->config = config;
- /* Create context for virtual machine VLAN workaround. */
- priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
- if (config.dv_flow_en) {
- err = mlx5_alloc_shared_dr(priv);
- if (err)
- goto error;
- /*
- * RSS id is shared with meter flow id. Meter flow id can only
- * use the 24 MSB of the register.
- */
- priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >>
- MLX5_MTR_COLOR_BITS);
- if (!priv->qrss_id_pool) {
- DRV_LOG(ERR, "can't create flow id pool");
- err = ENOMEM;
- goto error;
- }
- }
- /* Supported Verbs flow priority number detection. */
- err = mlx5_flow_discover_priorities(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
- }
- priv->config.flow_prio = err;
- if (!priv->config.dv_esw_en &&
- priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
- DRV_LOG(WARNING, "metadata mode %u is not supported "
- "(no E-Switch)", priv->config.dv_xmeta_en);
- priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY;
- }
- mlx5_set_metadata_mask(eth_dev);
- if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
- !priv->sh->dv_regc0_mask) {
- DRV_LOG(ERR, "metadata mode %u is not supported "
- "(no metadata reg_c[0] is available)",
- priv->config.dv_xmeta_en);
- err = ENOTSUP;
- goto error;
- }
- /* Query availibility of metadata reg_c's. */
- err = mlx5_flow_discover_mreg_c(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
- }
- if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
- DRV_LOG(DEBUG,
- "port %u extensive metadata register is not supported",
- eth_dev->data->port_id);
- if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
- DRV_LOG(ERR, "metadata mode %u is not supported "
- "(no metadata registers available)",
- priv->config.dv_xmeta_en);
- err = ENOTSUP;
- goto error;
- }
- }
- if (priv->config.dv_flow_en &&
- priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
- mlx5_flow_ext_mreg_supported(eth_dev) &&
- priv->sh->dv_regc0_mask) {
- priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
- MLX5_FLOW_MREG_HTABLE_SZ);
- if (!priv->mreg_cp_tbl) {
- err = ENOMEM;
- goto error;
- }
- }
- return eth_dev;
-error:
- if (priv) {
- if (priv->mreg_cp_tbl)
- mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
- if (priv->sh)
- mlx5_free_shared_dr(priv);
- if (priv->nl_socket_route >= 0)
- close(priv->nl_socket_route);
- if (priv->nl_socket_rdma >= 0)
- close(priv->nl_socket_rdma);
- if (priv->vmwa_context)
- mlx5_vlan_vmwa_exit(priv->vmwa_context);
- if (priv->qrss_id_pool)
- mlx5_flow_id_pool_release(priv->qrss_id_pool);
- if (own_domain_id)
- claim_zero(rte_eth_switch_domain_free(priv->domain_id));
- rte_free(priv);
- if (eth_dev != NULL)
- eth_dev->data->dev_private = NULL;
- }
- if (eth_dev != NULL) {
- /* mac_addrs must not be freed alone because part of dev_private */
- eth_dev->data->mac_addrs = NULL;
- rte_eth_dev_release_port(eth_dev);