- sh = mlx5_alloc_shared_ibctx(spawn, &config);
- if (!sh)
- return NULL;
- config.devx = sh->devx;
-#ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR
- config.dest_tir = 1;
-#endif
-#ifdef HAVE_IBV_MLX5_MOD_SWP
- dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP;
-#endif
- /*
- * Multi-packet send is supported by ConnectX-4 Lx PF as well
- * as all ConnectX-5 devices.
- */
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS;
-#endif
-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
- dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
-#endif
- mlx5_glue->dv_query_device(sh->ctx, &dv_attr);
- if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) {
- if (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) {
- DRV_LOG(DEBUG, "enhanced MPW is supported");
- mps = MLX5_MPW_ENHANCED;
- } else {
- DRV_LOG(DEBUG, "MPW is supported");
- mps = MLX5_MPW;
- }
- } else {
- DRV_LOG(DEBUG, "MPW isn't supported");
- mps = MLX5_MPW_DISABLED;
- }
-#ifdef HAVE_IBV_MLX5_MOD_SWP
- if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_SWP)
- swp = dv_attr.sw_parsing_caps.sw_parsing_offloads;
- DRV_LOG(DEBUG, "SWP support: %u", swp);
-#endif
- config.swp = !!swp;
-#ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
- if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) {
- struct mlx5dv_striding_rq_caps mprq_caps =
- dv_attr.striding_rq_caps;
-
- DRV_LOG(DEBUG, "\tmin_single_stride_log_num_of_bytes: %d",
- mprq_caps.min_single_stride_log_num_of_bytes);
- DRV_LOG(DEBUG, "\tmax_single_stride_log_num_of_bytes: %d",
- mprq_caps.max_single_stride_log_num_of_bytes);
- DRV_LOG(DEBUG, "\tmin_single_wqe_log_num_of_strides: %d",
- mprq_caps.min_single_wqe_log_num_of_strides);
- DRV_LOG(DEBUG, "\tmax_single_wqe_log_num_of_strides: %d",
- mprq_caps.max_single_wqe_log_num_of_strides);
- DRV_LOG(DEBUG, "\tsupported_qpts: %d",
- mprq_caps.supported_qpts);
- DRV_LOG(DEBUG, "device supports Multi-Packet RQ");
- mprq = 1;
- mprq_min_stride_size_n =
- mprq_caps.min_single_stride_log_num_of_bytes;
- mprq_max_stride_size_n =
- mprq_caps.max_single_stride_log_num_of_bytes;
- mprq_min_stride_num_n =
- mprq_caps.min_single_wqe_log_num_of_strides;
- mprq_max_stride_num_n =
- mprq_caps.max_single_wqe_log_num_of_strides;
- config.mprq.stride_num_n = RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
- mprq_min_stride_num_n);
- }
-#endif
- if (RTE_CACHE_LINE_SIZE == 128 &&
- !(dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP))
- cqe_comp = 0;
- else
- cqe_comp = 1;
- config.cqe_comp = cqe_comp;
-#ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD
- /* Whether device supports 128B Rx CQE padding. */
- cqe_pad = RTE_CACHE_LINE_SIZE == 128 &&
- (dv_attr.flags & MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD);
-#endif
-#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS) {
- tunnel_en = ((dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN) &&
- (dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE));
- }
- DRV_LOG(DEBUG, "tunnel offloading is %ssupported",
- tunnel_en ? "" : "not ");
-#else
- DRV_LOG(WARNING,
- "tunnel offloading disabled due to old OFED/rdma-core version");
-#endif
- config.tunnel_en = tunnel_en;
-#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
- mpls_en = ((dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) &&
- (dv_attr.tunnel_offloads_caps &
- MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP));
- DRV_LOG(DEBUG, "MPLS over GRE/UDP tunnel offloading is %ssupported",
- mpls_en ? "" : "not ");
-#else
- DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to"
- " old OFED/rdma-core version or firmware configuration");
-#endif
- config.mpls_en = mpls_en;
- /* Check port status. */
- err = mlx5_glue->query_port(sh->ctx, spawn->ibv_port, &port_attr);
- if (err) {
- DRV_LOG(ERR, "port query failed: %s", strerror(err));
- goto error;
- }
- if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
- DRV_LOG(ERR, "port is not configured in Ethernet mode");
- err = EINVAL;
- goto error;
- }
- if (port_attr.state != IBV_PORT_ACTIVE)
- DRV_LOG(DEBUG, "port is not active: \"%s\" (%d)",
- mlx5_glue->port_state_str(port_attr.state),
- port_attr.state);
- /* Allocate private eth device data. */
- priv = rte_zmalloc("ethdev private structure",
- sizeof(*priv),
- RTE_CACHE_LINE_SIZE);
- if (priv == NULL) {
- DRV_LOG(ERR, "priv allocation failure");
- err = ENOMEM;
- goto error;
- }
- priv->sh = sh;
- priv->ibv_port = spawn->ibv_port;
- priv->pci_dev = spawn->pci_dev;
- priv->mtu = RTE_ETHER_MTU;
-#ifndef RTE_ARCH_64
- /* Initialize UAR access locks for 32bit implementations. */
- rte_spinlock_init(&priv->uar_lock_cq);
- for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
- rte_spinlock_init(&priv->uar_lock[i]);
-#endif
- /* Some internal functions rely on Netlink sockets, open them now. */
- priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA);
- priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE);
- priv->nl_sn = 0;
- priv->representor = !!switch_info->representor;
- priv->master = !!switch_info->master;
- priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
- priv->vport_meta_tag = 0;
- priv->vport_meta_mask = 0;
- priv->pf_bond = spawn->pf_bond;
-#ifdef HAVE_MLX5DV_DR_DEVX_PORT
- /*
- * The DevX port query API is implemented. E-Switch may use
- * either vport or reg_c[0] metadata register to match on
- * vport index. The engaged part of metadata register is
- * defined by mask.
- */
- if (switch_info->representor || switch_info->master) {
- devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT |
- MLX5DV_DEVX_PORT_MATCH_REG_C_0;
- err = mlx5_glue->devx_port_query(sh->ctx, spawn->ibv_port,
- &devx_port);
- if (err) {
- DRV_LOG(WARNING,
- "can't query devx port %d on device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- devx_port.comp_mask = 0;
- }
- }
- if (devx_port.comp_mask & MLX5DV_DEVX_PORT_MATCH_REG_C_0) {
- priv->vport_meta_tag = devx_port.reg_c_0.value;
- priv->vport_meta_mask = devx_port.reg_c_0.mask;
- if (!priv->vport_meta_mask) {
- DRV_LOG(ERR, "vport zero mask for port %d"
- " on bonding device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- err = ENOTSUP;
- goto error;
- }
- if (priv->vport_meta_tag & ~priv->vport_meta_mask) {
- DRV_LOG(ERR, "invalid vport tag for port %d"
- " on bonding device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- err = ENOTSUP;
- goto error;
- }
- } else if (devx_port.comp_mask & MLX5DV_DEVX_PORT_VPORT) {
- priv->vport_id = devx_port.vport_num;
- } else if (spawn->pf_bond >= 0) {
- DRV_LOG(ERR, "can't deduce vport index for port %d"
- " on bonding device %s",
- spawn->ibv_port, spawn->ibv_dev->name);
- err = ENOTSUP;
- goto error;
- } else {
- /* Suppose vport index in compatible way. */
- priv->vport_id = switch_info->representor ?
- switch_info->port_name + 1 : -1;
- }
-#else
- /*
- * Kernel/rdma_core support single E-Switch per PF configurations
- * only and vport_id field contains the vport index for
- * associated VF, which is deduced from representor port name.
- * For example, let's have the IB device port 10, it has
- * attached network device eth0, which has port name attribute
- * pf0vf2, we can deduce the VF number as 2, and set vport index
- * as 3 (2+1). This assigning schema should be changed if the
- * multiple E-Switch instances per PF configurations or/and PCI
- * subfunctions are added.
- */
- priv->vport_id = switch_info->representor ?
- switch_info->port_name + 1 : -1;
-#endif
- /* representor_id field keeps the unmodified VF index. */
- priv->representor_id = switch_info->representor ?
- switch_info->port_name : -1;
- /*
- * Look for sibling devices in order to reuse their switch domain
- * if any, otherwise allocate one.
- */
- MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
- const struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id ==
- RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
- continue;
- priv->domain_id = opriv->domain_id;
- break;
- }
- if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
- err = rte_eth_switch_domain_alloc(&priv->domain_id);
- if (err) {
- err = rte_errno;
- DRV_LOG(ERR, "unable to allocate switch domain: %s",
- strerror(rte_errno));
- goto error;
- }
- own_domain_id = 1;
- }
- /* Override some values set by hardware configuration. */
- mlx5_args(&config, dpdk_dev->devargs);
- err = mlx5_dev_check_sibling_config(priv, &config);
- if (err)
- goto error;
- config.hw_csum = !!(sh->device_attr.device_cap_flags_ex &
- IBV_DEVICE_RAW_IP_CSUM);
- DRV_LOG(DEBUG, "checksum offloading is %ssupported",
- (config.hw_csum ? "" : "not "));
-#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
- !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
- DRV_LOG(DEBUG, "counters are not supported");
-#endif
-#ifndef HAVE_IBV_FLOW_DV_SUPPORT
- if (config.dv_flow_en) {
- DRV_LOG(WARNING, "DV flow is not supported");
- config.dv_flow_en = 0;
- }
-#endif
- config.ind_table_max_size =
- sh->device_attr.rss_caps.max_rwq_indirection_table_size;
- /*
- * Remove this check once DPDK supports larger/variable
- * indirection tables.
- */
- if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512)
- config.ind_table_max_size = ETH_RSS_RETA_SIZE_512;
- DRV_LOG(DEBUG, "maximum Rx indirection table size is %u",
- config.ind_table_max_size);
- config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps &
- IBV_RAW_PACKET_CAP_CVLAN_STRIPPING);
- DRV_LOG(DEBUG, "VLAN stripping is %ssupported",
- (config.hw_vlan_strip ? "" : "not "));
- config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps &
- IBV_RAW_PACKET_CAP_SCATTER_FCS);
- DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported",
- (config.hw_fcs_strip ? "" : "not "));
-#if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING)
- hw_padding = !!sh->device_attr.rx_pad_end_addr_align;
-#elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING)
- hw_padding = !!(sh->device_attr.device_cap_flags_ex &
- IBV_DEVICE_PCI_WRITE_END_PADDING);
-#endif
- if (config.hw_padding && !hw_padding) {
- DRV_LOG(DEBUG, "Rx end alignment padding isn't supported");
- config.hw_padding = 0;
- } else if (config.hw_padding) {
- DRV_LOG(DEBUG, "Rx end alignment padding is enabled");
- }
- config.tso = (sh->device_attr.tso_caps.max_tso > 0 &&
- (sh->device_attr.tso_caps.supported_qpts &
- (1 << IBV_QPT_RAW_PACKET)));
- if (config.tso)
- config.tso_max_payload_sz = sh->device_attr.tso_caps.max_tso;
- /*
- * MPW is disabled by default, while the Enhanced MPW is enabled
- * by default.
- */
- if (config.mps == MLX5_ARG_UNSET)
- config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED :
- MLX5_MPW_DISABLED;
- else
- config.mps = config.mps ? mps : MLX5_MPW_DISABLED;
- DRV_LOG(INFO, "%sMPS is %s",
- config.mps == MLX5_MPW_ENHANCED ? "enhanced " : "",
- config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
- if (config.cqe_comp && !cqe_comp) {
- DRV_LOG(WARNING, "Rx CQE compression isn't supported");
- config.cqe_comp = 0;
- }
- if (config.cqe_pad && !cqe_pad) {
- DRV_LOG(WARNING, "Rx CQE padding isn't supported");
- config.cqe_pad = 0;
- } else if (config.cqe_pad) {
- DRV_LOG(INFO, "Rx CQE padding is enabled");
- }
- if (config.devx) {
- priv->counter_fallback = 0;
- err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr);
- if (err) {
- err = -err;
- goto error;
- }
- if (!config.hca_attr.flow_counters_dump)
- priv->counter_fallback = 1;
-#ifndef HAVE_IBV_DEVX_ASYNC
- priv->counter_fallback = 1;
-#endif
- if (priv->counter_fallback)
- DRV_LOG(INFO, "Use fall-back DV counter management");
- /* Check for LRO support. */
- if (config.dest_tir && config.hca_attr.lro_cap &&
- config.dv_flow_en) {
- /* TBD check tunnel lro caps. */
- config.lro.supported = config.hca_attr.lro_cap;
- DRV_LOG(DEBUG, "Device supports LRO");
- /*
- * If LRO timeout is not configured by application,
- * use the minimal supported value.
- */
- if (!config.lro.timeout)
- config.lro.timeout =
- config.hca_attr.lro_timer_supported_periods[0];
- DRV_LOG(DEBUG, "LRO session timeout set to %d usec",
- config.lro.timeout);
- }
-#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
- if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup &&
- config.dv_flow_en) {
- uint8_t reg_c_mask =
- config.hca_attr.qos.flow_meter_reg_c_ids;
- /*
- * Meter needs two REG_C's for color match and pre-sfx
- * flow match. Here get the REG_C for color match.
- * REG_C_0 and REG_C_1 is reserved for metadata feature.
- */
- reg_c_mask &= 0xfc;
- if (__builtin_popcount(reg_c_mask) < 1) {
- priv->mtr_en = 0;
- DRV_LOG(WARNING, "No available register for"
- " meter.");
- } else {
- priv->mtr_color_reg = ffs(reg_c_mask) - 1 +
- REG_C_0;
- priv->mtr_en = 1;
- DRV_LOG(DEBUG, "The REG_C meter uses is %d",
- priv->mtr_color_reg);
- }
- }
-#endif
- }
- if (config.mprq.enabled && mprq) {
- if (config.mprq.stride_num_n > mprq_max_stride_num_n ||
- config.mprq.stride_num_n < mprq_min_stride_num_n) {
- config.mprq.stride_num_n =
- RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N,
- mprq_min_stride_num_n);
- DRV_LOG(WARNING,
- "the number of strides"
- " for Multi-Packet RQ is out of range,"
- " setting default value (%u)",
- 1 << config.mprq.stride_num_n);
- }
- config.mprq.min_stride_size_n = mprq_min_stride_size_n;
- config.mprq.max_stride_size_n = mprq_max_stride_size_n;
- } else if (config.mprq.enabled && !mprq) {
- DRV_LOG(WARNING, "Multi-Packet RQ isn't supported");
- config.mprq.enabled = 0;
- }
- if (config.max_dump_files_num == 0)
- config.max_dump_files_num = 128;
- eth_dev = rte_eth_dev_allocate(name);
- if (eth_dev == NULL) {
- DRV_LOG(ERR, "can not allocate rte ethdev");
- err = ENOMEM;
- goto error;
- }
- /* Flag to call rte_eth_dev_release_port() in rte_eth_dev_close(). */
- eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
- if (priv->representor) {
- eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
- eth_dev->data->representor_id = priv->representor_id;
- }
- /*
- * Store associated network device interface index. This index
- * is permanent throughout the lifetime of device. So, we may store
- * the ifindex here and use the cached value further.
- */
- assert(spawn->ifindex);
- priv->if_index = spawn->ifindex;
- eth_dev->data->dev_private = priv;
- priv->dev_data = eth_dev->data;
- eth_dev->data->mac_addrs = priv->mac;
- eth_dev->device = dpdk_dev;
- /* Configure the first MAC address by default. */
- if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
- DRV_LOG(ERR,
- "port %u cannot get MAC address, is mlx5_en"
- " loaded? (errno: %s)",
- eth_dev->data->port_id, strerror(rte_errno));
- err = ENODEV;
- goto error;
- }
- DRV_LOG(INFO,
- "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
- eth_dev->data->port_id,
- mac.addr_bytes[0], mac.addr_bytes[1],
- mac.addr_bytes[2], mac.addr_bytes[3],
- mac.addr_bytes[4], mac.addr_bytes[5]);
-#ifndef NDEBUG
- {
- char ifname[IF_NAMESIZE];
-
- if (mlx5_get_ifname(eth_dev, &ifname) == 0)
- DRV_LOG(DEBUG, "port %u ifname is \"%s\"",
- eth_dev->data->port_id, ifname);
- else
- DRV_LOG(DEBUG, "port %u ifname is unknown",
- eth_dev->data->port_id);
- }
-#endif
- /* Get actual MTU if possible. */
- err = mlx5_get_mtu(eth_dev, &priv->mtu);
- if (err) {
- err = rte_errno;
- goto error;
- }
- DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id,
- priv->mtu);
- /* Initialize burst functions to prevent crashes before link-up. */
- eth_dev->rx_pkt_burst = removed_rx_burst;
- eth_dev->tx_pkt_burst = removed_tx_burst;
- eth_dev->dev_ops = &mlx5_dev_ops;
- /* Register MAC address. */
- claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
- if (config.vf && config.vf_nl_en)
- mlx5_nl_mac_addr_sync(eth_dev);
- TAILQ_INIT(&priv->flows);
- TAILQ_INIT(&priv->ctrl_flows);
- TAILQ_INIT(&priv->flow_meters);
- TAILQ_INIT(&priv->flow_meter_profiles);
- /* Hint libmlx5 to use PMD allocator for data plane resources */
- struct mlx5dv_ctx_allocators alctr = {
- .alloc = &mlx5_alloc_verbs_buf,
- .free = &mlx5_free_verbs_buf,
- .data = priv,
- };
- mlx5_glue->dv_set_context_attr(sh->ctx,
- MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
- (void *)((uintptr_t)&alctr));
- /* Bring Ethernet device up. */
- DRV_LOG(DEBUG, "port %u forcing Ethernet interface up",
- eth_dev->data->port_id);
- mlx5_set_link_up(eth_dev);
- /*
- * Even though the interrupt handler is not installed yet,
- * interrupts will still trigger on the async_fd from
- * Verbs context returned by ibv_open_device().
- */
- mlx5_link_update(eth_dev, 0);
-#ifdef HAVE_MLX5DV_DR_ESWITCH
- if (!(config.hca_attr.eswitch_manager && config.dv_flow_en &&
- (switch_info->representor || switch_info->master)))
- config.dv_esw_en = 0;
-#else
- config.dv_esw_en = 0;
-#endif
- /* Detect minimal data bytes to inline. */
- mlx5_set_min_inline(spawn, &config);
- /* Store device configuration on private structure. */
- priv->config = config;
- /* Create context for virtual machine VLAN workaround. */
- priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
- if (config.dv_flow_en) {
- err = mlx5_alloc_shared_dr(priv);
- if (err)
- goto error;
- priv->qrss_id_pool = mlx5_flow_id_pool_alloc();
- if (!priv->qrss_id_pool) {
- DRV_LOG(ERR, "can't create flow id pool");
- err = ENOMEM;
- goto error;
- }
- }
- /* Supported Verbs flow priority number detection. */
- err = mlx5_flow_discover_priorities(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
- }
- priv->config.flow_prio = err;
- if (!priv->config.dv_esw_en &&
- priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
- DRV_LOG(WARNING, "metadata mode %u is not supported "
- "(no E-Switch)", priv->config.dv_xmeta_en);
- priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY;
- }
- mlx5_set_metadata_mask(eth_dev);
- if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
- !priv->sh->dv_regc0_mask) {
- DRV_LOG(ERR, "metadata mode %u is not supported "
- "(no metadata reg_c[0] is available)",
- priv->config.dv_xmeta_en);
- err = ENOTSUP;
- goto error;
- }
- /* Query availibility of metadata reg_c's. */
- err = mlx5_flow_discover_mreg_c(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
- }
- if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
- DRV_LOG(DEBUG,
- "port %u extensive metadata register is not supported",
- eth_dev->data->port_id);
- if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
- DRV_LOG(ERR, "metadata mode %u is not supported "
- "(no metadata registers available)",
- priv->config.dv_xmeta_en);
- err = ENOTSUP;
- goto error;
- }
- }
- if (priv->config.dv_flow_en &&
- priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
- mlx5_flow_ext_mreg_supported(eth_dev) &&
- priv->sh->dv_regc0_mask) {
- priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
- MLX5_FLOW_MREG_HTABLE_SZ);
- if (!priv->mreg_cp_tbl) {
- err = ENOMEM;
- goto error;
- }
- }
- return eth_dev;
-error:
- if (priv) {
- if (priv->mreg_cp_tbl)
- mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
- if (priv->sh)
- mlx5_free_shared_dr(priv);
- if (priv->nl_socket_route >= 0)
- close(priv->nl_socket_route);
- if (priv->nl_socket_rdma >= 0)
- close(priv->nl_socket_rdma);
- if (priv->vmwa_context)
- mlx5_vlan_vmwa_exit(priv->vmwa_context);
- if (priv->qrss_id_pool)
- mlx5_flow_id_pool_release(priv->qrss_id_pool);
- if (own_domain_id)
- claim_zero(rte_eth_switch_domain_free(priv->domain_id));
- rte_free(priv);
- if (eth_dev != NULL)
- eth_dev->data->dev_private = NULL;
- }
- if (eth_dev != NULL) {
- /* mac_addrs must not be freed alone because part of dev_private */
- eth_dev->data->mac_addrs = NULL;
- rte_eth_dev_release_port(eth_dev);
- }
- if (sh)
- mlx5_free_shared_ibctx(sh);
- assert(err > 0);
- rte_errno = err;
- return NULL;
-}
-
-/**
- * Comparison callback to sort device data.
- *
- * This is meant to be used with qsort().
- *
- * @param a[in]
- * Pointer to pointer to first data object.
- * @param b[in]
- * Pointer to pointer to second data object.
- *
- * @return
- * 0 if both objects are equal, less than 0 if the first argument is less
- * than the second, greater than 0 otherwise.
- */
-static int
-mlx5_dev_spawn_data_cmp(const void *a, const void *b)
-{
- const struct mlx5_switch_info *si_a =
- &((const struct mlx5_dev_spawn_data *)a)->info;
- const struct mlx5_switch_info *si_b =
- &((const struct mlx5_dev_spawn_data *)b)->info;
- int ret;
-
- /* Master device first. */
- ret = si_b->master - si_a->master;
- if (ret)
- return ret;
- /* Then representor devices. */
- ret = si_b->representor - si_a->representor;
- if (ret)
- return ret;
- /* Unidentified devices come last in no specific order. */
- if (!si_a->representor)
- return 0;
- /* Order representors by name. */
- return si_a->port_name - si_b->port_name;
-}
-
-/**
- * Match PCI information for possible slaves of bonding device.
- *
- * @param[in] ibv_dev
- * Pointer to Infiniband device structure.
- * @param[in] pci_dev
- * Pointer to PCI device structure to match PCI address.
- * @param[in] nl_rdma
- * Netlink RDMA group socket handle.
- *
- * @return
- * negative value if no bonding device found, otherwise
- * positive index of slave PF in bonding.
- */
-static int
-mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
- const struct rte_pci_device *pci_dev,
- int nl_rdma)
-{
- char ifname[IF_NAMESIZE + 1];
- unsigned int ifindex;
- unsigned int np, i;
- FILE *file = NULL;
- int pf = -1;
-
- /*
- * Try to get master device name. If something goes
- * wrong suppose the lack of kernel support and no
- * bonding devices.
- */
- if (nl_rdma < 0)
- return -1;
- if (!strstr(ibv_dev->name, "bond"))
- return -1;
- np = mlx5_nl_portnum(nl_rdma, ibv_dev->name);
- if (!np)
- return -1;
- /*
- * The Master device might not be on the predefined
- * port (not on port index 1, it is not garanted),
- * we have to scan all Infiniband device port and
- * find master.
- */
- for (i = 1; i <= np; ++i) {
- /* Check whether Infiniband port is populated. */
- ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i);
- if (!ifindex)
- continue;
- if (!if_indextoname(ifindex, ifname))
- continue;
- /* Try to read bonding slave names from sysfs. */
- MKSTR(slaves,
- "/sys/class/net/%s/master/bonding/slaves", ifname);
- file = fopen(slaves, "r");
- if (file)
- break;
- }
- if (!file)
- return -1;
- /* Use safe format to check maximal buffer length. */
- assert(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE);
- while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) {
- char tmp_str[IF_NAMESIZE + 32];
- struct rte_pci_addr pci_addr;
- struct mlx5_switch_info info;
-
- /* Process slave interface names in the loop. */
- snprintf(tmp_str, sizeof(tmp_str),
- "/sys/class/net/%s", ifname);
- if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) {
- DRV_LOG(WARNING, "can not get PCI address"
- " for netdev \"%s\"", ifname);
- continue;
- }
- if (pci_dev->addr.domain != pci_addr.domain ||
- pci_dev->addr.bus != pci_addr.bus ||
- pci_dev->addr.devid != pci_addr.devid ||
- pci_dev->addr.function != pci_addr.function)
- continue;
- /* Slave interface PCI address match found. */
- fclose(file);
- snprintf(tmp_str, sizeof(tmp_str),
- "/sys/class/net/%s/phys_port_name", ifname);
- file = fopen(tmp_str, "rb");
- if (!file)
- break;
- info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET;
- if (fscanf(file, "%32s", tmp_str) == 1)
- mlx5_translate_port_name(tmp_str, &info);
- if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY ||
- info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
- pf = info.port_name;
- break;
- }
- if (file)
- fclose(file);
- return pf;
-}
-
-/**
- * DPDK callback to register a PCI device.
- *
- * This function spawns Ethernet devices out of a given PCI device.
- *
- * @param[in] pci_drv
- * PCI driver structure (mlx5_driver).
- * @param[in] pci_dev
- * PCI device information.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
- struct rte_pci_device *pci_dev)
-{
- struct ibv_device **ibv_list;
- /*
- * Number of found IB Devices matching with requested PCI BDF.
- * nd != 1 means there are multiple IB devices over the same
- * PCI device and we have representors and master.
- */
- unsigned int nd = 0;
- /*
- * Number of found IB device Ports. nd = 1 and np = 1..n means
- * we have the single multiport IB device, and there may be
- * representors attached to some of found ports.
- */
- unsigned int np = 0;
- /*
- * Number of DPDK ethernet devices to Spawn - either over
- * multiple IB devices or multiple ports of single IB device.
- * Actually this is the number of iterations to spawn.
- */
- unsigned int ns = 0;
- /*
- * Bonding device
- * < 0 - no bonding device (single one)
- * >= 0 - bonding device (value is slave PF index)
- */
- int bd = -1;
- struct mlx5_dev_spawn_data *list = NULL;
- struct mlx5_dev_config dev_config;
- int ret;
-
- ret = mlx5_init_once();
- if (ret) {
- DRV_LOG(ERR, "unable to init PMD global data: %s",
- strerror(rte_errno));
- return -rte_errno;
- }
- assert(pci_drv == &mlx5_driver);
- errno = 0;
- ibv_list = mlx5_glue->get_device_list(&ret);
- if (!ibv_list) {
- rte_errno = errno ? errno : ENOSYS;
- DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
- return -rte_errno;
- }
- /*
- * First scan the list of all Infiniband devices to find
- * matching ones, gathering into the list.
- */
- struct ibv_device *ibv_match[ret + 1];
- int nl_route = mlx5_nl_init(NETLINK_ROUTE);
- int nl_rdma = mlx5_nl_init(NETLINK_RDMA);
- unsigned int i;
-
- while (ret-- > 0) {
- struct rte_pci_addr pci_addr;
-
- DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
- bd = mlx5_device_bond_pci_match
- (ibv_list[ret], pci_dev, nl_rdma);
- if (bd >= 0) {
- /*
- * Bonding device detected. Only one match is allowed,
- * the bonding is supported over multi-port IB device,
- * there should be no matches on representor PCI
- * functions or non VF LAG bonding devices with
- * specified address.
- */
- if (nd) {
- DRV_LOG(ERR,
- "multiple PCI match on bonding device"
- "\"%s\" found", ibv_list[ret]->name);
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- DRV_LOG(INFO, "PCI information matches for"
- " slave %d bonding device \"%s\"",
- bd, ibv_list[ret]->name);
- ibv_match[nd++] = ibv_list[ret];
- break;
- }
- if (mlx5_dev_to_pci_addr
- (ibv_list[ret]->ibdev_path, &pci_addr))
- continue;
- if (pci_dev->addr.domain != pci_addr.domain ||
- pci_dev->addr.bus != pci_addr.bus ||
- pci_dev->addr.devid != pci_addr.devid ||
- pci_dev->addr.function != pci_addr.function)
- continue;
- DRV_LOG(INFO, "PCI information matches for device \"%s\"",
- ibv_list[ret]->name);
- ibv_match[nd++] = ibv_list[ret];
- }
- ibv_match[nd] = NULL;
- if (!nd) {
- /* No device matches, just complain and bail out. */
- DRV_LOG(WARNING,
- "no Verbs device matches PCI device " PCI_PRI_FMT ","
- " are kernel drivers loaded?",
- pci_dev->addr.domain, pci_dev->addr.bus,
- pci_dev->addr.devid, pci_dev->addr.function);
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- if (nd == 1) {
- /*
- * Found single matching device may have multiple ports.
- * Each port may be representor, we have to check the port
- * number and check the representors existence.
- */
- if (nl_rdma >= 0)
- np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);
- if (!np)
- DRV_LOG(WARNING, "can not get IB device \"%s\""
- " ports number", ibv_match[0]->name);
- if (bd >= 0 && !np) {
- DRV_LOG(ERR, "can not get ports"
- " for bonding device");
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- }
-#ifndef HAVE_MLX5DV_DR_DEVX_PORT
- if (bd >= 0) {
- /*
- * This may happen if there is VF LAG kernel support and
- * application is compiled with older rdma_core library.
- */
- DRV_LOG(ERR,
- "No kernel/verbs support for VF LAG bonding found.");
- rte_errno = ENOTSUP;
- ret = -rte_errno;
- goto exit;
- }
-#endif
- /*
- * Now we can determine the maximal
- * amount of devices to be spawned.
- */
- list = rte_zmalloc("device spawn data",
- sizeof(struct mlx5_dev_spawn_data) *
- (np ? np : nd),
- RTE_CACHE_LINE_SIZE);
- if (!list) {
- DRV_LOG(ERR, "spawn data array allocation failure");
- rte_errno = ENOMEM;
- ret = -rte_errno;
- goto exit;
- }
- if (bd >= 0 || np > 1) {
- /*
- * Single IB device with multiple ports found,
- * it may be E-Switch master device and representors.
- * We have to perform identification trough the ports.
- */
- assert(nl_rdma >= 0);
- assert(ns == 0);
- assert(nd == 1);
- assert(np);
- for (i = 1; i <= np; ++i) {
- list[ns].max_port = np;
- list[ns].ibv_port = i;
- list[ns].ibv_dev = ibv_match[0];
- list[ns].eth_dev = NULL;
- list[ns].pci_dev = pci_dev;
- list[ns].pf_bond = bd;
- list[ns].ifindex = mlx5_nl_ifindex
- (nl_rdma, list[ns].ibv_dev->name, i);
- if (!list[ns].ifindex) {
- /*
- * No network interface index found for the
- * specified port, it means there is no
- * representor on this port. It's OK,
- * there can be disabled ports, for example
- * if sriov_numvfs < sriov_totalvfs.
- */
- continue;
- }
- ret = -1;
- if (nl_route >= 0)
- ret = mlx5_nl_switch_info
- (nl_route,
- list[ns].ifindex,
- &list[ns].info);
- if (ret || (!list[ns].info.representor &&
- !list[ns].info.master)) {
- /*
- * We failed to recognize representors with
- * Netlink, let's try to perform the task
- * with sysfs.
- */
- ret = mlx5_sysfs_switch_info
- (list[ns].ifindex,
- &list[ns].info);
- }
- if (!ret && bd >= 0) {
- switch (list[ns].info.name_type) {
- case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
- if (list[ns].info.port_name == bd)
- ns++;
- break;
- case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
- if (list[ns].info.pf_num == bd)
- ns++;
- break;
- default:
- break;
- }
- continue;
- }
- if (!ret && (list[ns].info.representor ^
- list[ns].info.master))
- ns++;
- }
- if (!ns) {
- DRV_LOG(ERR,
- "unable to recognize master/representors"
- " on the IB device with multiple ports");
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- } else {
- /*
- * The existence of several matching entries (nd > 1) means
- * port representors have been instantiated. No existing Verbs
- * call nor sysfs entries can tell them apart, this can only
- * be done through Netlink calls assuming kernel drivers are
- * recent enough to support them.
- *
- * In the event of identification failure through Netlink,
- * try again through sysfs, then:
- *
- * 1. A single IB device matches (nd == 1) with single
- * port (np=0/1) and is not a representor, assume
- * no switch support.
- *
- * 2. Otherwise no safe assumptions can be made;
- * complain louder and bail out.
- */
- np = 1;
- for (i = 0; i != nd; ++i) {
- memset(&list[ns].info, 0, sizeof(list[ns].info));
- list[ns].max_port = 1;
- list[ns].ibv_port = 1;
- list[ns].ibv_dev = ibv_match[i];
- list[ns].eth_dev = NULL;
- list[ns].pci_dev = pci_dev;
- list[ns].pf_bond = -1;
- list[ns].ifindex = 0;
- if (nl_rdma >= 0)
- list[ns].ifindex = mlx5_nl_ifindex
- (nl_rdma, list[ns].ibv_dev->name, 1);
- if (!list[ns].ifindex) {
- char ifname[IF_NAMESIZE];
-
- /*
- * Netlink failed, it may happen with old
- * ib_core kernel driver (before 4.16).
- * We can assume there is old driver because
- * here we are processing single ports IB
- * devices. Let's try sysfs to retrieve
- * the ifindex. The method works for
- * master device only.
- */
- if (nd > 1) {
- /*
- * Multiple devices found, assume
- * representors, can not distinguish
- * master/representor and retrieve
- * ifindex via sysfs.
- */
- continue;
- }
- ret = mlx5_get_master_ifname
- (ibv_match[i]->ibdev_path, &ifname);
- if (!ret)
- list[ns].ifindex =
- if_nametoindex(ifname);
- if (!list[ns].ifindex) {
- /*
- * No network interface index found
- * for the specified device, it means
- * there it is neither representor
- * nor master.
- */
- continue;
- }
- }
- ret = -1;
- if (nl_route >= 0)
- ret = mlx5_nl_switch_info
- (nl_route,
- list[ns].ifindex,
- &list[ns].info);
- if (ret || (!list[ns].info.representor &&
- !list[ns].info.master)) {
- /*
- * We failed to recognize representors with
- * Netlink, let's try to perform the task
- * with sysfs.
- */
- ret = mlx5_sysfs_switch_info
- (list[ns].ifindex,
- &list[ns].info);
- }
- if (!ret && (list[ns].info.representor ^
- list[ns].info.master)) {
- ns++;
- } else if ((nd == 1) &&
- !list[ns].info.representor &&
- !list[ns].info.master) {
- /*
- * Single IB device with
- * one physical port and
- * attached network device.
- * May be SRIOV is not enabled
- * or there is no representors.
- */
- DRV_LOG(INFO, "no E-Switch support detected");
- ns++;
- break;
- }
- }
- if (!ns) {
- DRV_LOG(ERR,
- "unable to recognize master/representors"
- " on the multiple IB devices");
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- }
- assert(ns);
- /*
- * Sort list to probe devices in natural order for users convenience
- * (i.e. master first, then representors from lowest to highest ID).
- */
- qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);
- /* Default configuration. */
- dev_config = (struct mlx5_dev_config){
- .hw_padding = 0,
- .mps = MLX5_ARG_UNSET,
- .dbnc = MLX5_ARG_UNSET,
- .rx_vec_en = 1,
- .txq_inline_max = MLX5_ARG_UNSET,
- .txq_inline_min = MLX5_ARG_UNSET,
- .txq_inline_mpw = MLX5_ARG_UNSET,
- .txqs_inline = MLX5_ARG_UNSET,
- .vf_nl_en = 1,
- .mr_ext_memseg_en = 1,
- .mprq = {
- .enabled = 0, /* Disabled by default. */
- .stride_num_n = MLX5_MPRQ_STRIDE_NUM_N,
- .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
- .min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
- },
- .dv_esw_en = 1,
- };
- /* Device specific configuration. */
- switch (pci_dev->id.device_id) {
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF:
- dev_config.vf = 1;
- break;
- default:
- break;
- }
- for (i = 0; i != ns; ++i) {
- uint32_t restore;
-
- list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
- &list[i],
- dev_config);
- if (!list[i].eth_dev) {
- if (rte_errno != EBUSY && rte_errno != EEXIST)
- break;
- /* Device is disabled or already spawned. Ignore it. */
- continue;
- }
- restore = list[i].eth_dev->data->dev_flags;
- rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
- /* Restore non-PCI flags cleared by the above call. */
- list[i].eth_dev->data->dev_flags |= restore;
- mlx5_dev_interrupt_handler_devx_install(list[i].eth_dev);
- rte_eth_dev_probing_finish(list[i].eth_dev);
- }
- if (i != ns) {
- DRV_LOG(ERR,
- "probe of PCI device " PCI_PRI_FMT " aborted after"
- " encountering an error: %s",
- pci_dev->addr.domain, pci_dev->addr.bus,
- pci_dev->addr.devid, pci_dev->addr.function,
- strerror(rte_errno));
- ret = -rte_errno;
- /* Roll back. */
- while (i--) {
- if (!list[i].eth_dev)
- continue;
- mlx5_dev_close(list[i].eth_dev);
- /* mac_addrs must not be freed because in dev_private */
- list[i].eth_dev->data->mac_addrs = NULL;
- claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
- }
- /* Restore original error. */
- rte_errno = -ret;
- } else {
- ret = 0;
- }
-exit:
- /*
- * Do the routine cleanup:
- * - close opened Netlink sockets
- * - free allocated spawn data array
- * - free the Infiniband device list
- */
- if (nl_rdma >= 0)
- close(nl_rdma);
- if (nl_route >= 0)
- close(nl_route);
- if (list)
- rte_free(list);
- assert(ibv_list);
- mlx5_glue->free_device_list(ibv_list);
- return ret;
-}
-
-/**
- * Look for the ethernet device belonging to mlx5 driver.
- *
- * @param[in] port_id
- * port_id to start looking for device.
- * @param[in] pci_dev
- * Pointer to the hint PCI device. When device is being probed
- * the its siblings (master and preceding representors might
- * not have assigned driver yet (because the mlx5_pci_probe()
- * is not completed yet, for this case match on hint PCI
- * device may be used to detect sibling device.
- *
- * @return
- * port_id of found device, RTE_MAX_ETHPORT if not found.
- */
-uint16_t
-mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev)
-{
- while (port_id < RTE_MAX_ETHPORTS) {
- struct rte_eth_dev *dev = &rte_eth_devices[port_id];
-
- if (dev->state != RTE_ETH_DEV_UNUSED &&
- dev->device &&
- (dev->device == &pci_dev->device ||
- (dev->device->driver &&
- dev->device->driver->name &&
- !strcmp(dev->device->driver->name, MLX5_DRIVER_NAME))))
- break;
- port_id++;
- }
- if (port_id >= RTE_MAX_ETHPORTS)
- return RTE_MAX_ETHPORTS;
- return port_id;
-}
-
-/**
- * DPDK callback to remove a PCI device.
- *
- * This function removes all Ethernet devices belong to a given PCI device.
- *
- * @param[in] pci_dev
- * Pointer to the PCI device.
- *
- * @return
- * 0 on success, the function cannot fail.
- */
-static int
-mlx5_pci_remove(struct rte_pci_device *pci_dev)
-{
- uint16_t port_id;
-
- RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
- rte_eth_dev_close(port_id);
- return 0;