- priv->config.flow_prio = err;
- if (!priv->config.dv_esw_en &&
- priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
- DRV_LOG(WARNING, "metadata mode %u is not supported "
- "(no E-Switch)", priv->config.dv_xmeta_en);
- priv->config.dv_xmeta_en = MLX5_XMETA_MODE_LEGACY;
- }
- mlx5_set_metadata_mask(eth_dev);
- if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
- !priv->sh->dv_regc0_mask) {
- DRV_LOG(ERR, "metadata mode %u is not supported "
- "(no metadata reg_c[0] is available)",
- priv->config.dv_xmeta_en);
- err = ENOTSUP;
- goto error;
- }
- /* Query availibility of metadata reg_c's. */
- err = mlx5_flow_discover_mreg_c(eth_dev);
- if (err < 0) {
- err = -err;
- goto error;
- }
- if (!mlx5_flow_ext_mreg_supported(eth_dev)) {
- DRV_LOG(DEBUG,
- "port %u extensive metadata register is not supported",
- eth_dev->data->port_id);
- if (priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY) {
- DRV_LOG(ERR, "metadata mode %u is not supported "
- "(no metadata registers available)",
- priv->config.dv_xmeta_en);
- err = ENOTSUP;
- goto error;
- }
- }
- if (priv->config.dv_flow_en &&
- priv->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
- mlx5_flow_ext_mreg_supported(eth_dev) &&
- priv->sh->dv_regc0_mask) {
- priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
- MLX5_FLOW_MREG_HTABLE_SZ);
- if (!priv->mreg_cp_tbl) {
- err = ENOMEM;
- goto error;
- }
- }
- return eth_dev;
-error:
- if (priv) {
- if (priv->mreg_cp_tbl)
- mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
- if (priv->sh)
- mlx5_free_shared_dr(priv);
- if (priv->nl_socket_route >= 0)
- close(priv->nl_socket_route);
- if (priv->nl_socket_rdma >= 0)
- close(priv->nl_socket_rdma);
- if (priv->vmwa_context)
- mlx5_vlan_vmwa_exit(priv->vmwa_context);
- if (priv->qrss_id_pool)
- mlx5_flow_id_pool_release(priv->qrss_id_pool);
- if (own_domain_id)
- claim_zero(rte_eth_switch_domain_free(priv->domain_id));
- rte_free(priv);
- if (eth_dev != NULL)
- eth_dev->data->dev_private = NULL;
- }
- if (eth_dev != NULL) {
- /* mac_addrs must not be freed alone because part of dev_private */
- eth_dev->data->mac_addrs = NULL;
- rte_eth_dev_release_port(eth_dev);
- }
- if (sh)
- mlx5_free_shared_ibctx(sh);
- assert(err > 0);
- rte_errno = err;
- return NULL;
-}
-
-/**
- * Comparison callback to sort device data.
- *
- * This is meant to be used with qsort().
- *
- * @param a[in]
- * Pointer to pointer to first data object.
- * @param b[in]
- * Pointer to pointer to second data object.
- *
- * @return
- * 0 if both objects are equal, less than 0 if the first argument is less
- * than the second, greater than 0 otherwise.
- */
-static int
-mlx5_dev_spawn_data_cmp(const void *a, const void *b)
-{
- const struct mlx5_switch_info *si_a =
- &((const struct mlx5_dev_spawn_data *)a)->info;
- const struct mlx5_switch_info *si_b =
- &((const struct mlx5_dev_spawn_data *)b)->info;
- int ret;
-
- /* Master device first. */
- ret = si_b->master - si_a->master;
- if (ret)
- return ret;
- /* Then representor devices. */
- ret = si_b->representor - si_a->representor;
- if (ret)
- return ret;
- /* Unidentified devices come last in no specific order. */
- if (!si_a->representor)
- return 0;
- /* Order representors by name. */
- return si_a->port_name - si_b->port_name;
-}
-
-/**
- * Match PCI information for possible slaves of bonding device.
- *
- * @param[in] ibv_dev
- * Pointer to Infiniband device structure.
- * @param[in] pci_dev
- * Pointer to PCI device structure to match PCI address.
- * @param[in] nl_rdma
- * Netlink RDMA group socket handle.
- *
- * @return
- * negative value if no bonding device found, otherwise
- * positive index of slave PF in bonding.
- */
-static int
-mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev,
- const struct rte_pci_device *pci_dev,
- int nl_rdma)
-{
- char ifname[IF_NAMESIZE + 1];
- unsigned int ifindex;
- unsigned int np, i;
- FILE *file = NULL;
- int pf = -1;
-
- /*
- * Try to get master device name. If something goes
- * wrong suppose the lack of kernel support and no
- * bonding devices.
- */
- if (nl_rdma < 0)
- return -1;
- if (!strstr(ibv_dev->name, "bond"))
- return -1;
- np = mlx5_nl_portnum(nl_rdma, ibv_dev->name);
- if (!np)
- return -1;
- /*
- * The Master device might not be on the predefined
- * port (not on port index 1, it is not garanted),
- * we have to scan all Infiniband device port and
- * find master.
- */
- for (i = 1; i <= np; ++i) {
- /* Check whether Infiniband port is populated. */
- ifindex = mlx5_nl_ifindex(nl_rdma, ibv_dev->name, i);
- if (!ifindex)
- continue;
- if (!if_indextoname(ifindex, ifname))
- continue;
- /* Try to read bonding slave names from sysfs. */
- MKSTR(slaves,
- "/sys/class/net/%s/master/bonding/slaves", ifname);
- file = fopen(slaves, "r");
- if (file)
- break;
- }
- if (!file)
- return -1;
- /* Use safe format to check maximal buffer length. */
- assert(atol(RTE_STR(IF_NAMESIZE)) == IF_NAMESIZE);
- while (fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", ifname) == 1) {
- char tmp_str[IF_NAMESIZE + 32];
- struct rte_pci_addr pci_addr;
- struct mlx5_switch_info info;
-
- /* Process slave interface names in the loop. */
- snprintf(tmp_str, sizeof(tmp_str),
- "/sys/class/net/%s", ifname);
- if (mlx5_dev_to_pci_addr(tmp_str, &pci_addr)) {
- DRV_LOG(WARNING, "can not get PCI address"
- " for netdev \"%s\"", ifname);
- continue;
- }
- if (pci_dev->addr.domain != pci_addr.domain ||
- pci_dev->addr.bus != pci_addr.bus ||
- pci_dev->addr.devid != pci_addr.devid ||
- pci_dev->addr.function != pci_addr.function)
- continue;
- /* Slave interface PCI address match found. */
- fclose(file);
- snprintf(tmp_str, sizeof(tmp_str),
- "/sys/class/net/%s/phys_port_name", ifname);
- file = fopen(tmp_str, "rb");
- if (!file)
- break;
- info.name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET;
- if (fscanf(file, "%32s", tmp_str) == 1)
- mlx5_translate_port_name(tmp_str, &info);
- if (info.name_type == MLX5_PHYS_PORT_NAME_TYPE_LEGACY ||
- info.name_type == MLX5_PHYS_PORT_NAME_TYPE_UPLINK)
- pf = info.port_name;
- break;
- }
- if (file)
- fclose(file);
- return pf;
-}
-
-/**
- * DPDK callback to register a PCI device.
- *
- * This function spawns Ethernet devices out of a given PCI device.
- *
- * @param[in] pci_drv
- * PCI driver structure (mlx5_driver).
- * @param[in] pci_dev
- * PCI device information.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
- struct rte_pci_device *pci_dev)
-{
- struct ibv_device **ibv_list;
- /*
- * Number of found IB Devices matching with requested PCI BDF.
- * nd != 1 means there are multiple IB devices over the same
- * PCI device and we have representors and master.
- */
- unsigned int nd = 0;
- /*
- * Number of found IB device Ports. nd = 1 and np = 1..n means
- * we have the single multiport IB device, and there may be
- * representors attached to some of found ports.
- */
- unsigned int np = 0;
- /*
- * Number of DPDK ethernet devices to Spawn - either over
- * multiple IB devices or multiple ports of single IB device.
- * Actually this is the number of iterations to spawn.
- */
- unsigned int ns = 0;
- /*
- * Bonding device
- * < 0 - no bonding device (single one)
- * >= 0 - bonding device (value is slave PF index)
- */
- int bd = -1;
- struct mlx5_dev_spawn_data *list = NULL;
- struct mlx5_dev_config dev_config;
- int ret;
-
- if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_NET) {
- DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5"
- " driver.");
- return 1;
- }
- if (rte_eal_process_type() == RTE_PROC_PRIMARY)
- mlx5_pmd_socket_init();
- ret = mlx5_init_once();
- if (ret) {
- DRV_LOG(ERR, "unable to init PMD global data: %s",
- strerror(rte_errno));
- return -rte_errno;
- }
- assert(pci_drv == &mlx5_driver);
- errno = 0;
- ibv_list = mlx5_glue->get_device_list(&ret);
- if (!ibv_list) {
- rte_errno = errno ? errno : ENOSYS;
- DRV_LOG(ERR, "cannot list devices, is ib_uverbs loaded?");
- return -rte_errno;
- }
- /*
- * First scan the list of all Infiniband devices to find
- * matching ones, gathering into the list.
- */
- struct ibv_device *ibv_match[ret + 1];
- int nl_route = mlx5_nl_init(NETLINK_ROUTE);
- int nl_rdma = mlx5_nl_init(NETLINK_RDMA);
- unsigned int i;
-
- while (ret-- > 0) {
- struct rte_pci_addr pci_addr;
-
- DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name);
- bd = mlx5_device_bond_pci_match
- (ibv_list[ret], pci_dev, nl_rdma);
- if (bd >= 0) {
- /*
- * Bonding device detected. Only one match is allowed,
- * the bonding is supported over multi-port IB device,
- * there should be no matches on representor PCI
- * functions or non VF LAG bonding devices with
- * specified address.
- */
- if (nd) {
- DRV_LOG(ERR,
- "multiple PCI match on bonding device"
- "\"%s\" found", ibv_list[ret]->name);
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- DRV_LOG(INFO, "PCI information matches for"
- " slave %d bonding device \"%s\"",
- bd, ibv_list[ret]->name);
- ibv_match[nd++] = ibv_list[ret];
- break;
- }
- if (mlx5_dev_to_pci_addr
- (ibv_list[ret]->ibdev_path, &pci_addr))
- continue;
- if (pci_dev->addr.domain != pci_addr.domain ||
- pci_dev->addr.bus != pci_addr.bus ||
- pci_dev->addr.devid != pci_addr.devid ||
- pci_dev->addr.function != pci_addr.function)
- continue;
- DRV_LOG(INFO, "PCI information matches for device \"%s\"",
- ibv_list[ret]->name);
- ibv_match[nd++] = ibv_list[ret];
- }
- ibv_match[nd] = NULL;
- if (!nd) {
- /* No device matches, just complain and bail out. */
- DRV_LOG(WARNING,
- "no Verbs device matches PCI device " PCI_PRI_FMT ","
- " are kernel drivers loaded?",
- pci_dev->addr.domain, pci_dev->addr.bus,
- pci_dev->addr.devid, pci_dev->addr.function);
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- if (nd == 1) {
- /*
- * Found single matching device may have multiple ports.
- * Each port may be representor, we have to check the port
- * number and check the representors existence.
- */
- if (nl_rdma >= 0)
- np = mlx5_nl_portnum(nl_rdma, ibv_match[0]->name);
- if (!np)
- DRV_LOG(WARNING, "can not get IB device \"%s\""
- " ports number", ibv_match[0]->name);
- if (bd >= 0 && !np) {
- DRV_LOG(ERR, "can not get ports"
- " for bonding device");
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- }
-#ifndef HAVE_MLX5DV_DR_DEVX_PORT
- if (bd >= 0) {
- /*
- * This may happen if there is VF LAG kernel support and
- * application is compiled with older rdma_core library.
- */
- DRV_LOG(ERR,
- "No kernel/verbs support for VF LAG bonding found.");
- rte_errno = ENOTSUP;
- ret = -rte_errno;
- goto exit;
- }
-#endif
- /*
- * Now we can determine the maximal
- * amount of devices to be spawned.
- */
- list = rte_zmalloc("device spawn data",
- sizeof(struct mlx5_dev_spawn_data) *
- (np ? np : nd),
- RTE_CACHE_LINE_SIZE);
- if (!list) {
- DRV_LOG(ERR, "spawn data array allocation failure");
- rte_errno = ENOMEM;
- ret = -rte_errno;
- goto exit;
- }
- if (bd >= 0 || np > 1) {
- /*
- * Single IB device with multiple ports found,
- * it may be E-Switch master device and representors.
- * We have to perform identification trough the ports.
- */
- assert(nl_rdma >= 0);
- assert(ns == 0);
- assert(nd == 1);
- assert(np);
- for (i = 1; i <= np; ++i) {
- list[ns].max_port = np;
- list[ns].ibv_port = i;
- list[ns].ibv_dev = ibv_match[0];
- list[ns].eth_dev = NULL;
- list[ns].pci_dev = pci_dev;
- list[ns].pf_bond = bd;
- list[ns].ifindex = mlx5_nl_ifindex
- (nl_rdma, list[ns].ibv_dev->name, i);
- if (!list[ns].ifindex) {
- /*
- * No network interface index found for the
- * specified port, it means there is no
- * representor on this port. It's OK,
- * there can be disabled ports, for example
- * if sriov_numvfs < sriov_totalvfs.
- */
- continue;
- }
- ret = -1;
- if (nl_route >= 0)
- ret = mlx5_nl_switch_info
- (nl_route,
- list[ns].ifindex,
- &list[ns].info);
- if (ret || (!list[ns].info.representor &&
- !list[ns].info.master)) {
- /*
- * We failed to recognize representors with
- * Netlink, let's try to perform the task
- * with sysfs.
- */
- ret = mlx5_sysfs_switch_info
- (list[ns].ifindex,
- &list[ns].info);
- }
- if (!ret && bd >= 0) {
- switch (list[ns].info.name_type) {
- case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
- if (list[ns].info.port_name == bd)
- ns++;
- break;
- case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
- if (list[ns].info.pf_num == bd)
- ns++;
- break;
- default:
- break;
- }
- continue;
- }
- if (!ret && (list[ns].info.representor ^
- list[ns].info.master))
- ns++;
- }
- if (!ns) {
- DRV_LOG(ERR,
- "unable to recognize master/representors"
- " on the IB device with multiple ports");
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- } else {
- /*
- * The existence of several matching entries (nd > 1) means
- * port representors have been instantiated. No existing Verbs
- * call nor sysfs entries can tell them apart, this can only
- * be done through Netlink calls assuming kernel drivers are
- * recent enough to support them.
- *
- * In the event of identification failure through Netlink,
- * try again through sysfs, then:
- *
- * 1. A single IB device matches (nd == 1) with single
- * port (np=0/1) and is not a representor, assume
- * no switch support.
- *
- * 2. Otherwise no safe assumptions can be made;
- * complain louder and bail out.
- */
- np = 1;
- for (i = 0; i != nd; ++i) {
- memset(&list[ns].info, 0, sizeof(list[ns].info));
- list[ns].max_port = 1;
- list[ns].ibv_port = 1;
- list[ns].ibv_dev = ibv_match[i];
- list[ns].eth_dev = NULL;
- list[ns].pci_dev = pci_dev;
- list[ns].pf_bond = -1;
- list[ns].ifindex = 0;
- if (nl_rdma >= 0)
- list[ns].ifindex = mlx5_nl_ifindex
- (nl_rdma, list[ns].ibv_dev->name, 1);
- if (!list[ns].ifindex) {
- char ifname[IF_NAMESIZE];
-
- /*
- * Netlink failed, it may happen with old
- * ib_core kernel driver (before 4.16).
- * We can assume there is old driver because
- * here we are processing single ports IB
- * devices. Let's try sysfs to retrieve
- * the ifindex. The method works for
- * master device only.
- */
- if (nd > 1) {
- /*
- * Multiple devices found, assume
- * representors, can not distinguish
- * master/representor and retrieve
- * ifindex via sysfs.
- */
- continue;
- }
- ret = mlx5_get_master_ifname
- (ibv_match[i]->ibdev_path, &ifname);
- if (!ret)
- list[ns].ifindex =
- if_nametoindex(ifname);
- if (!list[ns].ifindex) {
- /*
- * No network interface index found
- * for the specified device, it means
- * there it is neither representor
- * nor master.
- */
- continue;
- }
- }
- ret = -1;
- if (nl_route >= 0)
- ret = mlx5_nl_switch_info
- (nl_route,
- list[ns].ifindex,
- &list[ns].info);
- if (ret || (!list[ns].info.representor &&
- !list[ns].info.master)) {
- /*
- * We failed to recognize representors with
- * Netlink, let's try to perform the task
- * with sysfs.
- */
- ret = mlx5_sysfs_switch_info
- (list[ns].ifindex,
- &list[ns].info);
- }
- if (!ret && (list[ns].info.representor ^
- list[ns].info.master)) {
- ns++;
- } else if ((nd == 1) &&
- !list[ns].info.representor &&
- !list[ns].info.master) {
- /*
- * Single IB device with
- * one physical port and
- * attached network device.
- * May be SRIOV is not enabled
- * or there is no representors.
- */
- DRV_LOG(INFO, "no E-Switch support detected");
- ns++;
- break;
- }
- }
- if (!ns) {
- DRV_LOG(ERR,
- "unable to recognize master/representors"
- " on the multiple IB devices");
- rte_errno = ENOENT;
- ret = -rte_errno;
- goto exit;
- }
- }
- assert(ns);
- /*
- * Sort list to probe devices in natural order for users convenience
- * (i.e. master first, then representors from lowest to highest ID).
- */
- qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp);
- /* Default configuration. */
- dev_config = (struct mlx5_dev_config){
- .hw_padding = 0,
- .mps = MLX5_ARG_UNSET,
- .dbnc = MLX5_ARG_UNSET,
- .rx_vec_en = 1,
- .txq_inline_max = MLX5_ARG_UNSET,
- .txq_inline_min = MLX5_ARG_UNSET,
- .txq_inline_mpw = MLX5_ARG_UNSET,
- .txqs_inline = MLX5_ARG_UNSET,
- .vf_nl_en = 1,
- .mr_ext_memseg_en = 1,
- .mprq = {
- .enabled = 0, /* Disabled by default. */
- .stride_num_n = MLX5_MPRQ_STRIDE_NUM_N,
- .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN,
- .min_rxqs_num = MLX5_MPRQ_MIN_RXQS,
- },
- .dv_esw_en = 1,
- .dv_flow_en = 1,
- };
- /* Device specific configuration. */
- switch (pci_dev->id.device_id) {
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF:
- case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF:
- dev_config.vf = 1;
- break;
- default:
- break;
- }
- for (i = 0; i != ns; ++i) {
- uint32_t restore;
-
- list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
- &list[i],
- dev_config);
- if (!list[i].eth_dev) {
- if (rte_errno != EBUSY && rte_errno != EEXIST)
- break;
- /* Device is disabled or already spawned. Ignore it. */
- continue;
- }
- restore = list[i].eth_dev->data->dev_flags;
- rte_eth_copy_pci_info(list[i].eth_dev, pci_dev);
- /* Restore non-PCI flags cleared by the above call. */
- list[i].eth_dev->data->dev_flags |= restore;
- mlx5_dev_interrupt_handler_devx_install(list[i].eth_dev);
- rte_eth_dev_probing_finish(list[i].eth_dev);
- }
- if (i != ns) {
- DRV_LOG(ERR,
- "probe of PCI device " PCI_PRI_FMT " aborted after"
- " encountering an error: %s",
- pci_dev->addr.domain, pci_dev->addr.bus,
- pci_dev->addr.devid, pci_dev->addr.function,
- strerror(rte_errno));
- ret = -rte_errno;
- /* Roll back. */
- while (i--) {
- if (!list[i].eth_dev)
- continue;
- mlx5_dev_close(list[i].eth_dev);
- /* mac_addrs must not be freed because in dev_private */
- list[i].eth_dev->data->mac_addrs = NULL;
- claim_zero(rte_eth_dev_release_port(list[i].eth_dev));
- }
- /* Restore original error. */
- rte_errno = -ret;
- } else {
- ret = 0;
- }
-exit:
- /*
- * Do the routine cleanup:
- * - close opened Netlink sockets
- * - free allocated spawn data array
- * - free the Infiniband device list
- */
- if (nl_rdma >= 0)
- close(nl_rdma);
- if (nl_route >= 0)
- close(nl_route);
- if (list)
- rte_free(list);
- assert(ibv_list);
- mlx5_glue->free_device_list(ibv_list);
- return ret;
-}
-
-/**
- * Look for the ethernet device belonging to mlx5 driver.
- *
- * @param[in] port_id
- * port_id to start looking for device.
- * @param[in] pci_dev
- * Pointer to the hint PCI device. When device is being probed
- * the its siblings (master and preceding representors might
- * not have assigned driver yet (because the mlx5_pci_probe()
- * is not completed yet, for this case match on hint PCI
- * device may be used to detect sibling device.
- *
- * @return
- * port_id of found device, RTE_MAX_ETHPORT if not found.
- */
-uint16_t
-mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev)
-{
- while (port_id < RTE_MAX_ETHPORTS) {
- struct rte_eth_dev *dev = &rte_eth_devices[port_id];
-
- if (dev->state != RTE_ETH_DEV_UNUSED &&
- dev->device &&
- (dev->device == &pci_dev->device ||
- (dev->device->driver &&
- dev->device->driver->name &&
- !strcmp(dev->device->driver->name, MLX5_DRIVER_NAME))))
- break;
- port_id++;
- }
- if (port_id >= RTE_MAX_ETHPORTS)
- return RTE_MAX_ETHPORTS;
- return port_id;
-}
-
-/**
- * DPDK callback to remove a PCI device.
- *
- * This function removes all Ethernet devices belong to a given PCI device.
- *
- * @param[in] pci_dev
- * Pointer to the PCI device.
- *
- * @return
- * 0 on success, the function cannot fail.
- */
-static int
-mlx5_pci_remove(struct rte_pci_device *pci_dev)
-{
- uint16_t port_id;
-
- RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
- rte_eth_dev_close(port_id);