From: Xueming Li Date: Sun, 28 Mar 2021 13:48:10 +0000 (+0000) Subject: net/mlx5: refactor bonding representor probing X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=f926cce3fa94298cdbed88c6bb37adfd0ca87a54;p=dpdk.git net/mlx5: refactor bonding representor probing To probe representor on 2nd PF of kernel bonding device, had to specify PF1 BDF in devarg: ,representor=0 When closing bonding device, all representors had to be closed together and this implies all representors have to use primary PF of bonding device. So after probing representor port on 2nd PF, when locating new probed device using device argument, the filter used 2nd PF as PCI address and failed to locate new device. Conflict happened by using current representor devargs: - Use PCI BDF to specify representor owner PF - Use PCI BDF to locate probed representor device. - PMD uses primary PCI BDF as PCI device. To resolve such conflicts, new representor syntax is introduced here: ,representor=pfXvfY All representors must use primary PF as owner PCI device, PMD internally locate owner PCI address by checking representor "pfX" part. To EAL, all representors are registered to primary PCI device, the 2nd PF is hidden to EAL, thus all search should be consistent. Same to VF representor, HPF (host PF on BlueField) uses same syntax to probe, example: representor=pf1vf[0-3,-1] This patch also adds pf index into kernel bonding representor port name: __representor_pfvf Signed-off-by: Xueming Li Acked-by: Viacheslav Ovsiienko --- diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst index 2e2909d82d..92fe7c11e4 100644 --- a/doc/guides/nics/mlx5.rst +++ b/doc/guides/nics/mlx5.rst @@ -938,11 +938,11 @@ Driver options For instance, to probe VF port representors 0 through 2:: - representor=vf[0-2] + ,representor=vf[0-2] To probe SF port representors 0 through 2:: - representor=sf[0-2] + ,representor=sf[0-2] - ``max_dump_files_num`` parameter [int] diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 56ab153164..40ad9b726a 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -692,6 +692,71 @@ mlx5_queue_counter_id_prepare(struct rte_eth_dev *dev) "available.", dev->data->port_id); } +/** + * Check if representor spawn info match devargs. + * + * @param spawn + * Verbs device parameters (name, port, switch_info) to spawn. + * @param eth_da + * Device devargs to probe. + * + * @return + * Match result. + */ +static bool +mlx5_representor_match(struct mlx5_dev_spawn_data *spawn, + struct rte_eth_devargs *eth_da) +{ + struct mlx5_switch_info *switch_info = &spawn->info; + unsigned int p, f; + uint16_t id; + uint16_t repr_id = mlx5_representor_id_encode(switch_info); + + switch (eth_da->type) { + case RTE_ETH_REPRESENTOR_SF: + if (switch_info->name_type != MLX5_PHYS_PORT_NAME_TYPE_PFSF) { + rte_errno = EBUSY; + return false; + } + break; + case RTE_ETH_REPRESENTOR_VF: + /* Allows HPF representor index -1 as exception. */ + if (!(spawn->info.port_name == -1 && + switch_info->name_type == + MLX5_PHYS_PORT_NAME_TYPE_PFHPF) && + switch_info->name_type != MLX5_PHYS_PORT_NAME_TYPE_PFVF) { + rte_errno = EBUSY; + return false; + } + break; + case RTE_ETH_REPRESENTOR_NONE: + rte_errno = EBUSY; + return false; + default: + rte_errno = ENOTSUP; + DRV_LOG(ERR, "unsupported representor type"); + return false; + } + /* Check representor ID: */ + for (p = 0; p < eth_da->nb_ports; ++p) { + if (spawn->pf_bond < 0) { + /* For non-LAG mode, allow and ignore pf. */ + switch_info->pf_num = eth_da->ports[p]; + repr_id = mlx5_representor_id_encode(switch_info); + } + for (f = 0; f < eth_da->nb_representor_ports; ++f) { + id = MLX5_REPRESENTOR_ID + (eth_da->ports[p], eth_da->type, + eth_da->representor_ports[f]); + if (repr_id == id) + return true; + } + } + rte_errno = EBUSY; + return false; +} + + /** * Spawn an Ethernet device from Verbs information. * @@ -738,115 +803,44 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, char name[RTE_ETH_NAME_MAX_LEN]; int own_domain_id = 0; uint16_t port_id; - unsigned int i; #ifdef HAVE_MLX5DV_DR_DEVX_PORT struct mlx5dv_devx_port devx_port = { .comp_mask = 0 }; #endif /* Determine if this port representor is supposed to be spawned. */ - if (switch_info->representor && dpdk_dev->devargs) { - switch (eth_da->type) { - case RTE_ETH_REPRESENTOR_SF: - if (switch_info->name_type != - MLX5_PHYS_PORT_NAME_TYPE_PFSF) { - rte_errno = EBUSY; - return NULL; - } - break; - case RTE_ETH_REPRESENTOR_VF: - /* Allows HPF representor index -1 as exception. */ - if (!(spawn->info.port_name == -1 && - switch_info->name_type == - MLX5_PHYS_PORT_NAME_TYPE_PFHPF) && - switch_info->name_type != - MLX5_PHYS_PORT_NAME_TYPE_PFVF) { - rte_errno = EBUSY; - return NULL; - } - break; - case RTE_ETH_REPRESENTOR_NONE: - rte_errno = EBUSY; - return NULL; - break; - default: - rte_errno = ENOTSUP; - DRV_LOG(ERR, "unsupported representor type: %s", - dpdk_dev->devargs->args); - return NULL; - } - /* Check controller ID: */ - for (i = 0; i < eth_da->nb_mh_controllers; ++i) - if (eth_da->mh_controllers[i] == - (uint16_t)switch_info->ctrl_num) - break; - if (eth_da->nb_mh_controllers && - i == eth_da->nb_mh_controllers) { - rte_errno = EBUSY; - return NULL; - } - /* Check SF/VF ID: */ - for (i = 0; i < eth_da->nb_representor_ports; ++i) - if (eth_da->representor_ports[i] == - (uint16_t)switch_info->port_name) - break; - if (eth_da->type != RTE_ETH_REPRESENTOR_PF && - i == eth_da->nb_representor_ports) { - rte_errno = EBUSY; - return NULL; - } - /* Check PF ID. Check after repr port to avoid warning flood. */ - if (spawn->pf_bond >= 0) { - for (i = 0; i < eth_da->nb_ports; ++i) - if (eth_da->ports[i] == - (uint16_t)switch_info->pf_num) - break; - if (eth_da->nb_ports && i == eth_da->nb_ports) { - /* For backward compatibility, bonding - * representor syntax supported with limitation, - * device iterator won't find it: - * ,representor=# - */ - if (switch_info->pf_num > 0 && - eth_da->ports[0] == 0) { - DRV_LOG(WARNING, "Representor on Bonding PF should use pf#vf# format: %s", - dpdk_dev->devargs->args); - } else { - rte_errno = EBUSY; - return NULL; - } - } - } else if (eth_da->nb_ports > 1 || eth_da->ports[0]) { - rte_errno = EINVAL; - DRV_LOG(ERR, "PF id not supported by non-bond device: %s", - dpdk_dev->devargs->args); - return NULL; - } - } + if (switch_info->representor && dpdk_dev->devargs && + !mlx5_representor_match(spawn, eth_da)) + return NULL; /* Build device name. */ - if (spawn->pf_bond < 0) { + if (spawn->pf_bond < 0) { /* Single device. */ if (!switch_info->representor) strlcpy(name, dpdk_dev->name, sizeof(name)); else - snprintf(name, sizeof(name), "%s_representor_%s%u", + err = snprintf(name, sizeof(name), "%s_representor_%s%u", dpdk_dev->name, switch_info->name_type == MLX5_PHYS_PORT_NAME_TYPE_PFSF ? "sf" : "vf", switch_info->port_name); } else { /* Bonding device. */ - if (!switch_info->representor) - snprintf(name, sizeof(name), "%s_%s", + if (!switch_info->representor) { + err = snprintf(name, sizeof(name), "%s_%s", dpdk_dev->name, mlx5_os_get_dev_device_name(spawn->phys_dev)); - else - snprintf(name, sizeof(name), "%s_%s_representor_%s%u", - dpdk_dev->name, - mlx5_os_get_dev_device_name(spawn->phys_dev), - switch_info->name_type == - MLX5_PHYS_PORT_NAME_TYPE_PFSF ? "sf" : "vf", - switch_info->port_name); + } else { + err = snprintf(name, sizeof(name), "%s_%s_representor_c%dpf%d%s%u", + dpdk_dev->name, + mlx5_os_get_dev_device_name(spawn->phys_dev), + switch_info->ctrl_num, + switch_info->pf_num, + switch_info->name_type == + MLX5_PHYS_PORT_NAME_TYPE_PFSF ? "sf" : "vf", + switch_info->port_name); + } } + if (err >= (int)sizeof(name)) + DRV_LOG(WARNING, "device name overflow %s", name); /* check if the device is already spawned */ if (rte_eth_dev_get_port_by_name(name, &port_id) == 0) { rte_errno = EEXIST; @@ -1749,9 +1743,11 @@ mlx5_dev_spawn_data_cmp(const void *a, const void *b) * @param[in] ibv_dev * Pointer to Infiniband device structure. * @param[in] pci_dev - * Pointer to PCI device structure to match PCI address. + * Pointer to primary PCI address structure to match. * @param[in] nl_rdma * Netlink RDMA group socket handle. + * @param[in] owner + * Rerepsentor owner PF index. * * @return * negative value if no bonding device found, otherwise @@ -1759,8 +1755,8 @@ mlx5_dev_spawn_data_cmp(const void *a, const void *b) */ static int mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev, - const struct rte_pci_device *pci_dev, - int nl_rdma) + const struct rte_pci_addr *pci_dev, + int nl_rdma, uint16_t owner) { char ifname[IF_NAMESIZE + 1]; unsigned int ifindex; @@ -1817,10 +1813,10 @@ mlx5_device_bond_pci_match(const struct ibv_device *ibv_dev, " for netdev \"%s\"", ifname); continue; } - if (pci_dev->addr.domain != pci_addr.domain || - pci_dev->addr.bus != pci_addr.bus || - pci_dev->addr.devid != pci_addr.devid || - pci_dev->addr.function != pci_addr.function) + if (pci_dev->domain != pci_addr.domain || + pci_dev->bus != pci_addr.bus || + pci_dev->devid != pci_addr.devid || + pci_dev->function + owner != pci_addr.function) continue; /* Slave interface PCI address match found. */ fclose(file); @@ -1888,7 +1884,8 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, struct mlx5_dev_config dev_config; unsigned int dev_config_vf; struct rte_eth_devargs eth_da = { .type = RTE_ETH_REPRESENTOR_NONE }; - int ret; + struct rte_pci_addr owner_pci = pci_dev->addr; /* Owner PF. */ + int ret = -1; if (rte_eal_process_type() == RTE_PROC_PRIMARY) mlx5_pmd_socket_init(); @@ -1940,7 +1937,8 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, DRV_LOG(DEBUG, "checking device \"%s\"", ibv_list[ret]->name); bd = mlx5_device_bond_pci_match - (ibv_list[ret], pci_dev, nl_rdma); + (ibv_list[ret], &owner_pci, nl_rdma, + eth_da.ports[0]); if (bd >= 0) { /* * Bonding device detected. Only one match is allowed, @@ -1957,23 +1955,28 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, ret = -rte_errno; goto exit; } + /* Amend owner pci address if owner PF ID specified. */ + if (eth_da.nb_representor_ports) + owner_pci.function += eth_da.ports[0]; DRV_LOG(INFO, "PCI information matches for" " slave %d bonding device \"%s\"", bd, ibv_list[ret]->name); ibv_match[nd++] = ibv_list[ret]; break; + } else { + /* Bonding device not found. */ + if (mlx5_dev_to_pci_addr + (ibv_list[ret]->ibdev_path, &pci_addr)) + continue; + if (owner_pci.domain != pci_addr.domain || + owner_pci.bus != pci_addr.bus || + owner_pci.devid != pci_addr.devid || + owner_pci.function != pci_addr.function) + continue; + DRV_LOG(INFO, "PCI information matches for device \"%s\"", + ibv_list[ret]->name); + ibv_match[nd++] = ibv_list[ret]; } - if (mlx5_dev_to_pci_addr - (ibv_list[ret]->ibdev_path, &pci_addr)) - continue; - if (pci_dev->addr.domain != pci_addr.domain || - pci_dev->addr.bus != pci_addr.bus || - pci_dev->addr.devid != pci_addr.devid || - pci_dev->addr.function != pci_addr.function) - continue; - DRV_LOG(INFO, "PCI information matches for device \"%s\"", - ibv_list[ret]->name); - ibv_match[nd++] = ibv_list[ret]; } ibv_match[nd] = NULL; if (!nd) { @@ -1981,8 +1984,8 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, DRV_LOG(WARNING, "no Verbs device matches PCI device " PCI_PRI_FMT "," " are kernel drivers loaded?", - pci_dev->addr.domain, pci_dev->addr.bus, - pci_dev->addr.devid, pci_dev->addr.function); + owner_pci.domain, owner_pci.bus, + owner_pci.devid, owner_pci.function); rte_errno = ENOENT; ret = -rte_errno; goto exit; @@ -2247,6 +2250,24 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, dev_config_vf = 0; break; } + if (eth_da.type != RTE_ETH_REPRESENTOR_NONE) { + /* Set devargs default values. */ + if (eth_da.nb_mh_controllers == 0) { + eth_da.nb_mh_controllers = 1; + eth_da.mh_controllers[0] = 0; + } + if (eth_da.nb_ports == 0 && ns > 0) { + if (list[0].pf_bond >= 0 && list[0].info.representor) + DRV_LOG(WARNING, "Representor on Bonding device should use pf#vf# syntax: %s", + pci_dev->device.devargs->args); + eth_da.nb_ports = 1; + eth_da.ports[0] = list[0].info.pf_num; + } + if (eth_da.nb_representor_ports == 0) { + eth_da.nb_representor_ports = 1; + eth_da.representor_ports[0] = 0; + } + } for (i = 0; i != ns; ++i) { uint32_t restore; @@ -2288,8 +2309,8 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, DRV_LOG(ERR, "probe of PCI device " PCI_PRI_FMT " aborted after" " encountering an error: %s", - pci_dev->addr.domain, pci_dev->addr.bus, - pci_dev->addr.devid, pci_dev->addr.function, + owner_pci.domain, owner_pci.bus, + owner_pci.devid, owner_pci.function, strerror(rte_errno)); ret = -rte_errno; /* Roll back. */ diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 006ab95473..99a15afe5b 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -355,6 +355,26 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { #define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096 +/** + * Decide whether representor ID is a HPF(host PF) port on BF2. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * Non-zero if HPF, otherwise 0. + */ +bool +mlx5_is_hpf(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + uint16_t repr = MLX5_REPRESENTOR_REPR(priv->representor_id); + int type = MLX5_REPRESENTOR_TYPE(priv->representor_id); + + return priv->representor != 0 && type == RTE_ETH_REPRESENTOR_VF && + MLX5_REPRESENTOR_REPR(-1) == repr; +} + /** * Initialize the ASO aging management structure. * diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index a0f22f9953..68b599a126 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -934,7 +934,7 @@ struct mlx5_priv { uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ uint32_t vport_meta_mask; /* Used for vport index field match mask. */ - int32_t representor_id; /* Port representor identifier. */ + int32_t representor_id; /* -1 if not a representor. */ int32_t pf_bond; /* >=0 means PF index in bonding configuration. */ unsigned int if_index; /* Associated kernel network device index. */ uint32_t bond_ifindex; /**< Bond interface index. */ @@ -1010,6 +1010,7 @@ int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev, struct rte_eth_udp_tunnel *udp_tunnel); uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev); int mlx5_dev_close(struct rte_eth_dev *dev); +bool mlx5_is_hpf(struct rte_eth_dev *dev); void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh); /* Macro to iterate over all valid ports for mlx5 driver. */ diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h index af29d93901..8f2807dcd9 100644 --- a/drivers/net/mlx5/mlx5_defs.h +++ b/drivers/net/mlx5/mlx5_defs.h @@ -48,10 +48,6 @@ #define MLX5_PMD_SOFT_COUNTERS 1 #endif -/* Switch port ID parameters for bonding configurations. */ -#define MLX5_PORT_ID_BONDING_PF_MASK 0xf -#define MLX5_PORT_ID_BONDING_PF_SHIFT 12 - /* Alarm timeout. */ #define MLX5_ALARM_TIMEOUT_US 100000 diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 1ffb13cf2e..130980d4d6 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -330,33 +330,6 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) if (priv->representor) { uint16_t port_id; - if (priv->pf_bond >= 0) { - /* - * Switch port ID is opaque value with driver defined - * format. Push the PF index in bonding configurations - * in upper four bits of port ID. If we get too many - * representors (more than 4K) or PFs (more than 15) - * this approach must be reconsidered. - */ - /* Switch port ID for VF representors: 0 - 0xFFE */ - if ((info->switch_info.port_id != 0xffff && - info->switch_info.port_id >= - ((1 << MLX5_PORT_ID_BONDING_PF_SHIFT) - 1)) || - priv->pf_bond > MLX5_PORT_ID_BONDING_PF_MASK) { - DRV_LOG(ERR, "can't update switch port ID" - " for bonding device"); - MLX5_ASSERT(false); - return -ENODEV; - } - /* - * Switch port ID for Host PF representor - * (representor_id is -1) , set to 0xFFF - */ - if (info->switch_info.port_id == 0xffff) - info->switch_info.port_id = 0xfff; - info->switch_info.port_id |= - priv->pf_bond << MLX5_PORT_ID_BONDING_PF_SHIFT; - } MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) { struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private; diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index 6ffcfcd97a..7b2be04889 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -159,7 +159,7 @@ mlx5_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) * Configuring the VF instead of its representor, * need to skip the special case of HPF on Bluefield. */ - if (priv->representor && priv->representor_id >= 0) { + if (priv->representor && !mlx5_is_hpf(dev)) { DRV_LOG(DEBUG, "VF represented by port %u setting primary MAC address", dev->data->port_id); RTE_ETH_FOREACH_DEV_SIBLING(port_id, dev->data->port_id) { @@ -169,7 +169,9 @@ mlx5_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) return mlx5_os_vf_mac_addr_modify (priv, mlx5_ifindex(&rte_eth_devices[port_id]), - mac_addr, priv->representor_id); + mac_addr, + MLX5_REPRESENTOR_REPR + (priv->representor_id)); } } rte_errno = -ENOTSUP;