net/mlx5: reduce PCI dependency

To support more bus types, remove PCI dependency where possible.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>

diff --git a/drivers/net/mlx5/linux/mlx5_ethdev_os.c b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
index ddc1371..b05b9fc 100644 (file)
--- a/drivers/net/mlx5/linux/mlx5_ethdev_os.c
+++ b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
@@ -346,7 +346,7 @@ mlx5_find_master_dev(struct rte_eth_dev *dev)
        priv = dev->data->dev_private;
        domain_id = priv->domain_id;
        MLX5_ASSERT(priv->representor);
-       MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+       MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
                struct mlx5_priv *opriv =
                        rte_eth_devices[port_id].data->dev_private;
                if (opriv &&

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 05b1761..4ab30fd 100644 (file)
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1252,7 +1252,7 @@ err_secondary:
         * Look for sibling devices in order to reuse their switch domain
         * if any, otherwise allocate one.
         */
-       MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+       MLX5_ETH_FOREACH_DEV(port_id, NULL) {
                const struct mlx5_priv *opriv =
                        rte_eth_devices[port_id].data->dev_private;
 
@@ -2511,6 +2511,7 @@ mlx5_os_pci_probe_pf(struct rte_pci_device *pci_dev,
                dev_config.decap_en = 1;
                dev_config.log_hp_size = MLX5_ARG_UNSET;
                dev_config.allow_duplicate_pattern = 1;
+               list[i].numa_node = pci_dev->device.numa_node;
                list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device,
                                                 &list[i],
                                                 &dev_config,
@@ -2708,7 +2709,6 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
        int dbmap_env;
        int err = 0;
 
-       sh->numa_node = spawn->pci_dev->device.numa_node;
        pthread_mutex_init(&sh->txpp.mutex, NULL);
        /*
         * Configure environment variable "MLX5_BF_SHUT_UP"

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d9c90d5..8e64bf9 100644 (file)
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1120,6 +1120,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
                rte_errno  = ENOMEM;
                goto exit;
        }
+       sh->numa_node = spawn->numa_node;
        if (spawn->bond_info)
                sh->bond = *spawn->bond_info;
        err = mlx5_os_open_device(spawn, config, sh);
@@ -1197,7 +1198,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
         */
        err = mlx5_mr_btree_init(&sh->share_cache.cache,
                                 MLX5_MR_BTREE_CACHE_N * 2,
-                                spawn->pci_dev->device.numa_node);
+                                sh->numa_node);
        if (err) {
                err = rte_errno;
                goto error;
@@ -1635,7 +1636,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
                unsigned int c = 0;
                uint16_t port_id;
 
-               MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+               MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
                        struct mlx5_priv *opriv =
                                rte_eth_devices[port_id].data->dev_private;
 
@@ -2077,18 +2078,20 @@ mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
 {
        if (config->txq_inline_min != MLX5_ARG_UNSET) {
                /* Application defines size of inlined data explicitly. */
-               switch (spawn->pci_dev->id.device_id) {
-               case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
-               case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
-                       if (config->txq_inline_min <
-                                      (int)MLX5_INLINE_HSIZE_L2) {
-                               DRV_LOG(DEBUG,
-                                       "txq_inline_mix aligned to minimal"
-                                       " ConnectX-4 required value %d",
-                                       (int)MLX5_INLINE_HSIZE_L2);
-                               config->txq_inline_min = MLX5_INLINE_HSIZE_L2;
+               if (spawn->pci_dev != NULL) {
+                       switch (spawn->pci_dev->id.device_id) {
+                       case PCI_DEVICE_ID_MELLANOX_CONNECTX4:
+                       case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF:
+                               if (config->txq_inline_min <
+                                              (int)MLX5_INLINE_HSIZE_L2) {
+                                       DRV_LOG(DEBUG,
+                                               "txq_inline_mix aligned to minimal ConnectX-4 required value %d",
+                                               (int)MLX5_INLINE_HSIZE_L2);
+                                       config->txq_inline_min =
+                                                       MLX5_INLINE_HSIZE_L2;
+                               }
+                               break;
                        }
-                       break;
                }
                goto exit;
        }
@@ -2142,6 +2145,10 @@ mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn,
                        }
                }
        }
+       if (spawn->pci_dev == NULL) {
+               config->txq_inline_min = MLX5_INLINE_HSIZE_NONE;
+               goto exit;
+       }
        /*
         * We get here if we are unable to deduce
         * inline data size with DevX. Try PCI ID
@@ -2276,7 +2283,7 @@ mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
        if (sh->refcnt == 1)
                return 0;
        /* Find the device with shared context. */
-       MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+       MLX5_ETH_FOREACH_DEV(port_id, NULL) {
                struct mlx5_priv *opriv =
                        rte_eth_devices[port_id].data->dev_private;
 
@@ -2307,25 +2314,25 @@ mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
  *
  * @param[in] port_id
  *   port_id to start looking for device.
- * @param[in] pci_dev
- *   Pointer to the hint PCI device. When device is being probed
+ * @param[in] odev
+ *   Pointer to the hint device. When device is being probed
  *   the its siblings (master and preceding representors might
  *   not have assigned driver yet (because the mlx5_os_pci_probe()
- *   is not completed yet, for this case match on hint PCI
+ *   is not completed yet, for this case match on hint
  *   device may be used to detect sibling device.
  *
  * @return
  *   port_id of found device, RTE_MAX_ETHPORT if not found.
  */
 uint16_t
-mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev)
+mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev)
 {
        while (port_id < RTE_MAX_ETHPORTS) {
                struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 
                if (dev->state != RTE_ETH_DEV_UNUSED &&
                    dev->device &&
-                   (dev->device == &pci_dev->device ||
+                   (dev->device == odev ||
                     (dev->device->driver &&
                     dev->device->driver->name &&
                     !strcmp(dev->device->driver->name, MLX5_PCI_DRIVER_NAME))))

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 94618e1..749a9e9 100644 (file)
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -134,6 +134,7 @@ struct mlx5_dev_spawn_data {
        uint32_t max_port; /**< Device maximal port index. */
        uint32_t phys_port; /**< Device physical port index. */
        int pf_bond; /**< bonding device PF index. < 0 - no bonding */
+       int numa_node; /**< Device numa node. */
        struct mlx5_switch_info info; /**< Switch information. */
        void *phys_dev; /**< Associated physical device. */
        struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
@@ -1462,16 +1463,16 @@ int mlx5_proc_priv_init(struct rte_eth_dev *dev);
 void mlx5_proc_priv_uninit(struct rte_eth_dev *dev);
 int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
                              struct rte_eth_udp_tunnel *udp_tunnel);
-uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev);
+uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev);
 int mlx5_dev_close(struct rte_eth_dev *dev);
 bool mlx5_is_hpf(struct rte_eth_dev *dev);
 void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh);
 
 /* Macro to iterate over all valid ports for mlx5 driver. */
-#define MLX5_ETH_FOREACH_DEV(port_id, pci_dev) \
-       for (port_id = mlx5_eth_find_next(0, pci_dev); \
+#define MLX5_ETH_FOREACH_DEV(port_id, dev) \
+       for (port_id = mlx5_eth_find_next(0, dev); \
             port_id < RTE_MAX_ETHPORTS; \
-            port_id = mlx5_eth_find_next(port_id + 1, pci_dev))
+            port_id = mlx5_eth_find_next(port_id + 1, dev))
 int mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs);
 struct mlx5_dev_ctx_shared *
 mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,

diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index c32eeef..82e2284 100644 (file)
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -335,7 +335,7 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
        if (priv->representor) {
                uint16_t port_id;
 
-               MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+               MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
                        struct mlx5_priv *opriv =
                                rte_eth_devices[port_id].data->dev_private;
 

diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 89c43fc..87fb437 100644 (file)
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -199,23 +199,23 @@ mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
 }
 
 /**
- * Finds the first ethdev that match the pci device.
+ * Finds the first ethdev that match the device.
  * The existence of multiple ethdev per pci device is only with representors.
  * On such case, it is enough to get only one of the ports as they all share
  * the same ibv context.
  *
- * @param pdev
- *   Pointer to the PCI device.
+ * @param dev
+ *   Pointer to the device.
  *
  * @return
  *   Pointer to the ethdev if found, NULL otherwise.
  */
 static struct rte_eth_dev *
-pci_dev_to_eth_dev(struct rte_pci_device *pdev)
+dev_to_eth_dev(struct rte_device *dev)
 {
        uint16_t port_id;
 
-       port_id = rte_eth_find_next_of(0, &pdev->device);
+       port_id = rte_eth_find_next_of(0, dev);
        if (port_id == RTE_MAX_ETHPORTS)
                return NULL;
        return &rte_eth_devices[port_id];
@@ -245,7 +245,7 @@ mlx5_dma_map(struct rte_pci_device *pdev, void *addr,
        struct mlx5_priv *priv;
        struct mlx5_dev_ctx_shared *sh;
 
-       dev = pci_dev_to_eth_dev(pdev);
+       dev = dev_to_eth_dev(&pdev->device);
        if (!dev) {
                DRV_LOG(WARNING, "unable to find matching ethdev "
                                 "to PCI device %p", (void *)pdev);
@@ -295,7 +295,7 @@ mlx5_dma_unmap(struct rte_pci_device *pdev, void *addr,
        struct mlx5_mr *mr;
        struct mr_cache_entry entry;
 
-       dev = pci_dev_to_eth_dev(pdev);
+       dev = dev_to_eth_dev(&pdev->device);
        if (!dev) {
                DRV_LOG(WARNING, "unable to find matching ethdev "
                                 "to PCI device %p", (void *)pdev);

diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 7cb8920..6d2351f 100644 (file)
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -697,7 +697,7 @@ mlx5_hairpin_bind_single_port(struct rte_eth_dev *dev, uint16_t rx_port)
        uint32_t explicit;
        uint16_t rx_queue;
 
-       if (mlx5_eth_find_next(rx_port, priv->pci_dev) != rx_port) {
+       if (mlx5_eth_find_next(rx_port, dev->device) != rx_port) {
                rte_errno = ENODEV;
                DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port);
                return -rte_errno;
@@ -835,7 +835,7 @@ mlx5_hairpin_unbind_single_port(struct rte_eth_dev *dev, uint16_t rx_port)
        int ret;
        uint16_t cur_port = priv->dev_data->port_id;
 
-       if (mlx5_eth_find_next(rx_port, priv->pci_dev) != rx_port) {
+       if (mlx5_eth_find_next(rx_port, dev->device) != rx_port) {
                rte_errno = ENODEV;
                DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port);
                return -rte_errno;
@@ -893,7 +893,6 @@ mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
 {
        int ret = 0;
        uint16_t p, pp;
-       struct mlx5_priv *priv = dev->data->dev_private;
 
        /*
         * If the Rx port has no hairpin configuration with the current port,
@@ -902,7 +901,7 @@ mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
         * information updating.
         */
        if (rx_port == RTE_MAX_ETHPORTS) {
-               MLX5_ETH_FOREACH_DEV(p, priv->pci_dev) {
+               MLX5_ETH_FOREACH_DEV(p, dev->device) {
                        ret = mlx5_hairpin_bind_single_port(dev, p);
                        if (ret != 0)
                                goto unbind;
@@ -912,7 +911,7 @@ mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
                return mlx5_hairpin_bind_single_port(dev, rx_port);
        }
 unbind:
-       MLX5_ETH_FOREACH_DEV(pp, priv->pci_dev)
+       MLX5_ETH_FOREACH_DEV(pp, dev->device)
                if (pp < p)
                        mlx5_hairpin_unbind_single_port(dev, pp);
        return ret;
@@ -927,10 +926,9 @@ mlx5_hairpin_unbind(struct rte_eth_dev *dev, uint16_t rx_port)
 {
        int ret = 0;
        uint16_t p;
-       struct mlx5_priv *priv = dev->data->dev_private;
 
        if (rx_port == RTE_MAX_ETHPORTS)
-               MLX5_ETH_FOREACH_DEV(p, priv->pci_dev) {
+               MLX5_ETH_FOREACH_DEV(p, dev->device) {
                        ret = mlx5_hairpin_unbind_single_port(dev, p);
                        if (ret != 0)
                                return ret;

diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 3e5e944..f68c0c6 100644 (file)
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -816,7 +816,7 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
        if (config->txqs_inline == MLX5_ARG_UNSET)
                txqs_inline =
 #if defined(RTE_ARCH_ARM64)
-               (priv->pci_dev->id.device_id ==
+               (priv->pci_dev && priv->pci_dev->id.device_id ==
                        PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) ?
                        MLX5_INLINE_MAX_TXQS_BLUEFIELD :
 #endif

diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 5da362a..bf20ada 100644 (file)
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -391,7 +391,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
         * Look for sibling devices in order to reuse their switch domain
         * if any, otherwise allocate one.
         */
-       MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
+       MLX5_ETH_FOREACH_DEV(port_id, NULL) {
                const struct mlx5_priv *opriv =
                        rte_eth_devices[port_id].data->dev_private;
 
@@ -1080,6 +1080,7 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
        dev_config.dv_flow_en = 1;
        dev_config.decap_en = 0;
        dev_config.log_hp_size = MLX5_ARG_UNSET;
+       list[ns].numa_node = pci_dev->device.numa_node;
        list[ns].eth_dev = mlx5_dev_spawn(&pci_dev->device,
                                          &list[ns],
                                          &dev_config);