From 2786b7bf9084b32dde9a346d92ab1c27f0ffc476 Mon Sep 17 00:00:00 2001 From: Suanming Mou Date: Thu, 28 May 2020 14:59:49 +0800 Subject: [PATCH] net/mlx5: fix secondary process resources release When secondary process starts, it will allocate its own process private data, and also does remap to UAR register of the Tx queue. Once the secondary process exits, these resources should be released accordingly. And the shared resources owned by primary should not be touched. Currently, once one port in the secondary process spawn failed, all the other spawned ports will also be released during process exits. However, the mlx5_dev_close() function does not add the cases for secondary process, it means call the mlx5_dev_close() function directly in secondary process releases the resources it should not touch. Add the case for secondary process release to its own resources in mlx5_dev_close() function to help it quits gracefully. Fixes: 942d13e6e7d1 ("net/mlx5: fix sharing context destroy order") Fixes: 3a8207423a0f ("net/mlx5: close all ports on remove") Cc: stable@dpdk.org Signed-off-by: Suanming Mou Acked-by: Matan Azrad --- drivers/net/mlx5/mlx5.c | 47 +++++++++++++++++++++++++----------- drivers/net/mlx5/mlx5_rxtx.h | 1 + drivers/net/mlx5/mlx5_txq.c | 24 ++++++++++++++++++ 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 5589772eb8..81102631aa 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1423,6 +1423,17 @@ mlx5_dev_close(struct rte_eth_dev *dev) unsigned int i; int ret; + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + /* Check if process_private released. */ + if (!dev->process_private) + return; + mlx5_tx_uar_uninit_secondary(dev); + mlx5_proc_priv_uninit(dev); + rte_eth_dev_release_port(dev); + return; + } + if (!priv->sh) + return; DRV_LOG(DEBUG, "port %u closing device \"%s\"", dev->data->port_id, ((priv->sh->ctx != NULL) ? priv->sh->ctx->device->name : "")); @@ -1512,16 +1523,13 @@ mlx5_dev_close(struct rte_eth_dev *dev) if (ret) DRV_LOG(WARNING, "port %u some flows still remain", dev->data->port_id); - if (priv->sh) { - /* - * Free the shared context in last turn, because the cleanup - * routines above may use some shared fields, like - * mlx5_nl_mac_addr_flush() uses ibdev_path for retrieveing - * ifindex if Netlink fails. - */ - mlx5_free_shared_ibctx(priv->sh); - priv->sh = NULL; - } + /* + * Free the shared context in last turn, because the cleanup + * routines above may use some shared fields, like + * mlx5_nl_mac_addr_flush() uses ibdev_path for retrieveing + * ifindex if Netlink fails. + */ + mlx5_free_shared_ibctx(priv->sh); if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) { unsigned int c = 0; uint16_t port_id; @@ -2409,11 +2417,11 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, /* Receive command fd from primary process */ err = mlx5_mp_req_verbs_cmd_fd(&mp_id); if (err < 0) - return NULL; + goto err_secondary; /* Remap UAR for Tx queues. */ err = mlx5_tx_uar_init_secondary(eth_dev, err); if (err) - return NULL; + goto err_secondary; /* * Ethdev pointer is still required as input since * the primary device is not accessible from the @@ -2422,6 +2430,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, eth_dev->rx_pkt_burst = mlx5_select_rx_function(eth_dev); eth_dev->tx_pkt_burst = mlx5_select_tx_function(eth_dev); return eth_dev; +err_secondary: + mlx5_dev_close(eth_dev); + return NULL; } /* * Some parameters ("tx_db_nc" in particularly) are needed in @@ -3707,8 +3718,16 @@ mlx5_pci_remove(struct rte_pci_device *pci_dev) { uint16_t port_id; - RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) - rte_eth_dev_close(port_id); + RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device) { + /* + * mlx5_dev_close() is not registered to secondary process, + * call the close function explicitly for secondary process. + */ + if (rte_eal_process_type() == RTE_PROC_SECONDARY) + mlx5_dev_close(&rte_eth_devices[port_id]); + else + rte_eth_dev_close(port_id); + } return 0; } diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index 48f2b79411..26621ff193 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -434,6 +434,7 @@ int mlx5_tx_hairpin_queue_setup const struct rte_eth_hairpin_conf *hairpin_conf); void mlx5_tx_queue_release(void *dpdk_txq); int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd); +void mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev); struct mlx5_txq_obj *mlx5_txq_obj_new(struct rte_eth_dev *dev, uint16_t idx, enum mlx5_txq_obj_type type); struct mlx5_txq_obj *mlx5_txq_obj_get(struct rte_eth_dev *dev, uint16_t idx); diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 80d99ff946..2047a9a4e7 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -427,6 +427,30 @@ txq_uar_uninit_secondary(struct mlx5_txq_ctrl *txq_ctrl) munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size); } +/** + * Deinitialize Tx UAR registers for secondary process. + * + * @param dev + * Pointer to Ethernet device. + */ +void +mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_data *txq; + struct mlx5_txq_ctrl *txq_ctrl; + unsigned int i; + + MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY); + for (i = 0; i != priv->txqs_n; ++i) { + if (!(*priv->txqs)[i]) + continue; + txq = (*priv->txqs)[i]; + txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + txq_uar_uninit_secondary(txq_ctrl); + } +} + /** * Initialize Tx UAR registers for secondary process. * -- 2.20.1