net/bnxt: refactor VFR port clean up

When parent VF or PF ports are cleaned up, the child VF representor
ports also need to be cleaned up. If not cleaned up, then deleting
the parent VF shall result in not cleaning up the hardware rules and
updating the firmware of VFR removal. The issue can occur even when
application is exited without deleting VFR ports.

Signed-off-by: Kishore Padmanabha <kishore.padmanabha@broadcom.com>
Reviewed-by: Mike Baucom <michael.baucom@broadcom.com>
Reviewed-by: Shahaji Bhosle <sbhosle@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c
index 445c408..8908885 100644 (file)
--- a/drivers/net/bnxt/bnxt_ethdev.c
+++ b/drivers/net/bnxt/bnxt_ethdev.c
@@ -1309,6 +1309,9 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev)
        /* disable uio/vfio intr/eventfd mapping */
        rte_intr_disable(intr_handle);
 
+       /* Stop the child representors for this device */
+       bnxt_vf_rep_stop_all(bp);
+
        /* delete the bnxt ULP port details */
        bnxt_ulp_port_deinit(bp);
 
@@ -3724,8 +3727,13 @@ bnxt_filter_ctrl_op(struct rte_eth_dev *dev,
                struct bnxt_vf_representor *vfr = dev->data->dev_private;
                bp = vfr->parent_dev->data->dev_private;
                /* parent is deleted while children are still valid */
-               if (!bp)
+               if (!bp) {
+                       PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR Error %d:%d\n",
+                                   dev->data->port_id,
+                                   filter_type,
+                                   filter_op);
                        return -EIO;
+               }
        }
 
        ret = is_bnxt_in_error(bp);
@@ -5927,8 +5935,12 @@ static int bnxt_pci_remove_dev_with_reps(struct rte_eth_dev *eth_dev)
                vf_rep_eth_dev = bp->rep_info[i].vfr_eth_dev;
                if (!vf_rep_eth_dev)
                        continue;
+               PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR pci remove\n",
+                           vf_rep_eth_dev->data->port_id);
                rte_eth_dev_destroy(vf_rep_eth_dev, bnxt_vf_representor_uninit);
        }
+       PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci remove\n",
+                   eth_dev->data->port_id);
        ret = rte_eth_dev_destroy(eth_dev, bnxt_dev_uninit);
 
        return ret;
@@ -6055,6 +6067,8 @@ static int bnxt_rep_port_probe(struct rte_pci_device *pci_dev,
                                ret = -ENODEV;
                                return ret;
                        }
+                       PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR pci probe\n",
+                                   backing_eth_dev->data->port_id);
                        backing_bp->rep_info[representor.vf_id].vfr_eth_dev =
                                vf_rep_eth_dev;
                        backing_bp->num_reps++;
@@ -6103,7 +6117,8 @@ static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
                backing_eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
        }
-
+       PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci probe\n",
+                   backing_eth_dev->data->port_id);
        /* probe representor ports now */
        ret = bnxt_rep_port_probe(pci_dev, eth_da, backing_eth_dev);
 
@@ -6122,6 +6137,7 @@ static int bnxt_pci_remove(struct rte_pci_device *pci_dev)
                           * +ve value will at least help in proper cleanup
                           */
 
+       PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci remove\n", eth_dev->data->port_id);
        if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
                if (eth_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
                        return rte_eth_dev_destroy(eth_dev,

diff --git a/drivers/net/bnxt/bnxt_reps.c b/drivers/net/bnxt/bnxt_reps.c
index 00e44bc..b975d5a 100644 (file)
--- a/drivers/net/bnxt/bnxt_reps.c
+++ b/drivers/net/bnxt/bnxt_reps.c
@@ -166,6 +166,7 @@ int bnxt_vf_representor_init(struct rte_eth_dev *eth_dev, void *params)
        struct rte_eth_link *link;
        struct bnxt *parent_bp;
 
+       PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR init\n", eth_dev->data->port_id);
        vf_rep_bp->vf_id = rep_params->vf_id;
        vf_rep_bp->switch_domain_id = rep_params->switch_domain_id;
        vf_rep_bp->parent_dev = rep_params->parent_dev;
@@ -216,15 +217,18 @@ int bnxt_vf_representor_uninit(struct rte_eth_dev *eth_dev)
        struct bnxt *parent_bp;
        struct bnxt_vf_representor *rep =
                (struct bnxt_vf_representor *)eth_dev->data->dev_private;
-
        uint16_t vf_id;
 
+       PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR uninit\n", eth_dev->data->port_id);
        eth_dev->data->mac_addrs = NULL;
        eth_dev->dev_ops = NULL;
 
        parent_bp = rep->parent_dev->data->dev_private;
-       if (!parent_bp)
+       if (!parent_bp) {
+               PMD_DRV_LOG(DEBUG, "BNXT Port:%d already freed\n",
+                           eth_dev->data->port_id);
                return 0;
+       }
 
        parent_bp->num_reps--;
        vf_id = rep->vf_id;
@@ -296,7 +300,8 @@ static int bnxt_tf_vfr_alloc(struct rte_eth_dev *vfr_ethdev)
                            vfr->vf_id, rc);
                (void)bnxt_ulp_delete_vfr_default_rules(vfr);
        }
-
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d VFR created and initialized\n",
+                   vfr->dpdk_port_id);
        return rc;
 }
 
@@ -361,6 +366,7 @@ int bnxt_vf_rep_dev_start_op(struct rte_eth_dev *eth_dev)
        parent_bp = rep_bp->parent_dev->data->dev_private;
        rep_info = &parent_bp->rep_info[rep_bp->vf_id];
 
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d VFR start\n", eth_dev->data->port_id);
        pthread_mutex_lock(&rep_info->vfr_start_lock);
        if (!rep_info->conduit_valid) {
                rc = bnxt_get_dflt_vnic_svif(parent_bp, rep_bp);
@@ -386,6 +392,7 @@ int bnxt_vf_rep_dev_start_op(struct rte_eth_dev *eth_dev)
 
 static int bnxt_tf_vfr_free(struct bnxt_vf_representor *vfr)
 {
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d VFR ulp free\n", vfr->dpdk_port_id);
        return bnxt_ulp_delete_vfr_default_rules(vfr);
 }
 
@@ -401,8 +408,11 @@ static int bnxt_vfr_free(struct bnxt_vf_representor *vfr)
        }
 
        parent_bp = vfr->parent_dev->data->dev_private;
-       if (!parent_bp)
+       if (!parent_bp) {
+               PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR already freed\n",
+                           vfr->dpdk_port_id);
                return 0;
+       }
 
        /* Check if representor has been already freed in FW */
        if (!vfr->vfr_tx_cfa_action)
@@ -432,6 +442,8 @@ void bnxt_vf_rep_dev_stop_op(struct rte_eth_dev *eth_dev)
        eth_dev->rx_pkt_burst = &bnxt_dummy_recv_pkts;
        eth_dev->tx_pkt_burst = &bnxt_dummy_xmit_pkts;
 
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d VFR stop\n", eth_dev->data->port_id);
+
        bnxt_vfr_free(vfr_bp);
 
        if (eth_dev->data->dev_started)
@@ -442,6 +454,7 @@ void bnxt_vf_rep_dev_stop_op(struct rte_eth_dev *eth_dev)
 
 void bnxt_vf_rep_dev_close_op(struct rte_eth_dev *eth_dev)
 {
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d VFR close\n", eth_dev->data->port_id);
        bnxt_vf_representor_uninit(eth_dev);
 }
 
@@ -721,3 +734,20 @@ int bnxt_vf_rep_stats_reset_op(struct rte_eth_dev *eth_dev)
        }
        return 0;
 }
+
+void bnxt_vf_rep_stop_all(struct bnxt *bp)
+{
+       uint16_t vf_id;
+       struct rte_eth_dev *rep_eth_dev;
+
+       /* No vfrep ports just exit */
+       if (!bp->rep_info)
+               return;
+
+       for (vf_id = 0; vf_id < BNXT_MAX_VF_REPS; vf_id++) {
+               rep_eth_dev = bp->rep_info[vf_id].vfr_eth_dev;
+               if (!rep_eth_dev)
+                       continue;
+               bnxt_vf_rep_dev_stop_op(rep_eth_dev);
+       }
+}

diff --git a/drivers/net/bnxt/bnxt_reps.h b/drivers/net/bnxt/bnxt_reps.h
index 418b95a..d877b08 100644 (file)
--- a/drivers/net/bnxt/bnxt_reps.h
+++ b/drivers/net/bnxt/bnxt_reps.h
@@ -42,4 +42,5 @@ void bnxt_vf_rep_dev_close_op(struct rte_eth_dev *eth_dev);
 int bnxt_vf_rep_stats_get_op(struct rte_eth_dev *eth_dev,
                             struct rte_eth_stats *stats);
 int bnxt_vf_rep_stats_reset_op(struct rte_eth_dev *eth_dev);
+void bnxt_vf_rep_stop_all(struct bnxt *bp);
 #endif /* _BNXT_REPS_H_ */

diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
index 63d453f..2725364 100644 (file)
--- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
+++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c
@@ -905,7 +905,7 @@ bnxt_ulp_port_init(struct bnxt *bp)
        }
        /* create the default rules */
        bnxt_ulp_create_df_rules(bp);
-       BNXT_TF_DBG(DEBUG, "ULP Port:%d created and initialized\n",
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d ULP port init\n",
                    bp->eth_dev->data->port_id);
        return rc;
 
@@ -940,7 +940,7 @@ bnxt_ulp_port_deinit(struct bnxt *bp)
                return;
        }
 
-       BNXT_TF_DBG(DEBUG, "ULP Port:%d destroyed\n",
+       BNXT_TF_DBG(DEBUG, "BNXT Port:%d ULP port deinit\n",
                    bp->eth_dev->data->port_id);
 
        /* Get the session details  */