event/dlb2: rework queue drain handling

Previously, anything that prevented the CQs from being drained
would hang the PMD for an unacceptably long time. This commit updates
the drain logic to be resource and time based, thus eliminating
the potential for a long hang when draining the queues in preparation
for a reset.

Fixes: 1857f1922ce2 ("event/dlb2: use new implementation of resource file")
Cc: stable@dpdk.org
Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>

diff --git a/drivers/event/dlb2/pf/base/dlb2_hw_types.h b/drivers/event/dlb2/pf/base/dlb2_hw_types.h
index 6b8fee3416afa3564d99d93b9ff354d65aa1bdff..9511521e67a778285ef210d6e75fcf61ead368db 100644 (file)
--- a/drivers/event/dlb2/pf/base/dlb2_hw_types.h
+++ b/drivers/event/dlb2/pf/base/dlb2_hw_types.h
@@ -27,7 +27,7 @@
 #define DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS    2
 #define DLB2_MAX_NUM_SEQUENCE_NUMBER_MODES     5
 #define DLB2_MAX_CQ_COMP_CHECK_LOOPS           409600
-#define DLB2_MAX_QID_EMPTY_CHECK_LOOPS         (32 * 64 * 1024 * (800 / 30))
+#define DLB2_MAX_QID_EMPTY_CHECK_LOOPS         (4 * DLB2_MAX_NUM_LDB_CREDITS)
 
 #define DLB2_FUNC_BAR                          0
 #define DLB2_CSR_BAR                           2

diff --git a/drivers/event/dlb2/pf/base/dlb2_resource.c b/drivers/event/dlb2/pf/base/dlb2_resource.c
index d4c49c29925b454167e1a552f8bc76120725c08c..8c57bdfff76f1c3c80f1bf4e7b87fee8139cc9fb 100644 (file)
--- a/drivers/event/dlb2/pf/base/dlb2_resource.c
+++ b/drivers/event/dlb2/pf/base/dlb2_resource.c
@@ -1057,7 +1057,7 @@ static u32 dlb2_dir_cq_token_count(struct dlb2_hw *hw,
               port->init_tkn_cnt;
 }
 
-static void dlb2_drain_dir_cq(struct dlb2_hw *hw,
+static int dlb2_drain_dir_cq(struct dlb2_hw *hw,
                              struct dlb2_dir_pq_pair *port)
 {
        unsigned int port_id = port->id.phys_id;
@@ -1089,6 +1089,8 @@ static void dlb2_drain_dir_cq(struct dlb2_hw *hw,
 
                os_unmap_producer_port(hw, pp_addr);
        }
+
+       return cnt;
 }
 
 static void dlb2_dir_port_cq_enable(struct dlb2_hw *hw,
@@ -1107,6 +1109,7 @@ static int dlb2_domain_drain_dir_cqs(struct dlb2_hw *hw,
 {
        struct dlb2_list_entry *iter;
        struct dlb2_dir_pq_pair *port;
+       int drain_cnt = 0;
        RTE_SET_USED(iter);
 
        DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
@@ -1120,13 +1123,13 @@ static int dlb2_domain_drain_dir_cqs(struct dlb2_hw *hw,
                if (toggle_port)
                        dlb2_dir_port_cq_disable(hw, port);
 
-               dlb2_drain_dir_cq(hw, port);
+               drain_cnt = dlb2_drain_dir_cq(hw, port);
 
                if (toggle_port)
                        dlb2_dir_port_cq_enable(hw, port);
        }
 
-       return 0;
+       return drain_cnt;
 }
 
 static u32 dlb2_dir_queue_depth(struct dlb2_hw *hw,
@@ -1170,10 +1173,20 @@ static int dlb2_domain_drain_dir_queues(struct dlb2_hw *hw,
                return 0;
 
        for (i = 0; i < DLB2_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
-               dlb2_domain_drain_dir_cqs(hw, domain, true);
+               int drain_cnt;
+
+               drain_cnt = dlb2_domain_drain_dir_cqs(hw, domain, false);
 
                if (dlb2_domain_dir_queues_empty(hw, domain))
                        break;
+
+               /*
+                * Allow time for DLB to schedule QEs before draining
+                * the CQs again.
+                */
+               if (!drain_cnt)
+                       rte_delay_us(1);
+
        }
 
        if (i == DLB2_MAX_QID_EMPTY_CHECK_LOOPS) {
@@ -1249,7 +1262,7 @@ static u32 dlb2_ldb_cq_token_count(struct dlb2_hw *hw,
                port->init_tkn_cnt;
 }
 
-static void dlb2_drain_ldb_cq(struct dlb2_hw *hw, struct dlb2_ldb_port *port)
+static int dlb2_drain_ldb_cq(struct dlb2_hw *hw, struct dlb2_ldb_port *port)
 {
        u32 infl_cnt, tkn_cnt;
        unsigned int i;
@@ -1289,32 +1302,37 @@ static void dlb2_drain_ldb_cq(struct dlb2_hw *hw, struct dlb2_ldb_port *port)
 
                os_unmap_producer_port(hw, pp_addr);
        }
+
+       return tkn_cnt;
 }
 
-static void dlb2_domain_drain_ldb_cqs(struct dlb2_hw *hw,
+static int dlb2_domain_drain_ldb_cqs(struct dlb2_hw *hw,
                                      struct dlb2_hw_domain *domain,
                                      bool toggle_port)
 {
        struct dlb2_list_entry *iter;
        struct dlb2_ldb_port *port;
+       int drain_cnt = 0;
        int i;
        RTE_SET_USED(iter);
 
        /* If the domain hasn't been started, there's no traffic to drain */
        if (!domain->started)
-               return;
+               return 0;
 
        for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
                DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) {
                        if (toggle_port)
                                dlb2_ldb_port_cq_disable(hw, port);
 
-                       dlb2_drain_ldb_cq(hw, port);
+                       drain_cnt = dlb2_drain_ldb_cq(hw, port);
 
                        if (toggle_port)
                                dlb2_ldb_port_cq_enable(hw, port);
                }
        }
+
+       return drain_cnt;
 }
 
 static u32 dlb2_ldb_queue_depth(struct dlb2_hw *hw,
@@ -1375,10 +1393,19 @@ static int dlb2_domain_drain_mapped_queues(struct dlb2_hw *hw,
        }
 
        for (i = 0; i < DLB2_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
-               dlb2_domain_drain_ldb_cqs(hw, domain, true);
+               int drain_cnt;
+
+               drain_cnt = dlb2_domain_drain_ldb_cqs(hw, domain, false);
 
                if (dlb2_domain_mapped_queues_empty(hw, domain))
                        break;
+
+               /*
+                * Allow time for DLB to schedule QEs before draining
+                * the CQs again.
+                */
+               if (!drain_cnt)
+                       rte_delay_us(1);
        }
 
        if (i == DLB2_MAX_QID_EMPTY_CHECK_LOOPS) {