net/bnxt: fix scalar Rx datapath on Thor

[dpdk.git] / drivers / net / bnxt / bnxt_cpr.h

diff --git a/drivers/net/bnxt/bnxt_cpr.h b/drivers/net/bnxt/bnxt_cpr.h
index a763f60..73468ed 100644 (file)
--- a/drivers/net/bnxt/bnxt_cpr.h
+++ b/drivers/net/bnxt/bnxt_cpr.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2014-2018 Broadcom
+ * Copyright(c) 2014-2021 Broadcom
  * All rights reserved.
  */
 
@@ -8,41 +8,26 @@
 #include <stdbool.h>
 
 #include <rte_io.h>
+#include "hsi_struct_def_dpdk.h"
 
 struct bnxt_db_info;
 
-#define CMP_VALID(cmp, raw_cons, ring)                                 \
-       (!!(rte_le_to_cpu_32(((struct cmpl_base *)(cmp))->info3_v) &    \
-           CMPL_BASE_V) == !((raw_cons) & ((ring)->ring_size)))
-
-#define CMPL_VALID(cmp, v)                                             \
-       (!!(rte_le_to_cpu_32(((struct cmpl_base *)(cmp))->info3_v) &    \
-           CMPL_BASE_V) == !(v))
-
-#define NQ_CMP_VALID(nqcmp, raw_cons, ring)            \
-       (!!((nqcmp)->v & rte_cpu_to_le_32(NQ_CN_V)) ==  \
-        !((raw_cons) & ((ring)->ring_size)))
-
 #define CMP_TYPE(cmp)                                          \
        (((struct cmpl_base *)cmp)->type & CMPL_BASE_TYPE_MASK)
 
+/* Get completion length from completion type, in 16-byte units. */
+#define CMP_LEN(cmp_type) (((cmp_type) & 1) + 1)
+
+
 #define ADV_RAW_CMP(idx, n)    ((idx) + (n))
 #define NEXT_RAW_CMP(idx)      ADV_RAW_CMP(idx, 1)
 #define RING_CMP(ring, idx)    ((idx) & (ring)->ring_mask)
 #define RING_CMPL(ring_mask, idx)      ((idx) & (ring_mask))
 #define NEXT_CMP(idx)          RING_CMP(ADV_RAW_CMP(idx, 1))
-#define FLIP_VALID(cons, mask, val)    ((cons) >= (mask) ? !(val) : (val))
 
 #define DB_CP_REARM_FLAGS      (DB_KEY_CP | DB_IDX_VALID)
 #define DB_CP_FLAGS            (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 
-#define NEXT_CMPL(cpr, idx, v, inc)    do { \
-       (idx) += (inc); \
-       if (unlikely((idx) >= (cpr)->cp_ring_struct->ring_size)) { \
-               (v) = !(v); \
-               (idx) = 0; \
-       } \
-} while (0)
 #define B_CP_DB_REARM(cpr, raw_cons)                                   \
        rte_write32((DB_CP_REARM_FLAGS |                                \
                    DB_RING_IDX(&((cpr)->cp_db), raw_cons)),            \
@@ -81,9 +66,14 @@ struct bnxt_db_info {
        };
        bool                    db_64;
        uint32_t                db_ring_mask;
+       uint32_t                db_epoch_mask;
+       uint32_t                db_epoch_shift;
 };
 
-#define DB_RING_IDX(db, idx)   ((idx) & (db)->db_ring_mask)
+#define DB_EPOCH(db, idx)      (((idx) & (db)->db_epoch_mask) <<       \
+                                ((db)->db_epoch_shift))
+#define DB_RING_IDX(db, idx)   (((idx) & (db)->db_ring_mask) |         \
+                                DB_EPOCH(db, idx))
 
 struct bnxt_ring;
 struct bnxt_cp_ring_info {
@@ -98,7 +88,6 @@ struct bnxt_cp_ring_info {
        uint32_t                hw_stats_ctx_id;
 
        struct bnxt_ring        *cp_ring_struct;
-       bool                    valid;
 };
 
 #define RX_CMP_L2_ERRORS                                               \
@@ -126,7 +115,38 @@ void bnxt_wait_for_device_shutdown(struct bnxt *bp);
        HWRM_ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_RECOVERY_ENABLED
 
 bool bnxt_is_recovery_enabled(struct bnxt *bp);
-bool bnxt_is_master_func(struct bnxt *bp);
+bool bnxt_is_primary_func(struct bnxt *bp);
 
 void bnxt_stop_rxtx(struct bnxt *bp);
+
+/**
+ * Check validity of a completion ring entry. If the entry is valid, include a
+ * C11 __ATOMIC_ACQUIRE fence to ensure that subsequent loads of fields in the
+ * completion are not hoisted by the compiler or by the CPU to come before the
+ * loading of the "valid" field.
+ *
+ * Note: the caller must not access any fields in the specified completion
+ * entry prior to calling this function.
+ *
+ * @param cmpl
+ *   Pointer to an entry in the completion ring.
+ * @param raw_cons
+ *   Raw consumer index of entry in completion ring.
+ * @param ring_size
+ *   Size of completion ring.
+ */
+static __rte_always_inline bool
+bnxt_cpr_cmp_valid(const void *cmpl, uint32_t raw_cons, uint32_t ring_size)
+{
+       const struct cmpl_base *c = cmpl;
+       bool expected, valid;
+
+       expected = !(raw_cons & ring_size);
+       valid = !!(rte_le_to_cpu_32(c->info3_v) & CMPL_BASE_V);
+       if (valid == expected) {
+               rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
+               return true;
+       }
+       return false;
+}
 #endif