net/sfc/base: support equal stride super-buffer Rx mode

Equal stride super-buffer Rx mode is supported by DPDK firmware
variant. One Rx descriptor provides many Rx buffers to firmware.
Rx buffers follow each other with specified stride.
Also it supports head of line blocking with timeout to address
drops when no Rx descriptors are available. So it gives extra time
to the driver to provide Rx descriptors before drop.

Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>

diff --git a/drivers/net/sfc/base/ef10_ev.c b/drivers/net/sfc/base/ef10_ev.c
index 6e00099ccc0dcebc28da0a4c2054f81aabfb56e4..7f89a7bf0a33a15d700c0f1557397e295e8e21ee 100644 (file)
--- a/drivers/net/sfc/base/ef10_ev.c
+++ b/drivers/net/sfc/base/ef10_ev.c
@@ -749,7 +749,7 @@ ef10_ev_qstats_update(
 }
 #endif /* EFSYS_OPT_QSTATS */
 
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
 
 static __checkReturn   boolean_t
 ef10_ev_rx_packed_stream(
@@ -788,7 +788,18 @@ ef10_ev_rx_packed_stream(
 
        if (new_buffer) {
                flags |= EFX_PKT_PACKED_STREAM_NEW_BUFFER;
+#if EFSYS_OPT_RX_PACKED_STREAM
+               /*
+                * If both packed stream and equal stride super-buffer
+                * modes are compiled in, in theory credits should be
+                * be maintained for packed stream only, but right now
+                * these modes are not distinguished in the event queue
+                * Rx queue state and it is OK to increment the counter
+                * regardless (it might be event cheaper than branching
+                * since neighbour structure member are updated as well).
+                */
                eersp->eers_rx_packed_stream_credits++;
+#endif
                eersp->eers_rx_read_ptr++;
        }
        current_id = eersp->eers_rx_read_ptr & eersp->eers_rx_mask;
@@ -830,7 +841,7 @@ deliver:
        return (should_abort);
 }
 
-#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+#endif /* EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER */
 
 static __checkReturn   boolean_t
 ef10_ev_rx(
@@ -864,7 +875,7 @@ ef10_ev_rx(
        label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL);
        eersp = &eep->ee_rxq_state[label];
 
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
        /*
         * Packed stream events are very different,
         * so handle them separately
@@ -1364,8 +1375,9 @@ ef10_ev_rxlabel_init(
        __in            efx_rxq_type_t type)
 {
        efx_evq_rxq_state_t *eersp;
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
        boolean_t packed_stream = (type == EFX_RXQ_TYPE_PACKED_STREAM);
+       boolean_t es_super_buffer = (type == EFX_RXQ_TYPE_ES_SUPER_BUFFER);
 #endif
 
        _NOTE(ARGUNUSED(type))
@@ -1387,9 +1399,11 @@ ef10_ev_rxlabel_init(
        eersp->eers_rx_read_ptr = 0;
 #endif
        eersp->eers_rx_mask = erp->er_mask;
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
        eersp->eers_rx_stream_npackets = 0;
-       eersp->eers_rx_packed_stream = packed_stream;
+       eersp->eers_rx_packed_stream = packed_stream || es_super_buffer;
+#endif
+#if EFSYS_OPT_RX_PACKED_STREAM
        if (packed_stream) {
                eersp->eers_rx_packed_stream_credits = (eep->ee_mask + 1) /
                    EFX_DIV_ROUND_UP(EFX_RX_PACKED_STREAM_MEM_PER_CREDIT,
@@ -1423,9 +1437,11 @@ ef10_ev_rxlabel_fini(
 
        eersp->eers_rx_read_ptr = 0;
        eersp->eers_rx_mask = 0;
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
        eersp->eers_rx_stream_npackets = 0;
        eersp->eers_rx_packed_stream = B_FALSE;
+#endif
+#if EFSYS_OPT_RX_PACKED_STREAM
        eersp->eers_rx_packed_stream_credits = 0;
 #endif
 }

diff --git a/drivers/net/sfc/base/ef10_impl.h b/drivers/net/sfc/base/ef10_impl.h
index 36229a74e0a0453d9824f4aa4606ca719630ac8b..4751faf16842193c2b997bf04548370721b6ecae 100644 (file)
--- a/drivers/net/sfc/base/ef10_impl.h
+++ b/drivers/net/sfc/base/ef10_impl.h
@@ -1216,6 +1216,16 @@ efx_mcdi_set_nic_global(
 
 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+
+/*
+ * Maximum DMA length and buffer stride alignment.
+ * (see SF-119419-TC, 3.2)
+ */
+#define        EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT    64
+
+#endif
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/drivers/net/sfc/base/ef10_rx.c b/drivers/net/sfc/base/ef10_rx.c
index 0ed5ffdbc60d97a87d71b0ad4f8f2f012c8f971a..313a369180f133e62d88c2e836ddd0f8c991e2c3 100644 (file)
--- a/drivers/net/sfc/base/ef10_rx.c
+++ b/drivers/net/sfc/base/ef10_rx.c
@@ -21,12 +21,16 @@ efx_mcdi_init_rxq(
        __in            efsys_mem_t *esmp,
        __in            boolean_t disable_scatter,
        __in            boolean_t want_inner_classes,
-       __in            uint32_t ps_bufsize)
+       __in            uint32_t ps_bufsize,
+       __in            uint32_t es_bufs_per_desc,
+       __in            uint32_t es_max_dma_len,
+       __in            uint32_t es_buf_stride,
+       __in            uint32_t hol_block_timeout)
 {
        efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
        efx_mcdi_req_t req;
-       uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
-                           MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
+       uint8_t payload[MAX(MC_CMD_INIT_RXQ_V3_IN_LEN,
+                           MC_CMD_INIT_RXQ_V3_OUT_LEN)];
        int npages = EFX_RXQ_NBUFS(ndescs);
        int i;
        efx_qword_t *dma_addr;
@@ -44,6 +48,8 @@ efx_mcdi_init_rxq(
 
        if (ps_bufsize > 0)
                dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
+       else if (es_bufs_per_desc > 0)
+               dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER;
        else
                dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
 
@@ -70,9 +76,9 @@ efx_mcdi_init_rxq(
        (void) memset(payload, 0, sizeof (payload));
        req.emr_cmd = MC_CMD_INIT_RXQ;
        req.emr_in_buf = payload;
-       req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
+       req.emr_in_length = MC_CMD_INIT_RXQ_V3_IN_LEN;
        req.emr_out_buf = payload;
-       req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
+       req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
 
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
@@ -92,6 +98,19 @@ efx_mcdi_init_rxq(
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
 
+       if (es_bufs_per_desc > 0) {
+               MCDI_IN_SET_DWORD(req,
+                   INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET,
+                   es_bufs_per_desc);
+               MCDI_IN_SET_DWORD(req,
+                   INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len);
+               MCDI_IN_SET_DWORD(req,
+                   INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride);
+               MCDI_IN_SET_DWORD(req,
+                   INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT,
+                   hol_block_timeout);
+       }
+
        dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
        addr = EFSYS_MEM_ADDR(esmp);
 
@@ -1013,6 +1032,10 @@ ef10_rx_qcreate(
        boolean_t disable_scatter;
        boolean_t want_inner_classes;
        unsigned int ps_buf_size;
+       uint32_t es_bufs_per_desc = 0;
+       uint32_t es_max_dma_len = 0;
+       uint32_t es_buf_stride = 0;
+       uint32_t hol_block_timeout = 0;
 
        _NOTE(ARGUNUSED(id, erp, type_data))
 
@@ -1061,6 +1084,19 @@ ef10_rx_qcreate(
                }
                break;
 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+       case EFX_RXQ_TYPE_ES_SUPER_BUFFER:
+               ps_buf_size = 0;
+               es_bufs_per_desc =
+                   type_data->ertd_es_super_buffer.eessb_bufs_per_desc;
+               es_max_dma_len =
+                   type_data->ertd_es_super_buffer.eessb_max_dma_len;
+               es_buf_stride =
+                   type_data->ertd_es_super_buffer.eessb_buf_stride;
+               hol_block_timeout =
+                   type_data->ertd_es_super_buffer.eessb_hol_block_timeout;
+               break;
+#endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
        default:
                rc = ENOTSUP;
                goto fail4;
@@ -1084,6 +1120,27 @@ ef10_rx_qcreate(
        EFSYS_ASSERT(ps_buf_size == 0);
 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
 
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+       if (es_bufs_per_desc > 0) {
+               if (encp->enc_rx_es_super_buffer_supported == B_FALSE) {
+                       rc = ENOTSUP;
+                       goto fail7;
+               }
+               if (!IS_P2ALIGNED(es_max_dma_len,
+                           EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
+                       rc = EINVAL;
+                       goto fail8;
+               }
+               if (!IS_P2ALIGNED(es_buf_stride,
+                           EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) {
+                       rc = EINVAL;
+                       goto fail9;
+               }
+       }
+#else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
+       EFSYS_ASSERT(es_bufs_per_desc == 0);
+#endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
+
        /* Scatter can only be disabled if the firmware supports doing so */
        if (flags & EFX_RXQ_FLAG_SCATTER)
                disable_scatter = B_FALSE;
@@ -1097,8 +1154,9 @@ ef10_rx_qcreate(
 
        if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
                    esmp, disable_scatter, want_inner_classes,
-                   ps_buf_size)) != 0)
-               goto fail7;
+                   ps_buf_size, es_bufs_per_desc, es_max_dma_len,
+                   es_buf_stride, hol_block_timeout)) != 0)
+               goto fail10;
 
        erp->er_eep = eep;
        erp->er_label = label;
@@ -1109,8 +1167,16 @@ ef10_rx_qcreate(
 
        return (0);
 
+fail10:
+       EFSYS_PROBE(fail10);
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+fail9:
+       EFSYS_PROBE(fail9);
+fail8:
+       EFSYS_PROBE(fail8);
 fail7:
        EFSYS_PROBE(fail7);
+#endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */
 #if EFSYS_OPT_RX_PACKED_STREAM
 fail6:
        EFSYS_PROBE(fail6);

diff --git a/drivers/net/sfc/base/efx.h b/drivers/net/sfc/base/efx.h
index dea8d60d520b243c295058b1e9d7b3c636ef028e..b334cc5b4743bb228d28361103d051fa5bd96179 100644 (file)
--- a/drivers/net/sfc/base/efx.h
+++ b/drivers/net/sfc/base/efx.h
@@ -1864,7 +1864,7 @@ typedef   __checkReturn   boolean_t
        __in            uint32_t size,
        __in            uint16_t flags);
 
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
 
 /*
  * Packed stream mode is documented in SF-112241-TC.
@@ -1874,6 +1874,13 @@ typedef  __checkReturn   boolean_t
  * packets are put there in a continuous stream.
  * The main advantage of such an approach is that RX queue refilling
  * happens much less frequently.
+ *
+ * Equal stride packed stream mode is documented in SF-119419-TC.
+ * The general idea is to utilize advantages of the packed stream,
+ * but avoid indirection in packets representation.
+ * The main advantage of such an approach is that RX queue refilling
+ * happens much less frequently and packets buffers are independent
+ * from upper layers point of view.
  */
 
 typedef        __checkReturn   boolean_t
@@ -1974,7 +1981,7 @@ typedef __checkReturn     boolean_t
 typedef struct efx_ev_callbacks_s {
        efx_initialized_ev_t            eec_initialized;
        efx_rx_ev_t                     eec_rx;
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
        efx_rx_ps_ev_t                  eec_rx_ps;
 #endif
        efx_tx_ev_t                     eec_tx;
@@ -2281,6 +2288,7 @@ efx_pseudo_hdr_pkt_length_get(
 typedef enum efx_rxq_type_e {
        EFX_RXQ_TYPE_DEFAULT,
        EFX_RXQ_TYPE_PACKED_STREAM,
+       EFX_RXQ_TYPE_ES_SUPER_BUFFER,
        EFX_RXQ_NTYPES
 } efx_rxq_type_t;
 
@@ -2334,6 +2342,28 @@ efx_rx_qcreate_packed_stream(
 
 #endif
 
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+
+/* Maximum head-of-line block timeout in nanoseconds */
+#define        EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX   (400U * 1000 * 1000)
+
+extern __checkReturn   efx_rc_t
+efx_rx_qcreate_es_super_buffer(
+       __in            efx_nic_t *enp,
+       __in            unsigned int index,
+       __in            unsigned int label,
+       __in            uint32_t n_bufs_per_desc,
+       __in            uint32_t max_dma_len,
+       __in            uint32_t buf_stride,
+       __in            uint32_t hol_block_timeout,
+       __in            efsys_mem_t *esmp,
+       __in            size_t ndescs,
+       __in            unsigned int flags,
+       __in            efx_evq_t *eep,
+       __deref_out     efx_rxq_t **erpp);
+
+#endif
+
 typedef struct efx_buffer_s {
        efsys_dma_addr_t        eb_addr;
        size_t                  eb_size;

diff --git a/drivers/net/sfc/base/efx_check.h b/drivers/net/sfc/base/efx_check.h
index 52b0c7979a47e437d4d85c7ee19fadf457d6a0dd..ef5eadc6255798cf786c9c7d8f0d511687f0fa4f 100644 (file)
--- a/drivers/net/sfc/base/efx_check.h
+++ b/drivers/net/sfc/base/efx_check.h
@@ -343,6 +343,13 @@
 # endif
 #endif
 
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+/* Support equal stride super-buffer mode */
+# if !(EFSYS_OPT_MEDFORD2)
+#  error "ES_SUPER_BUFFER requires MEDFORD2"
+# endif
+#endif
+
 /* Support hardware assistance for tunnels */
 #if EFSYS_OPT_TUNNEL
 # if !(EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2)

diff --git a/drivers/net/sfc/base/efx_impl.h b/drivers/net/sfc/base/efx_impl.h
index f130713f3f3cd01196cb0737f7b06b7242bd0e42..548834f90b6a80aff096d2059e6318931d952c17 100644 (file)
--- a/drivers/net/sfc/base/efx_impl.h
+++ b/drivers/net/sfc/base/efx_impl.h
@@ -137,6 +137,14 @@ typedef union efx_rxq_type_data_u {
                uint32_t        eps_buf_size;
        } ertd_packed_stream;
 #endif
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+       struct {
+               uint32_t        eessb_bufs_per_desc;
+               uint32_t        eessb_max_dma_len;
+               uint32_t        eessb_buf_stride;
+               uint32_t        eessb_hol_block_timeout;
+       } ertd_es_super_buffer;
+#endif
 } efx_rxq_type_data_t;
 
 typedef struct efx_rx_ops_s {
@@ -735,9 +743,11 @@ typedef    boolean_t (*efx_ev_handler_t)(efx_evq_t *, efx_qword_t *,
 typedef struct efx_evq_rxq_state_s {
        unsigned int                    eers_rx_read_ptr;
        unsigned int                    eers_rx_mask;
-#if EFSYS_OPT_RX_PACKED_STREAM
+#if EFSYS_OPT_RX_PACKED_STREAM || EFSYS_OPT_RX_ES_SUPER_BUFFER
        unsigned int                    eers_rx_stream_npackets;
        boolean_t                       eers_rx_packed_stream;
+#endif
+#if EFSYS_OPT_RX_PACKED_STREAM
        unsigned int                    eers_rx_packed_stream_credits;
 #endif
 } efx_evq_rxq_state_t;

diff --git a/drivers/net/sfc/base/efx_rx.c b/drivers/net/sfc/base/efx_rx.c
index aab57952becaa2b56b48e029fad9874e253cd4cb..4fd73bab33468c7a97b498364e15db99f2ab9142 100644 (file)
--- a/drivers/net/sfc/base/efx_rx.c
+++ b/drivers/net/sfc/base/efx_rx.c
@@ -842,6 +842,58 @@ efx_rx_qcreate_packed_stream(
 
 #endif
 
+#if EFSYS_OPT_RX_ES_SUPER_BUFFER
+
+       __checkReturn   efx_rc_t
+efx_rx_qcreate_es_super_buffer(
+       __in            efx_nic_t *enp,
+       __in            unsigned int index,
+       __in            unsigned int label,
+       __in            uint32_t n_bufs_per_desc,
+       __in            uint32_t max_dma_len,
+       __in            uint32_t buf_stride,
+       __in            uint32_t hol_block_timeout,
+       __in            efsys_mem_t *esmp,
+       __in            size_t ndescs,
+       __in            unsigned int flags,
+       __in            efx_evq_t *eep,
+       __deref_out     efx_rxq_t **erpp)
+{
+       efx_rc_t rc;
+       efx_rxq_type_data_t type_data;
+
+       if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) {
+               rc = EINVAL;
+               goto fail1;
+       }
+
+       memset(&type_data, 0, sizeof(type_data));
+
+       type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc;
+       type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len;
+       type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride;
+       type_data.ertd_es_super_buffer.eessb_hol_block_timeout =
+           hol_block_timeout;
+
+       rc = efx_rx_qcreate_internal(enp, index, label,
+           EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs,
+           0 /* id unused on EF10 */, flags, eep, erpp);
+       if (rc != 0)
+               goto fail2;
+
+       return (0);
+
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+       return (rc);
+}
+
+#endif
+
+
                        void
 efx_rx_qdestroy(
        __in            efx_rxq_t *erp)

diff --git a/drivers/net/sfc/efsys.h b/drivers/net/sfc/efsys.h
index 12f77dc52fdf8b37b414db7aa326174cedf2a809..f71581c861dd186a6cba094386a8c1e739b15feb 100644 (file)
--- a/drivers/net/sfc/efsys.h
+++ b/drivers/net/sfc/efsys.h
@@ -198,6 +198,8 @@ prefetch_read_once(const volatile void *addr)
 
 #define EFSYS_OPT_RX_PACKED_STREAM 0
 
+#define EFSYS_OPT_RX_ES_SUPER_BUFFER 0
+
 #define EFSYS_OPT_TUNNEL 1
 
 #define EFSYS_OPT_FW_SUBVARIANT_AWARE 1