net/sfc/base: support the Rx event mode w/o continue

The recently added NO_CONT_EV mode is recommended for when looking for
maximum throughput on 100G links as it allows the firmware to operate
more efficiently. The biggest benefit is when using scatter and jumbo
frames, but it is also necessary to achieve line rate in other cases.

Support for NO_CONT_EV when scatter is disabled is simple - the main
datapath change is to always read the packet length from the prefix,
not the event. This requires storing a flag with the event queue so
the event handler knows NO_CONT_EV mode is in use.

Supporting NO_CONT_EV with scatter would require an API change.

(Now the ee_flags field in efx_evq_t is used on the datapath, move it
next to other fields which are also read on the datapath to try to avoid
reading an additional cache line.)

Signed-off-by: Mark Spender <mspender@solarflare.com>
Signed-off-by: Andrew Rybchenko <arybchenko@solarflare.com>

diff --git a/drivers/net/sfc/base/ef10_ev.c b/drivers/net/sfc/base/ef10_ev.c
index b80cb0a..2029308 100644 (file)
--- a/drivers/net/sfc/base/ef10_ev.c
+++ b/drivers/net/sfc/base/ef10_ev.c
@@ -457,6 +457,23 @@ ef10_ev_qcreate(
                goto fail2;
        }
 
+       /*
+        * NO_CONT_EV mode is only requested from the firmware when creating
+        * receive queues, but here it needs to be specified at event queue
+        * creation, as the event handler needs to know which format is in use.
+        *
+        * If EFX_EVQ_FLAGS_NO_CONT_EV is specified, all receive queues for this
+        * event queue will be created in NO_CONT_EV mode.
+        *
+        * See SF-109306-TC 5.11 "Events for RXQs in NO_CONT_EV mode".
+        */
+       if (flags & EFX_EVQ_FLAGS_NO_CONT_EV) {
+               if (enp->en_nic_cfg.enc_no_cont_ev_mode_supported == B_FALSE) {
+                       rc = EINVAL;
+                       goto fail3;
+               }
+       }
+
        /* Set up the handler table */
        eep->ee_rx      = ef10_ev_rx;
        eep->ee_tx      = ef10_ev_tx;
@@ -494,7 +511,7 @@ ef10_ev_qcreate(
                rc = efx_mcdi_init_evq_v2(enp, index, esmp, ndescs, irq, us,
                    flags);
                if (rc != 0)
-                       goto fail3;
+                       goto fail4;
        } else {
                /*
                 * On Huntington we need to specify the settings to use.
@@ -511,11 +528,13 @@ ef10_ev_qcreate(
                rc = efx_mcdi_init_evq(enp, index, esmp, ndescs, irq, us, flags,
                    low_latency);
                if (rc != 0)
-                       goto fail4;
+                       goto fail5;
        }
 
        return (0);
 
+fail5:
+       EFSYS_PROBE(fail5);
 fail4:
        EFSYS_PROBE(fail4);
 fail3:
@@ -912,23 +931,47 @@ ef10_ev_rx(
        if (mac_class == ESE_DZ_MAC_CLASS_UCAST)
                flags |= EFX_PKT_UNICAST;
 
-       /* Increment the count of descriptors read */
+       /*
+        * Increment the count of descriptors read.
+        *
+        * In NO_CONT_EV mode, RX_DSC_PTR_LBITS is actually a packet count, but
+        * when scatter is disabled, there is only one descriptor per packet and
+        * so it can be treated the same.
+        *
+        * TODO: Support scatter in NO_CONT_EV mode.
+        */
        desc_count = (next_read_lbits - eersp->eers_rx_read_ptr) &
            EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
        eersp->eers_rx_read_ptr += desc_count;
 
-       /*
-        * FIXME: add error checking to make sure this a batched event.
-        * This could also be an aborted scatter, see Bug36629.
-        */
-       if (desc_count > 1) {
+       /* Calculate the index of the last descriptor consumed */
+       last_used_id = (eersp->eers_rx_read_ptr - 1) & eersp->eers_rx_mask;
+
+       if (eep->ee_flags & EFX_EVQ_FLAGS_NO_CONT_EV) {
+               if (desc_count > 1)
+                       EFX_EV_QSTAT_INCR(eep, EV_RX_BATCH);
+
+               /* Always read the length from the prefix in NO_CONT_EV mode. */
+               flags |= EFX_PKT_PREFIX_LEN;
+
+               /*
+                * Check for an aborted scatter, signalled by the ABORT bit in
+                * NO_CONT_EV mode. The ABORT bit was not used before NO_CONT_EV
+                * mode was added as it was broken in Huntington silicon.
+                */
+               if (EFX_QWORD_FIELD(*eqp, ESF_EZ_RX_ABORT) != 0) {
+                       flags |= EFX_DISCARD;
+                       goto deliver;
+               }
+       } else if (desc_count > 1) {
+               /*
+                * FIXME: add error checking to make sure this a batched event.
+                * This could also be an aborted scatter, see Bug36629.
+                */
                EFX_EV_QSTAT_INCR(eep, EV_RX_BATCH);
                flags |= EFX_PKT_PREFIX_LEN;
        }
 
-       /* Calculate the index of the last descriptor consumed */
-       last_used_id = (eersp->eers_rx_read_ptr - 1) & eersp->eers_rx_mask;
-
        /* Check for errors that invalidate checksum and L3/L4 fields */
        if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TRUNC_ERR) != 0) {
                /* RX frame truncated */

diff --git a/drivers/net/sfc/base/ef10_nic.c b/drivers/net/sfc/base/ef10_nic.c
index d689206..39ca53f 100644 (file)
--- a/drivers/net/sfc/base/ef10_nic.c
+++ b/drivers/net/sfc/base/ef10_nic.c
@@ -1197,6 +1197,14 @@ ef10_get_datapath_caps(
        else
                encp->enc_init_evq_v2_supported = B_FALSE;
 
+       /*
+        * Check if the NO_CONT_EV mode for RX events is supported.
+        */
+       if (CAP_FLAGS2(req, INIT_RXQ_NO_CONT_EV))
+               encp->enc_no_cont_ev_mode_supported = B_TRUE;
+       else
+               encp->enc_no_cont_ev_mode_supported = B_FALSE;
+
        /*
         * Check if firmware-verified NVRAM updates must be used.
         *

diff --git a/drivers/net/sfc/base/ef10_rx.c b/drivers/net/sfc/base/ef10_rx.c
index 23b80d7..3b296e4 100644 (file)
--- a/drivers/net/sfc/base/ef10_rx.c
+++ b/drivers/net/sfc/base/ef10_rx.c
@@ -15,7 +15,7 @@ static        __checkReturn   efx_rc_t
 efx_mcdi_init_rxq(
        __in            efx_nic_t *enp,
        __in            uint32_t ndescs,
-       __in            uint32_t target_evq,
+       __in            efx_evq_t *eep,
        __in            uint32_t label,
        __in            uint32_t instance,
        __in            efsys_mem_t *esmp,
@@ -38,6 +38,7 @@ efx_mcdi_init_rxq(
        efx_rc_t rc;
        uint32_t dma_mode;
        boolean_t want_outer_classes;
+       boolean_t no_cont_ev;
 
        EFSYS_ASSERT3U(ndescs, <=, encp->enc_rxq_max_ndescs);
 
@@ -47,6 +48,13 @@ efx_mcdi_init_rxq(
                goto fail1;
        }
 
+       no_cont_ev = (eep->ee_flags & EFX_EVQ_FLAGS_NO_CONT_EV);
+       if ((no_cont_ev == B_TRUE) && (disable_scatter == B_FALSE)) {
+               /* TODO: Support scatter in NO_CONT_EV mode */
+               rc = EINVAL;
+               goto fail2;
+       }
+
        if (ps_bufsize > 0)
                dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
        else if (es_bufs_per_desc > 0)
@@ -81,10 +89,10 @@ efx_mcdi_init_rxq(
        req.emr_out_length = MC_CMD_INIT_RXQ_V3_OUT_LEN;
 
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
-       MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
+       MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, eep->ee_index);
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
-       MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
+       MCDI_IN_POPULATE_DWORD_10(req, INIT_RXQ_EXT_IN_FLAGS,
            INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
            INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
            INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
@@ -94,7 +102,8 @@ efx_mcdi_init_rxq(
            INIT_RXQ_EXT_IN_DMA_MODE,
            dma_mode,
            INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
-           INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
+           INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes,
+           INIT_RXQ_EXT_IN_FLAG_NO_CONT_EV, no_cont_ev);
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
        MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
 
@@ -127,11 +136,13 @@ efx_mcdi_init_rxq(
 
        if (req.emr_rc != 0) {
                rc = req.emr_rc;
-               goto fail2;
+               goto fail3;
        }
 
        return (0);
 
+fail3:
+       EFSYS_PROBE(fail3);
 fail2:
        EFSYS_PROBE(fail2);
 fail1:
@@ -1122,7 +1133,7 @@ ef10_rx_qcreate(
        else
                want_inner_classes = B_FALSE;
 
-       if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
+       if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index,
                    esmp, disable_scatter, want_inner_classes,
                    ps_buf_size, es_bufs_per_desc, es_max_dma_len,
                    es_buf_stride, hol_block_timeout)) != 0)

diff --git a/drivers/net/sfc/base/efx.h b/drivers/net/sfc/base/efx.h
index f7d0a4f..f5ad095 100644 (file)
--- a/drivers/net/sfc/base/efx.h
+++ b/drivers/net/sfc/base/efx.h
@@ -1370,6 +1370,7 @@ typedef struct efx_nic_cfg_s {
        boolean_t               enc_allow_set_mac_with_installed_filters;
        boolean_t               enc_enhanced_set_mac_supported;
        boolean_t               enc_init_evq_v2_supported;
+       boolean_t               enc_no_cont_ev_mode_supported;
        boolean_t               enc_rx_packed_stream_supported;
        boolean_t               enc_rx_var_packed_stream_supported;
        boolean_t               enc_rx_es_super_buffer_supported;
@@ -2000,6 +2001,16 @@ efx_evq_nbufs(
 #define        EFX_EVQ_FLAGS_NOTIFY_INTERRUPT  (0x0)   /* Interrupting (default) */
 #define        EFX_EVQ_FLAGS_NOTIFY_DISABLED   (0x4)   /* Non-interrupting */
 
+/*
+ * Use the NO_CONT_EV RX event format, which allows the firmware to operate more
+ * efficiently at high data rates. See SF-109306-TC 5.11 "Events for RXQs in
+ * NO_CONT_EV mode".
+ *
+ * NO_CONT_EV requires EVQ_RX_MERGE and RXQ_FORCED_EV_MERGING to both be set,
+ * which is the case when an event queue is set to THROUGHPUT mode.
+ */
+#define        EFX_EVQ_FLAGS_NO_CONT_EV        (0x10)
+
 extern __checkReturn   efx_rc_t
 efx_ev_qcreate(
        __in            efx_nic_t *enp,

diff --git a/drivers/net/sfc/base/efx_impl.h b/drivers/net/sfc/base/efx_impl.h
index bad23f8..e2cdba6 100644 (file)
--- a/drivers/net/sfc/base/efx_impl.h
+++ b/drivers/net/sfc/base/efx_impl.h
@@ -760,6 +760,7 @@ typedef struct efx_evq_rxq_state_s {
 
 struct efx_evq_s {
        uint32_t                        ee_magic;
+       uint32_t                        ee_flags;
        efx_nic_t                       *ee_enp;
        unsigned int                    ee_index;
        unsigned int                    ee_mask;
@@ -778,8 +779,6 @@ struct efx_evq_s {
 #endif /* EFSYS_OPT_MCDI */
 
        efx_evq_rxq_state_t             ee_rxq_state[EFX_EV_RX_NLABELS];
-
-       uint32_t                        ee_flags;
 };
 
 #define        EFX_EVQ_MAGIC   0x08081997