net/sfc: support parameter to choose performance profile
[dpdk.git] / drivers / net / sfc / sfc_ev.c
index 2bcc2be..de42845 100644 (file)
@@ -30,6 +30,7 @@
 #include <rte_debug.h>
 #include <rte_cycles.h>
 #include <rte_alarm.h>
+#include <rte_branch_prediction.h>
 
 #include "efx.h"
 
@@ -38,6 +39,8 @@
 #include "sfc_log.h"
 #include "sfc_ev.h"
 #include "sfc_rx.h"
+#include "sfc_tx.h"
+#include "sfc_kvargs.h"
 
 
 /* Initial delay when waiting for event queue init complete event */
@@ -66,22 +69,103 @@ sfc_ev_initialized(void *arg)
 }
 
 static boolean_t
-sfc_ev_rx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id,
-         __rte_unused uint32_t size, __rte_unused uint16_t flags)
+sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
+         uint32_t size, uint16_t flags)
 {
        struct sfc_evq *evq = arg;
+       struct sfc_rxq *rxq;
+       unsigned int stop;
+       unsigned int pending_id;
+       unsigned int delta;
+       unsigned int i;
+       struct sfc_rx_sw_desc *rxd;
 
-       sfc_err(evq->sa, "EVQ %u unexpected Rx event", evq->evq_index);
-       return B_TRUE;
+       if (unlikely(evq->exception))
+               goto done;
+
+       rxq = evq->rxq;
+
+       SFC_ASSERT(rxq != NULL);
+       SFC_ASSERT(rxq->evq == evq);
+       SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
+
+       stop = (id + 1) & rxq->ptr_mask;
+       pending_id = rxq->pending & rxq->ptr_mask;
+       delta = (stop >= pending_id) ? (stop - pending_id) :
+               (rxq->ptr_mask + 1 - pending_id + stop);
+
+       if (delta == 0) {
+               /*
+                * Rx event with no new descriptors done and zero length
+                * is used to abort scattered packet when there is no room
+                * for the tail.
+                */
+               if (unlikely(size != 0)) {
+                       evq->exception = B_TRUE;
+                       sfc_err(evq->sa,
+                               "EVQ %u RxQ %u invalid RX abort "
+                               "(id=%#x size=%u flags=%#x); needs restart\n",
+                               evq->evq_index, sfc_rxq_sw_index(rxq),
+                               id, size, flags);
+                       goto done;
+               }
+
+               /* Add discard flag to the first fragment */
+               rxq->sw_desc[pending_id].flags |= EFX_DISCARD;
+               /* Remove continue flag from the last fragment */
+               rxq->sw_desc[id].flags &= ~EFX_PKT_CONT;
+       } else if (unlikely(delta > rxq->batch_max)) {
+               evq->exception = B_TRUE;
+
+               sfc_err(evq->sa,
+                       "EVQ %u RxQ %u completion out of order "
+                       "(id=%#x delta=%u flags=%#x); needs restart\n",
+                       evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
+                       flags);
+
+               goto done;
+       }
+
+       for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) {
+               rxd = &rxq->sw_desc[i];
+
+               rxd->flags = flags;
+
+               SFC_ASSERT(size < (1 << 16));
+               rxd->size = (uint16_t)size;
+       }
+
+       rxq->pending += delta;
+
+done:
+       return B_FALSE;
 }
 
 static boolean_t
-sfc_ev_tx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id)
+sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
 {
        struct sfc_evq *evq = arg;
+       struct sfc_txq *txq;
+       unsigned int stop;
+       unsigned int delta;
 
-       sfc_err(evq->sa, "EVQ %u unexpected Tx event", evq->evq_index);
-       return B_TRUE;
+       txq = evq->txq;
+
+       SFC_ASSERT(txq != NULL);
+       SFC_ASSERT(txq->evq == evq);
+
+       if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
+               goto done;
+
+       stop = (id + 1) & txq->ptr_mask;
+       id = txq->pending & txq->ptr_mask;
+
+       delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop);
+
+       txq->pending += delta;
+
+done:
+       return B_FALSE;
 }
 
 static boolean_t
@@ -145,10 +229,15 @@ static boolean_t
 sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
 {
        struct sfc_evq *evq = arg;
+       struct sfc_txq *txq;
 
-       sfc_err(evq->sa, "EVQ %u unexpected Tx flush done event",
-               evq->evq_index);
-       return B_TRUE;
+       txq = evq->txq;
+       SFC_ASSERT(txq != NULL);
+       SFC_ASSERT(txq->hw_index == txq_hw_index);
+       SFC_ASSERT(txq->evq == evq);
+       sfc_tx_qflush_done(txq);
+
+       return B_FALSE;
 }
 
 static boolean_t
@@ -233,6 +322,45 @@ sfc_ev_qpoll(struct sfc_evq *evq)
 
        efx_ev_qpoll(evq->common, &evq->read_ptr, &sfc_ev_callbacks, evq);
 
+       if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) {
+               struct sfc_adapter *sa = evq->sa;
+               int rc;
+
+               if ((evq->rxq != NULL) && (evq->rxq->state & SFC_RXQ_RUNNING)) {
+                       unsigned int rxq_sw_index = sfc_rxq_sw_index(evq->rxq);
+
+                       sfc_warn(sa,
+                                "restart RxQ %u because of exception on its EvQ %u",
+                                rxq_sw_index, evq->evq_index);
+
+                       sfc_rx_qstop(sa, rxq_sw_index);
+                       rc = sfc_rx_qstart(sa, rxq_sw_index);
+                       if (rc != 0)
+                               sfc_err(sa, "cannot restart RxQ %u",
+                                       rxq_sw_index);
+               }
+
+               if (evq->txq != NULL) {
+                       unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
+
+                       sfc_warn(sa,
+                                "restart TxQ %u because of exception on its EvQ %u",
+                                txq_sw_index, evq->evq_index);
+
+                       sfc_tx_qstop(sa, txq_sw_index);
+                       rc = sfc_tx_qstart(sa, txq_sw_index);
+                       if (rc != 0)
+                               sfc_err(sa, "cannot restart TxQ %u",
+                                       txq_sw_index);
+               }
+
+               if (evq->exception)
+                       sfc_panic(sa, "unrecoverable exception on EvQ %u",
+                                 evq->evq_index);
+
+               sfc_adapter_unlock(sa);
+       }
+
        /* Poll-mode driver does not re-prime the event queue for interrupts */
 }
 
@@ -278,9 +406,7 @@ sfc_ev_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 
        /* Create the common code event queue */
        rc = efx_ev_qcreate(sa->nic, sw_index, esmp, evq_info->entries,
-                           0 /* unused on EF10 */, 0,
-                           EFX_EVQ_FLAGS_TYPE_THROUGHPUT |
-                           EFX_EVQ_FLAGS_NOTIFY_DISABLED,
+                           0 /* unused on EF10 */, 0, evq_info->flags,
                            &evq->common);
        if (rc != 0)
                goto fail_ev_qcreate;
@@ -513,6 +639,25 @@ sfc_ev_qinit_info(struct sfc_adapter *sa, unsigned int sw_index)
        SFC_ASSERT(rte_is_power_of_2(max_entries));
 
        evq_info->max_entries = max_entries;
+       evq_info->flags = sa->evq_flags | EFX_EVQ_FLAGS_NOTIFY_DISABLED;
+
+       return 0;
+}
+
+static int
+sfc_kvarg_perf_profile_handler(__rte_unused const char *key,
+                              const char *value_str, void *opaque)
+{
+       uint64_t *value = opaque;
+
+       if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0)
+               *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
+       else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0)
+               *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY;
+       else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0)
+               *value = EFX_EVQ_FLAGS_TYPE_AUTO;
+       else
+               return -EINVAL;
 
        return 0;
 }
@@ -533,6 +678,16 @@ sfc_ev_init(struct sfc_adapter *sa)
 
        sfc_log_init(sa, "entry");
 
+       sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
+       rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE,
+                               sfc_kvarg_perf_profile_handler,
+                               &sa->evq_flags);
+       if (rc != 0) {
+               sfc_err(sa, "invalid %s parameter value",
+                       SFC_KVARG_PERF_PROFILE);
+               goto fail_kvarg_perf_profile;
+       }
+
        sa->evq_count = sfc_ev_qcount(sa);
        sa->mgmt_evq_index = 0;
        rte_spinlock_init(&sa->mgmt_evq_lock);
@@ -573,6 +728,8 @@ fail_ev_qinit_info:
 
 fail_evqs_alloc:
        sa->evq_count = 0;
+
+fail_kvarg_perf_profile:
        sfc_log_init(sa, "failed %d", rc);
        return rc;
 }