#include <rte_debug.h>
#include <rte_cycles.h>
#include <rte_alarm.h>
+#include <rte_branch_prediction.h>
#include "efx.h"
#include "sfc_log.h"
#include "sfc_ev.h"
#include "sfc_rx.h"
+#include "sfc_tx.h"
+#include "sfc_kvargs.h"
/* Initial delay when waiting for event queue init complete event */
}
static boolean_t
-sfc_ev_rx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id,
- __rte_unused uint32_t size, __rte_unused uint16_t flags)
+sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
+ uint32_t size, uint16_t flags)
{
struct sfc_evq *evq = arg;
+ struct sfc_rxq *rxq;
+ unsigned int stop;
+ unsigned int pending_id;
+ unsigned int delta;
+ unsigned int i;
+ struct sfc_rx_sw_desc *rxd;
- sfc_err(evq->sa, "EVQ %u unexpected Rx event", evq->evq_index);
- return B_TRUE;
+ if (unlikely(evq->exception))
+ goto done;
+
+ rxq = evq->rxq;
+
+ SFC_ASSERT(rxq != NULL);
+ SFC_ASSERT(rxq->evq == evq);
+ SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
+
+ stop = (id + 1) & rxq->ptr_mask;
+ pending_id = rxq->pending & rxq->ptr_mask;
+ delta = (stop >= pending_id) ? (stop - pending_id) :
+ (rxq->ptr_mask + 1 - pending_id + stop);
+
+ if (delta == 0) {
+ /*
+ * Rx event with no new descriptors done and zero length
+ * is used to abort scattered packet when there is no room
+ * for the tail.
+ */
+ if (unlikely(size != 0)) {
+ evq->exception = B_TRUE;
+ sfc_err(evq->sa,
+ "EVQ %u RxQ %u invalid RX abort "
+ "(id=%#x size=%u flags=%#x); needs restart\n",
+ evq->evq_index, sfc_rxq_sw_index(rxq),
+ id, size, flags);
+ goto done;
+ }
+
+ /* Add discard flag to the first fragment */
+ rxq->sw_desc[pending_id].flags |= EFX_DISCARD;
+ /* Remove continue flag from the last fragment */
+ rxq->sw_desc[id].flags &= ~EFX_PKT_CONT;
+ } else if (unlikely(delta > rxq->batch_max)) {
+ evq->exception = B_TRUE;
+
+ sfc_err(evq->sa,
+ "EVQ %u RxQ %u completion out of order "
+ "(id=%#x delta=%u flags=%#x); needs restart\n",
+ evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
+ flags);
+
+ goto done;
+ }
+
+ for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) {
+ rxd = &rxq->sw_desc[i];
+
+ rxd->flags = flags;
+
+ SFC_ASSERT(size < (1 << 16));
+ rxd->size = (uint16_t)size;
+ }
+
+ rxq->pending += delta;
+
+done:
+ return B_FALSE;
}
static boolean_t
-sfc_ev_tx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id)
+sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
{
struct sfc_evq *evq = arg;
+ struct sfc_txq *txq;
+ unsigned int stop;
+ unsigned int delta;
- sfc_err(evq->sa, "EVQ %u unexpected Tx event", evq->evq_index);
- return B_TRUE;
+ txq = evq->txq;
+
+ SFC_ASSERT(txq != NULL);
+ SFC_ASSERT(txq->evq == evq);
+
+ if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
+ goto done;
+
+ stop = (id + 1) & txq->ptr_mask;
+ id = txq->pending & txq->ptr_mask;
+
+ delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop);
+
+ txq->pending += delta;
+
+done:
+ return B_FALSE;
}
static boolean_t
sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_txq *txq;
- sfc_err(evq->sa, "EVQ %u unexpected Tx flush done event",
- evq->evq_index);
- return B_TRUE;
+ txq = evq->txq;
+ SFC_ASSERT(txq != NULL);
+ SFC_ASSERT(txq->hw_index == txq_hw_index);
+ SFC_ASSERT(txq->evq == evq);
+ sfc_tx_qflush_done(txq);
+
+ return B_FALSE;
}
static boolean_t
efx_ev_qpoll(evq->common, &evq->read_ptr, &sfc_ev_callbacks, evq);
+ if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) {
+ struct sfc_adapter *sa = evq->sa;
+ int rc;
+
+ if ((evq->rxq != NULL) && (evq->rxq->state & SFC_RXQ_RUNNING)) {
+ unsigned int rxq_sw_index = sfc_rxq_sw_index(evq->rxq);
+
+ sfc_warn(sa,
+ "restart RxQ %u because of exception on its EvQ %u",
+ rxq_sw_index, evq->evq_index);
+
+ sfc_rx_qstop(sa, rxq_sw_index);
+ rc = sfc_rx_qstart(sa, rxq_sw_index);
+ if (rc != 0)
+ sfc_err(sa, "cannot restart RxQ %u",
+ rxq_sw_index);
+ }
+
+ if (evq->txq != NULL) {
+ unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
+
+ sfc_warn(sa,
+ "restart TxQ %u because of exception on its EvQ %u",
+ txq_sw_index, evq->evq_index);
+
+ sfc_tx_qstop(sa, txq_sw_index);
+ rc = sfc_tx_qstart(sa, txq_sw_index);
+ if (rc != 0)
+ sfc_err(sa, "cannot restart TxQ %u",
+ txq_sw_index);
+ }
+
+ if (evq->exception)
+ sfc_panic(sa, "unrecoverable exception on EvQ %u",
+ evq->evq_index);
+
+ sfc_adapter_unlock(sa);
+ }
+
/* Poll-mode driver does not re-prime the event queue for interrupts */
}
/* Create the common code event queue */
rc = efx_ev_qcreate(sa->nic, sw_index, esmp, evq_info->entries,
- 0 /* unused on EF10 */, 0,
- EFX_EVQ_FLAGS_TYPE_THROUGHPUT |
- EFX_EVQ_FLAGS_NOTIFY_DISABLED,
+ 0 /* unused on EF10 */, 0, evq_info->flags,
&evq->common);
if (rc != 0)
goto fail_ev_qcreate;
SFC_ASSERT(rte_is_power_of_2(max_entries));
evq_info->max_entries = max_entries;
+ evq_info->flags = sa->evq_flags | EFX_EVQ_FLAGS_NOTIFY_DISABLED;
+
+ return 0;
+}
+
+static int
+sfc_kvarg_perf_profile_handler(__rte_unused const char *key,
+ const char *value_str, void *opaque)
+{
+ uint64_t *value = opaque;
+
+ if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0)
+ *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
+ else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0)
+ *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY;
+ else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0)
+ *value = EFX_EVQ_FLAGS_TYPE_AUTO;
+ else
+ return -EINVAL;
return 0;
}
sfc_log_init(sa, "entry");
+ sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
+ rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE,
+ sfc_kvarg_perf_profile_handler,
+ &sa->evq_flags);
+ if (rc != 0) {
+ sfc_err(sa, "invalid %s parameter value",
+ SFC_KVARG_PERF_PROFILE);
+ goto fail_kvarg_perf_profile;
+ }
+
sa->evq_count = sfc_ev_qcount(sa);
sa->mgmt_evq_index = 0;
rte_spinlock_init(&sa->mgmt_evq_lock);
fail_evqs_alloc:
sa->evq_count = 0;
+
+fail_kvarg_perf_profile:
sfc_log_init(sa, "failed %d", rc);
return rc;
}
git.droids-corp.org / dpdk.git / blobdiff