#include <rte_debug.h>
#include <rte_cycles.h>
+#include <rte_alarm.h>
+#include <rte_branch_prediction.h>
#include "efx.h"
#include "sfc_debug.h"
#include "sfc_log.h"
#include "sfc_ev.h"
+#include "sfc_rx.h"
+#include "sfc_tx.h"
+#include "sfc_kvargs.h"
/* Initial delay when waiting for event queue init complete event */
/* Event queue init approx timeout */
#define SFC_EVQ_INIT_TIMEOUT_US (2 * US_PER_S)
+/* Management event queue polling period in microseconds */
+#define SFC_MGMT_EV_QPOLL_PERIOD_US (US_PER_S)
+
static boolean_t
sfc_ev_initialized(void *arg)
}
static boolean_t
-sfc_ev_rx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id,
- __rte_unused uint32_t size, __rte_unused uint16_t flags)
+sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
+ uint32_t size, uint16_t flags)
{
struct sfc_evq *evq = arg;
+ struct sfc_rxq *rxq;
+ unsigned int stop;
+ unsigned int pending_id;
+ unsigned int delta;
+ unsigned int i;
+ struct sfc_rx_sw_desc *rxd;
+
+ if (unlikely(evq->exception))
+ goto done;
+
+ rxq = evq->rxq;
+
+ SFC_ASSERT(rxq != NULL);
+ SFC_ASSERT(rxq->evq == evq);
+ SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
+
+ stop = (id + 1) & rxq->ptr_mask;
+ pending_id = rxq->pending & rxq->ptr_mask;
+ delta = (stop >= pending_id) ? (stop - pending_id) :
+ (rxq->ptr_mask + 1 - pending_id + stop);
+
+ if (delta == 0) {
+ /*
+ * Rx event with no new descriptors done and zero length
+ * is used to abort scattered packet when there is no room
+ * for the tail.
+ */
+ if (unlikely(size != 0)) {
+ evq->exception = B_TRUE;
+ sfc_err(evq->sa,
+ "EVQ %u RxQ %u invalid RX abort "
+ "(id=%#x size=%u flags=%#x); needs restart\n",
+ evq->evq_index, sfc_rxq_sw_index(rxq),
+ id, size, flags);
+ goto done;
+ }
+
+ /* Add discard flag to the first fragment */
+ rxq->sw_desc[pending_id].flags |= EFX_DISCARD;
+ /* Remove continue flag from the last fragment */
+ rxq->sw_desc[id].flags &= ~EFX_PKT_CONT;
+ } else if (unlikely(delta > rxq->batch_max)) {
+ evq->exception = B_TRUE;
+
+ sfc_err(evq->sa,
+ "EVQ %u RxQ %u completion out of order "
+ "(id=%#x delta=%u flags=%#x); needs restart\n",
+ evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
+ flags);
+
+ goto done;
+ }
- sfc_err(evq->sa, "EVQ %u unexpected Rx event", evq->evq_index);
- return B_TRUE;
+ for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) {
+ rxd = &rxq->sw_desc[i];
+
+ rxd->flags = flags;
+
+ SFC_ASSERT(size < (1 << 16));
+ rxd->size = (uint16_t)size;
+ }
+
+ rxq->pending += delta;
+
+done:
+ return B_FALSE;
}
static boolean_t
-sfc_ev_tx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id)
+sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
{
struct sfc_evq *evq = arg;
+ struct sfc_txq *txq;
+ unsigned int stop;
+ unsigned int delta;
- sfc_err(evq->sa, "EVQ %u unexpected Tx event", evq->evq_index);
- return B_TRUE;
+ txq = evq->txq;
+
+ SFC_ASSERT(txq != NULL);
+ SFC_ASSERT(txq->evq == evq);
+
+ if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
+ goto done;
+
+ stop = (id + 1) & txq->ptr_mask;
+ id = txq->pending & txq->ptr_mask;
+
+ delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop);
+
+ txq->pending += delta;
+
+done:
+ return B_FALSE;
}
static boolean_t
sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_rxq *rxq;
- sfc_err(evq->sa, "EVQ %u unexpected Rx flush done event",
- evq->evq_index);
- return B_TRUE;
+ rxq = evq->rxq;
+ SFC_ASSERT(rxq != NULL);
+ SFC_ASSERT(rxq->hw_index == rxq_hw_index);
+ SFC_ASSERT(rxq->evq == evq);
+ sfc_rx_qflush_done(rxq);
+
+ return B_FALSE;
}
static boolean_t
sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_rxq *rxq;
- sfc_err(evq->sa, "EVQ %u unexpected Rx flush failed event",
- evq->evq_index);
- return B_TRUE;
+ rxq = evq->rxq;
+ SFC_ASSERT(rxq != NULL);
+ SFC_ASSERT(rxq->hw_index == rxq_hw_index);
+ SFC_ASSERT(rxq->evq == evq);
+ sfc_rx_qflush_failed(rxq);
+
+ return B_FALSE;
}
static boolean_t
sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
{
struct sfc_evq *evq = arg;
+ struct sfc_txq *txq;
- sfc_err(evq->sa, "EVQ %u unexpected Tx flush done event",
- evq->evq_index);
- return B_TRUE;
+ txq = evq->txq;
+ SFC_ASSERT(txq != NULL);
+ SFC_ASSERT(txq->hw_index == txq_hw_index);
+ SFC_ASSERT(txq->evq == evq);
+ sfc_tx_qflush_done(txq);
+
+ return B_FALSE;
}
static boolean_t
}
static boolean_t
-sfc_ev_link_change(void *arg, __rte_unused efx_link_mode_t link_mode)
+sfc_ev_link_change(void *arg, efx_link_mode_t link_mode)
{
struct sfc_evq *evq = arg;
+ struct sfc_adapter *sa = evq->sa;
+ struct rte_eth_link *dev_link = &sa->eth_dev->data->dev_link;
+ struct rte_eth_link new_link;
- sfc_err(evq->sa, "EVQ %u unexpected link change",
- evq->evq_index);
- return B_TRUE;
+ EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t));
+
+ sfc_port_link_mode_to_info(link_mode, &new_link);
+ rte_atomic64_set((rte_atomic64_t *)dev_link, *(uint64_t *)&new_link);
+
+ return B_FALSE;
}
static const efx_ev_callbacks_t sfc_ev_callbacks = {
efx_ev_qpoll(evq->common, &evq->read_ptr, &sfc_ev_callbacks, evq);
+ if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) {
+ struct sfc_adapter *sa = evq->sa;
+ int rc;
+
+ if ((evq->rxq != NULL) && (evq->rxq->state & SFC_RXQ_RUNNING)) {
+ unsigned int rxq_sw_index = sfc_rxq_sw_index(evq->rxq);
+
+ sfc_warn(sa,
+ "restart RxQ %u because of exception on its EvQ %u",
+ rxq_sw_index, evq->evq_index);
+
+ sfc_rx_qstop(sa, rxq_sw_index);
+ rc = sfc_rx_qstart(sa, rxq_sw_index);
+ if (rc != 0)
+ sfc_err(sa, "cannot restart RxQ %u",
+ rxq_sw_index);
+ }
+
+ if (evq->txq != NULL) {
+ unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
+
+ sfc_warn(sa,
+ "restart TxQ %u because of exception on its EvQ %u",
+ txq_sw_index, evq->evq_index);
+
+ sfc_tx_qstop(sa, txq_sw_index);
+ rc = sfc_tx_qstart(sa, txq_sw_index);
+ if (rc != 0)
+ sfc_err(sa, "cannot restart TxQ %u",
+ txq_sw_index);
+ }
+
+ if (evq->exception)
+ sfc_panic(sa, "unrecoverable exception on EvQ %u",
+ evq->evq_index);
+
+ sfc_adapter_unlock(sa);
+ }
+
/* Poll-mode driver does not re-prime the event queue for interrupts */
}
+void
+sfc_ev_mgmt_qpoll(struct sfc_adapter *sa)
+{
+ if (rte_spinlock_trylock(&sa->mgmt_evq_lock)) {
+ struct sfc_evq *mgmt_evq = sa->evq_info[sa->mgmt_evq_index].evq;
+
+ if (mgmt_evq->init_state == SFC_EVQ_STARTED)
+ sfc_ev_qpoll(mgmt_evq);
+
+ rte_spinlock_unlock(&sa->mgmt_evq_lock);
+ }
+}
+
int
sfc_ev_qprime(struct sfc_evq *evq)
{
/* Create the common code event queue */
rc = efx_ev_qcreate(sa->nic, sw_index, esmp, evq_info->entries,
- 0 /* unused on EF10 */, 0,
- EFX_EVQ_FLAGS_TYPE_THROUGHPUT |
- EFX_EVQ_FLAGS_NOTIFY_DISABLED,
+ 0 /* unused on EF10 */, 0, evq_info->flags,
&evq->common);
if (rc != 0)
goto fail_ev_qcreate;
efx_ev_qdestroy(evq->common);
}
+static void
+sfc_ev_mgmt_periodic_qpoll(void *arg)
+{
+ struct sfc_adapter *sa = arg;
+ int rc;
+
+ sfc_ev_mgmt_qpoll(sa);
+
+ rc = rte_eal_alarm_set(SFC_MGMT_EV_QPOLL_PERIOD_US,
+ sfc_ev_mgmt_periodic_qpoll, sa);
+ if (rc != 0)
+ sfc_panic(sa,
+ "cannot rearm management EVQ polling alarm (rc=%d)",
+ rc);
+}
+
+static void
+sfc_ev_mgmt_periodic_qpoll_start(struct sfc_adapter *sa)
+{
+ sfc_ev_mgmt_periodic_qpoll(sa);
+}
+
+static void
+sfc_ev_mgmt_periodic_qpoll_stop(struct sfc_adapter *sa)
+{
+ rte_eal_alarm_cancel(sfc_ev_mgmt_periodic_qpoll, sa);
+}
+
int
sfc_ev_start(struct sfc_adapter *sa)
{
if (rc != 0)
goto fail_ev_init;
+ /* Start management EVQ used for global events */
+ rte_spinlock_lock(&sa->mgmt_evq_lock);
+
+ rc = sfc_ev_qstart(sa, sa->mgmt_evq_index);
+ if (rc != 0)
+ goto fail_mgmt_evq_start;
+
+ rte_spinlock_unlock(&sa->mgmt_evq_lock);
+
+ /*
+ * Start management EVQ polling. If interrupts are disabled
+ * (not used), it is required to process link status change
+ * and other device level events to avoid unrecoverable
+ * error because the event queue overflow.
+ */
+ sfc_ev_mgmt_periodic_qpoll_start(sa);
+
/*
- * Rx/Tx event queues are started/stopped when corresponding queue
- * is started/stopped.
+ * Rx/Tx event queues are started/stopped when corresponding
+ * Rx/Tx queue is started/stopped.
*/
return 0;
+fail_mgmt_evq_start:
+ rte_spinlock_unlock(&sa->mgmt_evq_lock);
+ efx_ev_fini(sa->nic);
+
fail_ev_init:
sfc_log_init(sa, "failed %d", rc);
return rc;
sfc_log_init(sa, "entry");
+ sfc_ev_mgmt_periodic_qpoll_stop(sa);
+
/* Make sure that all event queues are stopped */
sw_index = sa->evq_count;
- while (sw_index-- > 0)
- sfc_ev_qstop(sa, sw_index);
+ while (sw_index-- > 0) {
+ if (sw_index == sa->mgmt_evq_index) {
+ /* Locks are required for the management EVQ */
+ rte_spinlock_lock(&sa->mgmt_evq_lock);
+ sfc_ev_qstop(sa, sa->mgmt_evq_index);
+ rte_spinlock_unlock(&sa->mgmt_evq_lock);
+ } else {
+ sfc_ev_qstop(sa, sw_index);
+ }
+ }
efx_ev_fini(sa->nic);
}
SFC_ASSERT(rte_is_power_of_2(max_entries));
evq_info->max_entries = max_entries;
+ evq_info->flags = sa->evq_flags | EFX_EVQ_FLAGS_NOTIFY_DISABLED;
+
+ return 0;
+}
+
+static int
+sfc_kvarg_perf_profile_handler(__rte_unused const char *key,
+ const char *value_str, void *opaque)
+{
+ uint64_t *value = opaque;
+
+ if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0)
+ *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
+ else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0)
+ *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY;
+ else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0)
+ *value = EFX_EVQ_FLAGS_TYPE_AUTO;
+ else
+ return -EINVAL;
return 0;
}
sfc_log_init(sa, "entry");
+ sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
+ rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE,
+ sfc_kvarg_perf_profile_handler,
+ &sa->evq_flags);
+ if (rc != 0) {
+ sfc_err(sa, "invalid %s parameter value",
+ SFC_KVARG_PERF_PROFILE);
+ goto fail_kvarg_perf_profile;
+ }
+
sa->evq_count = sfc_ev_qcount(sa);
sa->mgmt_evq_index = 0;
+ rte_spinlock_init(&sa->mgmt_evq_lock);
/* Allocate EVQ info array */
rc = ENOMEM;
goto fail_ev_qinit_info;
}
+ rc = sfc_ev_qinit(sa, sa->mgmt_evq_index, SFC_MGMT_EVQ_ENTRIES,
+ sa->socket_id);
+ if (rc != 0)
+ goto fail_mgmt_evq_init;
+
/*
* Rx/Tx event queues are created/destroyed when corresponding
* Rx/Tx queue is created/destroyed.
return 0;
+fail_mgmt_evq_init:
fail_ev_qinit_info:
while (sw_index-- > 0)
sfc_ev_qfini_info(sa, sw_index);
fail_evqs_alloc:
sa->evq_count = 0;
+
+fail_kvarg_perf_profile:
sfc_log_init(sa, "failed %d", rc);
return rc;
}
git.droids-corp.org / dpdk.git / blobdiff