4 * Copyright (c) 2016-2017 Solarflare Communications Inc.
7 * This software was jointly developed between OKTET Labs (under contract
8 * for Solarflare) and Solarflare Communications, Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 #include <rte_debug.h>
33 #include <rte_cycles.h>
34 #include <rte_alarm.h>
35 #include <rte_branch_prediction.h>
40 #include "sfc_debug.h"
45 #include "sfc_kvargs.h"
48 /* Initial delay when waiting for event queue init complete event */
49 #define SFC_EVQ_INIT_BACKOFF_START_US (1)
50 /* Maximum delay between event queue polling attempts */
51 #define SFC_EVQ_INIT_BACKOFF_MAX_US (10 * 1000)
52 /* Event queue init approx timeout */
53 #define SFC_EVQ_INIT_TIMEOUT_US (2 * US_PER_S)
55 /* Management event queue polling period in microseconds */
56 #define SFC_MGMT_EV_QPOLL_PERIOD_US (US_PER_S)
60 sfc_ev_initialized(void *arg)
62 struct sfc_evq *evq = arg;
64 /* Init done events may be duplicated on SFN7xxx (SFC bug 31631) */
65 SFC_ASSERT(evq->init_state == SFC_EVQ_STARTING ||
66 evq->init_state == SFC_EVQ_STARTED);
68 evq->init_state = SFC_EVQ_STARTED;
74 sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
75 uint32_t size, uint16_t flags)
77 struct sfc_evq *evq = arg;
80 unsigned int pending_id;
83 struct sfc_rx_sw_desc *rxd;
85 if (unlikely(evq->exception))
90 SFC_ASSERT(rxq != NULL);
91 SFC_ASSERT(rxq->evq == evq);
92 SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
94 stop = (id + 1) & rxq->ptr_mask;
95 pending_id = rxq->pending & rxq->ptr_mask;
96 delta = (stop >= pending_id) ? (stop - pending_id) :
97 (rxq->ptr_mask + 1 - pending_id + stop);
101 * Rx event with no new descriptors done and zero length
102 * is used to abort scattered packet when there is no room
105 if (unlikely(size != 0)) {
106 evq->exception = B_TRUE;
108 "EVQ %u RxQ %u invalid RX abort "
109 "(id=%#x size=%u flags=%#x); needs restart",
110 evq->evq_index, sfc_rxq_sw_index(rxq),
115 /* Add discard flag to the first fragment */
116 rxq->sw_desc[pending_id].flags |= EFX_DISCARD;
117 /* Remove continue flag from the last fragment */
118 rxq->sw_desc[id].flags &= ~EFX_PKT_CONT;
119 } else if (unlikely(delta > rxq->batch_max)) {
120 evq->exception = B_TRUE;
123 "EVQ %u RxQ %u completion out of order "
124 "(id=%#x delta=%u flags=%#x); needs restart",
125 evq->evq_index, sfc_rxq_sw_index(rxq), id, delta,
131 for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) {
132 rxd = &rxq->sw_desc[i];
136 SFC_ASSERT(size < (1 << 16));
137 rxd->size = (uint16_t)size;
140 rxq->pending += delta;
147 sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id)
149 struct sfc_evq *evq = arg;
156 SFC_ASSERT(txq != NULL);
157 SFC_ASSERT(txq->evq == evq);
159 if (unlikely((txq->state & SFC_TXQ_STARTED) == 0))
162 stop = (id + 1) & txq->ptr_mask;
163 id = txq->pending & txq->ptr_mask;
165 delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop);
167 txq->pending += delta;
174 sfc_ev_exception(void *arg, __rte_unused uint32_t code,
175 __rte_unused uint32_t data)
177 struct sfc_evq *evq = arg;
179 if (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT)
182 evq->exception = B_TRUE;
184 "hardware exception %s (code=%u, data=%#x) on EVQ %u;"
186 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
187 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
188 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
189 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
190 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
191 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
192 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
193 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
195 code, data, evq->evq_index);
201 sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index)
203 struct sfc_evq *evq = arg;
207 SFC_ASSERT(rxq != NULL);
208 SFC_ASSERT(rxq->hw_index == rxq_hw_index);
209 SFC_ASSERT(rxq->evq == evq);
210 sfc_rx_qflush_done(rxq);
216 sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index)
218 struct sfc_evq *evq = arg;
222 SFC_ASSERT(rxq != NULL);
223 SFC_ASSERT(rxq->hw_index == rxq_hw_index);
224 SFC_ASSERT(rxq->evq == evq);
225 sfc_rx_qflush_failed(rxq);
231 sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index)
233 struct sfc_evq *evq = arg;
237 SFC_ASSERT(txq != NULL);
238 SFC_ASSERT(txq->hw_index == txq_hw_index);
239 SFC_ASSERT(txq->evq == evq);
240 sfc_tx_qflush_done(txq);
246 sfc_ev_software(void *arg, uint16_t magic)
248 struct sfc_evq *evq = arg;
250 sfc_err(evq->sa, "EVQ %u unexpected software event magic=%#.4x",
251 evq->evq_index, magic);
256 sfc_ev_sram(void *arg, uint32_t code)
258 struct sfc_evq *evq = arg;
260 sfc_err(evq->sa, "EVQ %u unexpected SRAM event code=%u",
261 evq->evq_index, code);
266 sfc_ev_wake_up(void *arg, uint32_t index)
268 struct sfc_evq *evq = arg;
270 sfc_err(evq->sa, "EVQ %u unexpected wake up event index=%u",
271 evq->evq_index, index);
276 sfc_ev_timer(void *arg, uint32_t index)
278 struct sfc_evq *evq = arg;
280 sfc_err(evq->sa, "EVQ %u unexpected timer event index=%u",
281 evq->evq_index, index);
286 sfc_ev_link_change(void *arg, efx_link_mode_t link_mode)
288 struct sfc_evq *evq = arg;
289 struct sfc_adapter *sa = evq->sa;
290 struct rte_eth_link *dev_link = &sa->eth_dev->data->dev_link;
291 struct rte_eth_link new_link;
292 uint64_t new_link_u64;
293 uint64_t old_link_u64;
295 EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t));
297 sfc_port_link_mode_to_info(link_mode, &new_link);
299 new_link_u64 = *(uint64_t *)&new_link;
301 old_link_u64 = rte_atomic64_read((rte_atomic64_t *)dev_link);
302 if (old_link_u64 == new_link_u64)
305 if (rte_atomic64_cmpset((volatile uint64_t *)dev_link,
306 old_link_u64, new_link_u64)) {
307 evq->sa->port.lsc_seq++;
315 static const efx_ev_callbacks_t sfc_ev_callbacks = {
316 .eec_initialized = sfc_ev_initialized,
319 .eec_exception = sfc_ev_exception,
320 .eec_rxq_flush_done = sfc_ev_rxq_flush_done,
321 .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed,
322 .eec_txq_flush_done = sfc_ev_txq_flush_done,
323 .eec_software = sfc_ev_software,
324 .eec_sram = sfc_ev_sram,
325 .eec_wake_up = sfc_ev_wake_up,
326 .eec_timer = sfc_ev_timer,
327 .eec_link_change = sfc_ev_link_change,
332 sfc_ev_qpoll(struct sfc_evq *evq)
334 SFC_ASSERT(evq->init_state == SFC_EVQ_STARTED ||
335 evq->init_state == SFC_EVQ_STARTING);
337 /* Synchronize the DMA memory for reading not required */
339 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfc_ev_callbacks, evq);
341 if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) {
342 struct sfc_adapter *sa = evq->sa;
345 if ((evq->rxq != NULL) && (evq->rxq->state & SFC_RXQ_RUNNING)) {
346 unsigned int rxq_sw_index = sfc_rxq_sw_index(evq->rxq);
349 "restart RxQ %u because of exception on its EvQ %u",
350 rxq_sw_index, evq->evq_index);
352 sfc_rx_qstop(sa, rxq_sw_index);
353 rc = sfc_rx_qstart(sa, rxq_sw_index);
355 sfc_err(sa, "cannot restart RxQ %u",
359 if (evq->txq != NULL) {
360 unsigned int txq_sw_index = sfc_txq_sw_index(evq->txq);
363 "restart TxQ %u because of exception on its EvQ %u",
364 txq_sw_index, evq->evq_index);
366 sfc_tx_qstop(sa, txq_sw_index);
367 rc = sfc_tx_qstart(sa, txq_sw_index);
369 sfc_err(sa, "cannot restart TxQ %u",
374 sfc_panic(sa, "unrecoverable exception on EvQ %u",
377 sfc_adapter_unlock(sa);
380 /* Poll-mode driver does not re-prime the event queue for interrupts */
384 sfc_ev_mgmt_qpoll(struct sfc_adapter *sa)
386 if (rte_spinlock_trylock(&sa->mgmt_evq_lock)) {
387 struct sfc_evq *mgmt_evq = sa->evq_info[sa->mgmt_evq_index].evq;
389 if (mgmt_evq->init_state == SFC_EVQ_STARTED)
390 sfc_ev_qpoll(mgmt_evq);
392 rte_spinlock_unlock(&sa->mgmt_evq_lock);
397 sfc_ev_qprime(struct sfc_evq *evq)
399 SFC_ASSERT(evq->init_state == SFC_EVQ_STARTED);
400 return efx_ev_qprime(evq->common, evq->read_ptr);
404 sfc_ev_qstart(struct sfc_adapter *sa, unsigned int sw_index)
406 const struct sfc_evq_info *evq_info;
409 unsigned int total_delay_us;
410 unsigned int delay_us;
413 sfc_log_init(sa, "sw_index=%u", sw_index);
415 evq_info = &sa->evq_info[sw_index];
419 /* Clear all events */
420 (void)memset((void *)esmp->esm_base, 0xff,
421 EFX_EVQ_SIZE(evq_info->entries));
423 /* Create the common code event queue */
424 rc = efx_ev_qcreate(sa->nic, sw_index, esmp, evq_info->entries,
425 0 /* unused on EF10 */, 0, evq_info->flags,
428 goto fail_ev_qcreate;
430 evq->init_state = SFC_EVQ_STARTING;
432 /* Wait for the initialization event */
434 delay_us = SFC_EVQ_INIT_BACKOFF_START_US;
436 (void)sfc_ev_qpoll(evq);
438 /* Check to see if the initialization complete indication
439 * posted by the hardware.
441 if (evq->init_state == SFC_EVQ_STARTED)
444 /* Give event queue some time to init */
445 rte_delay_us(delay_us);
447 total_delay_us += delay_us;
449 /* Exponential backoff */
451 if (delay_us > SFC_EVQ_INIT_BACKOFF_MAX_US)
452 delay_us = SFC_EVQ_INIT_BACKOFF_MAX_US;
454 } while (total_delay_us < SFC_EVQ_INIT_TIMEOUT_US);
463 evq->init_state = SFC_EVQ_INITIALIZED;
464 efx_ev_qdestroy(evq->common);
467 sfc_log_init(sa, "failed %d", rc);
472 sfc_ev_qstop(struct sfc_adapter *sa, unsigned int sw_index)
474 const struct sfc_evq_info *evq_info;
477 sfc_log_init(sa, "sw_index=%u", sw_index);
479 SFC_ASSERT(sw_index < sa->evq_count);
481 evq_info = &sa->evq_info[sw_index];
484 if (evq == NULL || evq->init_state != SFC_EVQ_STARTED)
487 evq->init_state = SFC_EVQ_INITIALIZED;
489 evq->exception = B_FALSE;
491 efx_ev_qdestroy(evq->common);
495 sfc_ev_mgmt_periodic_qpoll(void *arg)
497 struct sfc_adapter *sa = arg;
500 sfc_ev_mgmt_qpoll(sa);
502 rc = rte_eal_alarm_set(SFC_MGMT_EV_QPOLL_PERIOD_US,
503 sfc_ev_mgmt_periodic_qpoll, sa);
504 if (rc == -ENOTSUP) {
505 sfc_warn(sa, "alarms are not supported");
506 sfc_warn(sa, "management EVQ must be polled indirectly using no-wait link status update");
507 } else if (rc != 0) {
509 "cannot rearm management EVQ polling alarm (rc=%d)",
515 sfc_ev_mgmt_periodic_qpoll_start(struct sfc_adapter *sa)
517 sfc_ev_mgmt_periodic_qpoll(sa);
521 sfc_ev_mgmt_periodic_qpoll_stop(struct sfc_adapter *sa)
523 rte_eal_alarm_cancel(sfc_ev_mgmt_periodic_qpoll, sa);
527 sfc_ev_start(struct sfc_adapter *sa)
531 sfc_log_init(sa, "entry");
533 rc = efx_ev_init(sa->nic);
537 /* Start management EVQ used for global events */
538 rte_spinlock_lock(&sa->mgmt_evq_lock);
540 rc = sfc_ev_qstart(sa, sa->mgmt_evq_index);
542 goto fail_mgmt_evq_start;
544 if (sa->intr.lsc_intr) {
545 rc = sfc_ev_qprime(sa->evq_info[sa->mgmt_evq_index].evq);
547 goto fail_evq0_prime;
550 rte_spinlock_unlock(&sa->mgmt_evq_lock);
553 * Start management EVQ polling. If interrupts are disabled
554 * (not used), it is required to process link status change
555 * and other device level events to avoid unrecoverable
556 * error because the event queue overflow.
558 sfc_ev_mgmt_periodic_qpoll_start(sa);
561 * Rx/Tx event queues are started/stopped when corresponding
562 * Rx/Tx queue is started/stopped.
571 rte_spinlock_unlock(&sa->mgmt_evq_lock);
572 efx_ev_fini(sa->nic);
575 sfc_log_init(sa, "failed %d", rc);
580 sfc_ev_stop(struct sfc_adapter *sa)
582 unsigned int sw_index;
584 sfc_log_init(sa, "entry");
586 sfc_ev_mgmt_periodic_qpoll_stop(sa);
588 /* Make sure that all event queues are stopped */
589 sw_index = sa->evq_count;
590 while (sw_index-- > 0) {
591 if (sw_index == sa->mgmt_evq_index) {
592 /* Locks are required for the management EVQ */
593 rte_spinlock_lock(&sa->mgmt_evq_lock);
594 sfc_ev_qstop(sa, sa->mgmt_evq_index);
595 rte_spinlock_unlock(&sa->mgmt_evq_lock);
597 sfc_ev_qstop(sa, sw_index);
601 efx_ev_fini(sa->nic);
605 sfc_ev_qinit(struct sfc_adapter *sa, unsigned int sw_index,
606 unsigned int entries, int socket_id)
608 struct sfc_evq_info *evq_info;
612 sfc_log_init(sa, "sw_index=%u", sw_index);
614 evq_info = &sa->evq_info[sw_index];
616 SFC_ASSERT(rte_is_power_of_2(entries));
617 SFC_ASSERT(entries <= evq_info->max_entries);
618 evq_info->entries = entries;
620 evq = rte_zmalloc_socket("sfc-evq", sizeof(*evq), RTE_CACHE_LINE_SIZE,
626 evq->evq_index = sw_index;
628 /* Allocate DMA space */
629 rc = sfc_dma_alloc(sa, "evq", sw_index, EFX_EVQ_SIZE(evq_info->entries),
630 socket_id, &evq->mem);
634 evq->init_state = SFC_EVQ_INITIALIZED;
642 sfc_ev_qfini(struct sfc_adapter *sa, unsigned int sw_index)
646 sfc_log_init(sa, "sw_index=%u", sw_index);
648 evq = sa->evq_info[sw_index].evq;
650 SFC_ASSERT(evq->init_state == SFC_EVQ_INITIALIZED);
652 sa->evq_info[sw_index].evq = NULL;
654 sfc_dma_free(sa, &evq->mem);
660 sfc_ev_qinit_info(struct sfc_adapter *sa, unsigned int sw_index)
662 struct sfc_evq_info *evq_info = &sa->evq_info[sw_index];
663 unsigned int max_entries;
665 sfc_log_init(sa, "sw_index=%u", sw_index);
667 max_entries = sfc_evq_max_entries(sa, sw_index);
668 SFC_ASSERT(rte_is_power_of_2(max_entries));
670 evq_info->max_entries = max_entries;
671 evq_info->flags = sa->evq_flags |
672 ((sa->intr.lsc_intr && sw_index == sa->mgmt_evq_index) ?
673 EFX_EVQ_FLAGS_NOTIFY_INTERRUPT :
674 EFX_EVQ_FLAGS_NOTIFY_DISABLED);
680 sfc_kvarg_perf_profile_handler(__rte_unused const char *key,
681 const char *value_str, void *opaque)
683 uint64_t *value = opaque;
685 if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0)
686 *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
687 else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0)
688 *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY;
689 else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0)
690 *value = EFX_EVQ_FLAGS_TYPE_AUTO;
698 sfc_ev_qfini_info(struct sfc_adapter *sa, unsigned int sw_index)
700 sfc_log_init(sa, "sw_index=%u", sw_index);
702 /* Nothing to cleanup */
706 sfc_ev_init(struct sfc_adapter *sa)
709 unsigned int sw_index;
711 sfc_log_init(sa, "entry");
713 sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT;
714 rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE,
715 sfc_kvarg_perf_profile_handler,
718 sfc_err(sa, "invalid %s parameter value",
719 SFC_KVARG_PERF_PROFILE);
720 goto fail_kvarg_perf_profile;
723 sa->evq_count = sfc_ev_qcount(sa);
724 sa->mgmt_evq_index = 0;
725 rte_spinlock_init(&sa->mgmt_evq_lock);
727 /* Allocate EVQ info array */
729 sa->evq_info = rte_calloc_socket("sfc-evqs", sa->evq_count,
730 sizeof(struct sfc_evq_info), 0,
732 if (sa->evq_info == NULL)
733 goto fail_evqs_alloc;
735 for (sw_index = 0; sw_index < sa->evq_count; ++sw_index) {
736 rc = sfc_ev_qinit_info(sa, sw_index);
738 goto fail_ev_qinit_info;
741 rc = sfc_ev_qinit(sa, sa->mgmt_evq_index, SFC_MGMT_EVQ_ENTRIES,
744 goto fail_mgmt_evq_init;
747 * Rx/Tx event queues are created/destroyed when corresponding
748 * Rx/Tx queue is created/destroyed.
755 while (sw_index-- > 0)
756 sfc_ev_qfini_info(sa, sw_index);
758 rte_free(sa->evq_info);
764 fail_kvarg_perf_profile:
765 sfc_log_init(sa, "failed %d", rc);
770 sfc_ev_fini(struct sfc_adapter *sa)
774 sfc_log_init(sa, "entry");
776 /* Cleanup all event queues */
777 sw_index = sa->evq_count;
778 while (--sw_index >= 0) {
779 if (sa->evq_info[sw_index].evq != NULL)
780 sfc_ev_qfini(sa, sw_index);
781 sfc_ev_qfini_info(sa, sw_index);
784 rte_free(sa->evq_info);