X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fsfc%2Fsfc_ev.c;h=8a5030b7a5a681c2a921ce6e11bc2fc6b67338b6;hb=f0fdd5158e12283e234827202a88ed80ca95002c;hp=075172addf9d2c91fce3dee3127ff9e349ad4351;hpb=fed9aeb46c19974a3fda2536840598196f002f8d;p=dpdk.git diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c index 075172addf..8a5030b7a5 100644 --- a/drivers/net/sfc/sfc_ev.c +++ b/drivers/net/sfc/sfc_ev.c @@ -1,35 +1,16 @@ -/*- - * Copyright (c) 2016 Solarflare Communications Inc. +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright (c) 2016-2018 Solarflare Communications Inc. * All rights reserved. * * This software was jointly developed between OKTET Labs (under contract * for Solarflare) and Solarflare Communications, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include +#include #include "efx.h" @@ -39,6 +20,7 @@ #include "sfc_ev.h" #include "sfc_rx.h" #include "sfc_tx.h" +#include "sfc_kvargs.h" /* Initial delay when waiting for event queue init complete event */ @@ -51,6 +33,21 @@ /* Management event queue polling period in microseconds */ #define SFC_MGMT_EV_QPOLL_PERIOD_US (US_PER_S) +static const char * +sfc_evq_type2str(enum sfc_evq_type type) +{ + switch (type) { + case SFC_EVQ_TYPE_MGMT: + return "mgmt-evq"; + case SFC_EVQ_TYPE_RX: + return "rx-evq"; + case SFC_EVQ_TYPE_TX: + return "tx-evq"; + default: + SFC_ASSERT(B_FALSE); + return NULL; + } +} static boolean_t sfc_ev_initialized(void *arg) @@ -67,25 +64,37 @@ sfc_ev_initialized(void *arg) } static boolean_t -sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id, - uint32_t size, uint16_t flags) +sfc_ev_nop_rx(void *arg, uint32_t label, uint32_t id, + uint32_t size, uint16_t flags) { struct sfc_evq *evq = arg; - struct sfc_rxq *rxq; + + sfc_err(evq->sa, + "EVQ %u unexpected Rx event label=%u id=%#x size=%u flags=%#x", + evq->evq_index, label, id, size, flags); + return B_TRUE; +} + +static boolean_t +sfc_ev_efx_rx(void *arg, __rte_unused uint32_t label, uint32_t id, + uint32_t size, uint16_t flags) +{ + struct sfc_evq *evq = arg; + struct sfc_efx_rxq *rxq; unsigned int stop; unsigned int pending_id; unsigned int delta; unsigned int i; - struct sfc_rx_sw_desc *rxd; + struct sfc_efx_rx_sw_desc *rxd; if (unlikely(evq->exception)) goto done; - rxq = evq->rxq; + rxq = sfc_efx_rxq_by_dp_rxq(evq->dp_rxq); SFC_ASSERT(rxq != NULL); SFC_ASSERT(rxq->evq == evq); - SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); + SFC_ASSERT(rxq->flags & SFC_EFX_RXQ_FLAG_STARTED); stop = (id + 1) & rxq->ptr_mask; pending_id = rxq->pending & rxq->ptr_mask; @@ -102,8 +111,8 @@ sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id, evq->exception = B_TRUE; sfc_err(evq->sa, "EVQ %u RxQ %u invalid RX abort " - "(id=%#x size=%u flags=%#x); needs restart\n", - evq->evq_index, sfc_rxq_sw_index(rxq), + "(id=%#x size=%u flags=%#x); needs restart", + evq->evq_index, rxq->dp.dpq.queue_id, id, size, flags); goto done; } @@ -117,9 +126,9 @@ sfc_ev_rx(void *arg, __rte_unused uint32_t label, uint32_t id, sfc_err(evq->sa, "EVQ %u RxQ %u completion out of order " - "(id=%#x delta=%u flags=%#x); needs restart\n", - evq->evq_index, sfc_rxq_sw_index(rxq), id, delta, - flags); + "(id=%#x delta=%u flags=%#x); needs restart", + evq->evq_index, rxq->dp.dpq.queue_id, + id, delta, flags); goto done; } @@ -140,17 +149,73 @@ done: } static boolean_t -sfc_ev_tx(void *arg, __rte_unused uint32_t label, __rte_unused uint32_t id) +sfc_ev_dp_rx(void *arg, __rte_unused uint32_t label, uint32_t id, + __rte_unused uint32_t size, __rte_unused uint16_t flags) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_rxq *dp_rxq; + + dp_rxq = evq->dp_rxq; + SFC_ASSERT(dp_rxq != NULL); + + SFC_ASSERT(evq->sa->dp_rx->qrx_ev != NULL); + return evq->sa->dp_rx->qrx_ev(dp_rxq, id); +} + +static boolean_t +sfc_ev_nop_tx(void *arg, uint32_t label, uint32_t id) { struct sfc_evq *evq = arg; - sfc_err(evq->sa, "EVQ %u unexpected Tx event", evq->evq_index); + sfc_err(evq->sa, "EVQ %u unexpected Tx event label=%u id=%#x", + evq->evq_index, label, id); return B_TRUE; } static boolean_t -sfc_ev_exception(void *arg, __rte_unused uint32_t code, - __rte_unused uint32_t data) +sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; + struct sfc_efx_txq *txq; + unsigned int stop; + unsigned int delta; + + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + txq = sfc_efx_txq_by_dp_txq(dp_txq); + SFC_ASSERT(txq->evq == evq); + + if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0)) + goto done; + + stop = (id + 1) & txq->ptr_mask; + id = txq->pending & txq->ptr_mask; + + delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop); + + txq->pending += delta; + +done: + return B_FALSE; +} + +static boolean_t +sfc_ev_dp_tx(void *arg, __rte_unused uint32_t label, uint32_t id) +{ + struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; + + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + SFC_ASSERT(evq->sa->dp_tx->qtx_ev != NULL); + return evq->sa->dp_tx->qtx_ev(dp_txq, id); +} + +static boolean_t +sfc_ev_exception(void *arg, uint32_t code, __rte_unused uint32_t data) { struct sfc_evq *evq = arg; @@ -175,13 +240,27 @@ sfc_ev_exception(void *arg, __rte_unused uint32_t code, return B_TRUE; } +static boolean_t +sfc_ev_nop_rxq_flush_done(void *arg, uint32_t rxq_hw_index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected RxQ %u flush done", + evq->evq_index, rxq_hw_index); + return B_TRUE; +} + static boolean_t sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index) { struct sfc_evq *evq = arg; + struct sfc_dp_rxq *dp_rxq; struct sfc_rxq *rxq; - rxq = evq->rxq; + dp_rxq = evq->dp_rxq; + SFC_ASSERT(dp_rxq != NULL); + + rxq = sfc_rxq_by_dp_rxq(dp_rxq); SFC_ASSERT(rxq != NULL); SFC_ASSERT(rxq->hw_index == rxq_hw_index); SFC_ASSERT(rxq->evq == evq); @@ -190,13 +269,27 @@ sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index) return B_FALSE; } +static boolean_t +sfc_ev_nop_rxq_flush_failed(void *arg, uint32_t rxq_hw_index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected RxQ %u flush failed", + evq->evq_index, rxq_hw_index); + return B_TRUE; +} + static boolean_t sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index) { struct sfc_evq *evq = arg; + struct sfc_dp_rxq *dp_rxq; struct sfc_rxq *rxq; - rxq = evq->rxq; + dp_rxq = evq->dp_rxq; + SFC_ASSERT(dp_rxq != NULL); + + rxq = sfc_rxq_by_dp_rxq(dp_rxq); SFC_ASSERT(rxq != NULL); SFC_ASSERT(rxq->hw_index == rxq_hw_index); SFC_ASSERT(rxq->evq == evq); @@ -205,13 +298,27 @@ sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index) return B_FALSE; } +static boolean_t +sfc_ev_nop_txq_flush_done(void *arg, uint32_t txq_hw_index) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected TxQ %u flush done", + evq->evq_index, txq_hw_index); + return B_TRUE; +} + static boolean_t sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index) { struct sfc_evq *evq = arg; + struct sfc_dp_txq *dp_txq; struct sfc_txq *txq; - txq = evq->txq; + dp_txq = evq->dp_txq; + SFC_ASSERT(dp_txq != NULL); + + txq = sfc_txq_by_dp_txq(dp_txq); SFC_ASSERT(txq != NULL); SFC_ASSERT(txq->hw_index == txq_hw_index); SFC_ASSERT(txq->evq == evq); @@ -260,35 +367,103 @@ sfc_ev_timer(void *arg, uint32_t index) return B_TRUE; } +static boolean_t +sfc_ev_nop_link_change(void *arg, __rte_unused efx_link_mode_t link_mode) +{ + struct sfc_evq *evq = arg; + + sfc_err(evq->sa, "EVQ %u unexpected link change event", + evq->evq_index); + return B_TRUE; +} + static boolean_t sfc_ev_link_change(void *arg, efx_link_mode_t link_mode) { struct sfc_evq *evq = arg; struct sfc_adapter *sa = evq->sa; - struct rte_eth_link *dev_link = &sa->eth_dev->data->dev_link; struct rte_eth_link new_link; - EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t)); - sfc_port_link_mode_to_info(link_mode, &new_link); - rte_atomic64_set((rte_atomic64_t *)dev_link, *(uint64_t *)&new_link); + if (rte_eth_linkstatus_set(sa->eth_dev, &new_link)) + evq->sa->port.lsc_seq++; return B_FALSE; } static const efx_ev_callbacks_t sfc_ev_callbacks = { .eec_initialized = sfc_ev_initialized, - .eec_rx = sfc_ev_rx, - .eec_tx = sfc_ev_tx, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_nop_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_efx_rx, + .eec_tx = sfc_ev_nop_tx, .eec_exception = sfc_ev_exception, .eec_rxq_flush_done = sfc_ev_rxq_flush_done, .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_dp_rx, + .eec_tx = sfc_ev_nop_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, .eec_txq_flush_done = sfc_ev_txq_flush_done, .eec_software = sfc_ev_software, .eec_sram = sfc_ev_sram, .eec_wake_up = sfc_ev_wake_up, .eec_timer = sfc_ev_timer, - .eec_link_change = sfc_ev_link_change, + .eec_link_change = sfc_ev_nop_link_change, +}; + +static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = { + .eec_initialized = sfc_ev_initialized, + .eec_rx = sfc_ev_nop_rx, + .eec_tx = sfc_ev_dp_tx, + .eec_exception = sfc_ev_exception, + .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, + .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, + .eec_txq_flush_done = sfc_ev_txq_flush_done, + .eec_software = sfc_ev_software, + .eec_sram = sfc_ev_sram, + .eec_wake_up = sfc_ev_wake_up, + .eec_timer = sfc_ev_timer, + .eec_link_change = sfc_ev_nop_link_change, }; @@ -300,7 +475,50 @@ sfc_ev_qpoll(struct sfc_evq *evq) /* Synchronize the DMA memory for reading not required */ - efx_ev_qpoll(evq->common, &evq->read_ptr, &sfc_ev_callbacks, evq); + efx_ev_qpoll(evq->common, &evq->read_ptr, evq->callbacks, evq); + + if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) { + struct sfc_adapter *sa = evq->sa; + int rc; + + if (evq->dp_rxq != NULL) { + unsigned int rxq_sw_index; + + rxq_sw_index = evq->dp_rxq->dpq.queue_id; + + sfc_warn(sa, + "restart RxQ %u because of exception on its EvQ %u", + rxq_sw_index, evq->evq_index); + + sfc_rx_qstop(sa, rxq_sw_index); + rc = sfc_rx_qstart(sa, rxq_sw_index); + if (rc != 0) + sfc_err(sa, "cannot restart RxQ %u", + rxq_sw_index); + } + + if (evq->dp_txq != NULL) { + unsigned int txq_sw_index; + + txq_sw_index = evq->dp_txq->dpq.queue_id; + + sfc_warn(sa, + "restart TxQ %u because of exception on its EvQ %u", + txq_sw_index, evq->evq_index); + + sfc_tx_qstop(sa, txq_sw_index); + rc = sfc_tx_qstart(sa, txq_sw_index); + if (rc != 0) + sfc_err(sa, "cannot restart TxQ %u", + txq_sw_index); + } + + if (evq->exception) + sfc_panic(sa, "unrecoverable exception on EvQ %u", + evq->evq_index); + + sfc_adapter_unlock(sa); + } /* Poll-mode driver does not re-prime the event queue for interrupts */ } @@ -309,10 +527,8 @@ void sfc_ev_mgmt_qpoll(struct sfc_adapter *sa) { if (rte_spinlock_trylock(&sa->mgmt_evq_lock)) { - struct sfc_evq *mgmt_evq = sa->evq_info[sa->mgmt_evq_index].evq; - - if (mgmt_evq->init_state == SFC_EVQ_STARTED) - sfc_ev_qpoll(mgmt_evq); + if (sa->mgmt_evq_running) + sfc_ev_qpoll(sa->mgmt_evq); rte_spinlock_unlock(&sa->mgmt_evq_lock); } @@ -325,35 +541,53 @@ sfc_ev_qprime(struct sfc_evq *evq) return efx_ev_qprime(evq->common, evq->read_ptr); } +/* Event queue HW index allocation scheme is described in sfc_ev.h. */ int -sfc_ev_qstart(struct sfc_adapter *sa, unsigned int sw_index) +sfc_ev_qstart(struct sfc_evq *evq, unsigned int hw_index) { - const struct sfc_evq_info *evq_info; - struct sfc_evq *evq; + struct sfc_adapter *sa = evq->sa; efsys_mem_t *esmp; + uint32_t evq_flags = sa->evq_flags; unsigned int total_delay_us; unsigned int delay_us; int rc; - sfc_log_init(sa, "sw_index=%u", sw_index); + sfc_log_init(sa, "hw_index=%u", hw_index); - evq_info = &sa->evq_info[sw_index]; - evq = evq_info->evq; esmp = &evq->mem; + evq->evq_index = hw_index; + /* Clear all events */ - (void)memset((void *)esmp->esm_base, 0xff, - EFX_EVQ_SIZE(evq_info->entries)); + (void)memset((void *)esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); + + if (sa->intr.lsc_intr && hw_index == sa->mgmt_evq_index) + evq_flags |= EFX_EVQ_FLAGS_NOTIFY_INTERRUPT; + else + evq_flags |= EFX_EVQ_FLAGS_NOTIFY_DISABLED; /* Create the common code event queue */ - rc = efx_ev_qcreate(sa->nic, sw_index, esmp, evq_info->entries, - 0 /* unused on EF10 */, 0, - EFX_EVQ_FLAGS_TYPE_THROUGHPUT | - EFX_EVQ_FLAGS_NOTIFY_DISABLED, + rc = efx_ev_qcreate(sa->nic, hw_index, esmp, evq->entries, + 0 /* unused on EF10 */, 0, evq_flags, &evq->common); if (rc != 0) goto fail_ev_qcreate; + SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL); + if (evq->dp_rxq != 0) { + if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) + evq->callbacks = &sfc_ev_callbacks_efx_rx; + else + evq->callbacks = &sfc_ev_callbacks_dp_rx; + } else if (evq->dp_txq != 0) { + if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) + evq->callbacks = &sfc_ev_callbacks_efx_tx; + else + evq->callbacks = &sfc_ev_callbacks_dp_tx; + } else { + evq->callbacks = &sfc_ev_callbacks; + } + evq->init_state = SFC_EVQ_STARTING; /* Wait for the initialization event */ @@ -396,26 +630,24 @@ fail_ev_qcreate: } void -sfc_ev_qstop(struct sfc_adapter *sa, unsigned int sw_index) +sfc_ev_qstop(struct sfc_evq *evq) { - const struct sfc_evq_info *evq_info; - struct sfc_evq *evq; - - sfc_log_init(sa, "sw_index=%u", sw_index); - - SFC_ASSERT(sw_index < sa->evq_count); + if (evq == NULL) + return; - evq_info = &sa->evq_info[sw_index]; - evq = evq_info->evq; + sfc_log_init(evq->sa, "hw_index=%u", evq->evq_index); - if (evq == NULL || evq->init_state != SFC_EVQ_STARTED) + if (evq->init_state != SFC_EVQ_STARTED) return; evq->init_state = SFC_EVQ_INITIALIZED; + evq->callbacks = NULL; evq->read_ptr = 0; evq->exception = B_FALSE; efx_ev_qdestroy(evq->common); + + evq->evq_index = 0; } static void @@ -428,10 +660,14 @@ sfc_ev_mgmt_periodic_qpoll(void *arg) rc = rte_eal_alarm_set(SFC_MGMT_EV_QPOLL_PERIOD_US, sfc_ev_mgmt_periodic_qpoll, sa); - if (rc != 0) - sfc_panic(sa, - "cannot rearm management EVQ polling alarm (rc=%d)", - rc); + if (rc == -ENOTSUP) { + sfc_warn(sa, "alarms are not supported"); + sfc_warn(sa, "management EVQ must be polled indirectly using no-wait link status update"); + } else if (rc != 0) { + sfc_err(sa, + "cannot rearm management EVQ polling alarm (rc=%d)", + rc); + } } static void @@ -458,14 +694,26 @@ sfc_ev_start(struct sfc_adapter *sa) goto fail_ev_init; /* Start management EVQ used for global events */ - rte_spinlock_lock(&sa->mgmt_evq_lock); - rc = sfc_ev_qstart(sa, sa->mgmt_evq_index); + /* + * Management event queue start polls the queue, but it cannot + * interfere with other polling contexts since mgmt_evq_running + * is false yet. + */ + rc = sfc_ev_qstart(sa->mgmt_evq, sa->mgmt_evq_index); if (rc != 0) goto fail_mgmt_evq_start; + rte_spinlock_lock(&sa->mgmt_evq_lock); + sa->mgmt_evq_running = true; rte_spinlock_unlock(&sa->mgmt_evq_lock); + if (sa->intr.lsc_intr) { + rc = sfc_ev_qprime(sa->mgmt_evq); + if (rc != 0) + goto fail_mgmt_evq_prime; + } + /* * Start management EVQ polling. If interrupts are disabled * (not used), it is required to process link status change @@ -481,8 +729,10 @@ sfc_ev_start(struct sfc_adapter *sa) return 0; +fail_mgmt_evq_prime: + sfc_ev_qstop(sa->mgmt_evq); + fail_mgmt_evq_start: - rte_spinlock_unlock(&sa->mgmt_evq_lock); efx_ev_fini(sa->nic); fail_ev_init: @@ -493,135 +743,120 @@ fail_ev_init: void sfc_ev_stop(struct sfc_adapter *sa) { - unsigned int sw_index; - sfc_log_init(sa, "entry"); sfc_ev_mgmt_periodic_qpoll_stop(sa); - /* Make sure that all event queues are stopped */ - sw_index = sa->evq_count; - while (sw_index-- > 0) { - if (sw_index == sa->mgmt_evq_index) { - /* Locks are required for the management EVQ */ - rte_spinlock_lock(&sa->mgmt_evq_lock); - sfc_ev_qstop(sa, sa->mgmt_evq_index); - rte_spinlock_unlock(&sa->mgmt_evq_lock); - } else { - sfc_ev_qstop(sa, sw_index); - } - } + rte_spinlock_lock(&sa->mgmt_evq_lock); + sa->mgmt_evq_running = false; + rte_spinlock_unlock(&sa->mgmt_evq_lock); + + sfc_ev_qstop(sa->mgmt_evq); efx_ev_fini(sa->nic); } int -sfc_ev_qinit(struct sfc_adapter *sa, unsigned int sw_index, - unsigned int entries, int socket_id) +sfc_ev_qinit(struct sfc_adapter *sa, + enum sfc_evq_type type, unsigned int type_index, + unsigned int entries, int socket_id, struct sfc_evq **evqp) { - struct sfc_evq_info *evq_info; struct sfc_evq *evq; int rc; - sfc_log_init(sa, "sw_index=%u", sw_index); - - evq_info = &sa->evq_info[sw_index]; + sfc_log_init(sa, "type=%s type_index=%u", + sfc_evq_type2str(type), type_index); SFC_ASSERT(rte_is_power_of_2(entries)); - SFC_ASSERT(entries <= evq_info->max_entries); - evq_info->entries = entries; + rc = ENOMEM; evq = rte_zmalloc_socket("sfc-evq", sizeof(*evq), RTE_CACHE_LINE_SIZE, socket_id); if (evq == NULL) - return ENOMEM; + goto fail_evq_alloc; evq->sa = sa; - evq->evq_index = sw_index; + evq->type = type; + evq->entries = entries; /* Allocate DMA space */ - rc = sfc_dma_alloc(sa, "evq", sw_index, EFX_EVQ_SIZE(evq_info->entries), - socket_id, &evq->mem); + rc = sfc_dma_alloc(sa, sfc_evq_type2str(type), type_index, + EFX_EVQ_SIZE(evq->entries), socket_id, &evq->mem); if (rc != 0) - return rc; + goto fail_dma_alloc; evq->init_state = SFC_EVQ_INITIALIZED; - evq_info->evq = evq; + sa->evq_count++; + + *evqp = evq; return 0; + +fail_dma_alloc: + rte_free(evq); + +fail_evq_alloc: + + sfc_log_init(sa, "failed %d", rc); + return rc; } void -sfc_ev_qfini(struct sfc_adapter *sa, unsigned int sw_index) +sfc_ev_qfini(struct sfc_evq *evq) { - struct sfc_evq *evq; - - sfc_log_init(sa, "sw_index=%u", sw_index); - - evq = sa->evq_info[sw_index].evq; + struct sfc_adapter *sa = evq->sa; SFC_ASSERT(evq->init_state == SFC_EVQ_INITIALIZED); - sa->evq_info[sw_index].evq = NULL; - sfc_dma_free(sa, &evq->mem); rte_free(evq); + + SFC_ASSERT(sa->evq_count > 0); + sa->evq_count--; } static int -sfc_ev_qinit_info(struct sfc_adapter *sa, unsigned int sw_index) +sfc_kvarg_perf_profile_handler(__rte_unused const char *key, + const char *value_str, void *opaque) { - struct sfc_evq_info *evq_info = &sa->evq_info[sw_index]; - unsigned int max_entries; - - sfc_log_init(sa, "sw_index=%u", sw_index); + uint32_t *value = opaque; - max_entries = sfc_evq_max_entries(sa, sw_index); - SFC_ASSERT(rte_is_power_of_2(max_entries)); - - evq_info->max_entries = max_entries; + if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0) + *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; + else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0) + *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY; + else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0) + *value = EFX_EVQ_FLAGS_TYPE_AUTO; + else + return -EINVAL; return 0; } -static void -sfc_ev_qfini_info(struct sfc_adapter *sa, unsigned int sw_index) -{ - sfc_log_init(sa, "sw_index=%u", sw_index); - - /* Nothing to cleanup */ -} - int -sfc_ev_init(struct sfc_adapter *sa) +sfc_ev_attach(struct sfc_adapter *sa) { int rc; - unsigned int sw_index; sfc_log_init(sa, "entry"); - sa->evq_count = sfc_ev_qcount(sa); + sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; + rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE, + sfc_kvarg_perf_profile_handler, + &sa->evq_flags); + if (rc != 0) { + sfc_err(sa, "invalid %s parameter value", + SFC_KVARG_PERF_PROFILE); + goto fail_kvarg_perf_profile; + } + sa->mgmt_evq_index = 0; rte_spinlock_init(&sa->mgmt_evq_lock); - /* Allocate EVQ info array */ - rc = ENOMEM; - sa->evq_info = rte_calloc_socket("sfc-evqs", sa->evq_count, - sizeof(struct sfc_evq_info), 0, - sa->socket_id); - if (sa->evq_info == NULL) - goto fail_evqs_alloc; - - for (sw_index = 0; sw_index < sa->evq_count; ++sw_index) { - rc = sfc_ev_qinit_info(sa, sw_index); - if (rc != 0) - goto fail_ev_qinit_info; - } - - rc = sfc_ev_qinit(sa, sa->mgmt_evq_index, SFC_MGMT_EVQ_ENTRIES, - sa->socket_id); + rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_MGMT, 0, SFC_MGMT_EVQ_ENTRIES, + sa->socket_id, &sa->mgmt_evq); if (rc != 0) goto fail_mgmt_evq_init; @@ -633,35 +868,20 @@ sfc_ev_init(struct sfc_adapter *sa) return 0; fail_mgmt_evq_init: -fail_ev_qinit_info: - while (sw_index-- > 0) - sfc_ev_qfini_info(sa, sw_index); - rte_free(sa->evq_info); - sa->evq_info = NULL; - -fail_evqs_alloc: - sa->evq_count = 0; +fail_kvarg_perf_profile: sfc_log_init(sa, "failed %d", rc); return rc; } void -sfc_ev_fini(struct sfc_adapter *sa) +sfc_ev_detach(struct sfc_adapter *sa) { - int sw_index; - sfc_log_init(sa, "entry"); - /* Cleanup all event queues */ - sw_index = sa->evq_count; - while (--sw_index >= 0) { - if (sa->evq_info[sw_index].evq != NULL) - sfc_ev_qfini(sa, sw_index); - sfc_ev_qfini_info(sa, sw_index); - } + sfc_ev_qfini(sa->mgmt_evq); - rte_free(sa->evq_info); - sa->evq_info = NULL; - sa->evq_count = 0; + if (sa->evq_count != 0) + sfc_err(sa, "%u EvQs are not destroyed before detach", + sa->evq_count); }