X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fsfc%2Fsfc_rx.c;h=734ce24f7128dd7fa78e7e5c6cd9a039a3cede32;hb=c2817ba40a1cc3196a86902d908870eb74e9eb58;hp=56e48ab04c50d99e6e08afec4edd3191e4f60650;hpb=99a4949ff73622c659bf7083a731b0345aa982cc;p=dpdk.git diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c index 56e48ab04c..734ce24f71 100644 --- a/drivers/net/sfc/sfc_rx.c +++ b/drivers/net/sfc/sfc_rx.c @@ -1,32 +1,10 @@ -/*- - * BSD LICENSE +/* SPDX-License-Identifier: BSD-3-Clause * - * Copyright (c) 2016-2017 Solarflare Communications Inc. + * Copyright (c) 2016-2018 Solarflare Communications Inc. * All rights reserved. * * This software was jointly developed between OKTET Labs (under contract * for Solarflare) and Solarflare Communications, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include @@ -88,24 +66,29 @@ sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq) struct rte_mbuf *m; uint16_t port_id = rxq->dp.dpq.port_id; - free_space = EFX_RXQ_LIMIT(rxq->ptr_mask + 1) - - (added - rxq->completed); + free_space = rxq->max_fill_level - (added - rxq->completed); if (free_space < rxq->refill_threshold) return; bulks = free_space / RTE_DIM(objs); + /* refill_threshold guarantees that bulks is positive */ + SFC_ASSERT(bulks > 0); id = added & rxq->ptr_mask; - while (bulks-- > 0) { - if (rte_mempool_get_bulk(rxq->refill_mb_pool, objs, - RTE_DIM(objs)) < 0) { + do { + if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, + RTE_DIM(objs)) < 0)) { /* * It is hardly a safe way to increment counter * from different contexts, but all PMDs do it. */ rxq->evq->sa->eth_dev->data->rx_mbuf_alloc_failed += RTE_DIM(objs); + /* Return if we have posted nothing yet */ + if (added == rxq->added) + return; + /* Push posted */ break; } @@ -116,25 +99,23 @@ sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq) rxd = &rxq->sw_desc[id]; rxd->mbuf = m; - rte_mbuf_refcnt_set(m, 1); + SFC_ASSERT(rte_mbuf_refcnt_read(m) == 1); m->data_off = RTE_PKTMBUF_HEADROOM; - m->next = NULL; - m->nb_segs = 1; + SFC_ASSERT(m->next == NULL); + SFC_ASSERT(m->nb_segs == 1); m->port = port_id; - addr[i] = rte_pktmbuf_mtophys(m); + addr[i] = rte_pktmbuf_iova(m); } efx_rx_qpost(rxq->common, addr, rxq->buf_size, RTE_DIM(objs), rxq->completed, added); added += RTE_DIM(objs); - } + } while (--bulks > 0); - /* Push doorbell if something is posted */ - if (rxq->added != added) { - rxq->added = added; - efx_rx_qpush(rxq->common, added, &rxq->pushed); - } + SFC_ASSERT(added != rxq->added); + rxq->added = added; + efx_rx_qpush(rxq->common, added, &rxq->pushed); } static uint64_t @@ -189,7 +170,7 @@ sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags) } static const uint32_t * -sfc_efx_supported_ptypes_get(void) +sfc_efx_supported_ptypes_get(__rte_unused uint32_t tunnel_encaps) { static const uint32_t ptypes[] = { RTE_PTYPE_L2_ETHER, @@ -203,11 +184,11 @@ sfc_efx_supported_ptypes_get(void) return ptypes; } +#if EFSYS_OPT_RX_SCALE static void sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags, struct rte_mbuf *m) { -#if EFSYS_OPT_RX_SCALE uint8_t *mbuf_data; @@ -223,8 +204,15 @@ sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags, m->ol_flags |= PKT_RX_RSS_HASH; } -#endif } +#else +static void +sfc_efx_rx_set_rss_hash(__rte_unused struct sfc_efx_rxq *rxq, + __rte_unused unsigned int flags, + __rte_unused struct rte_mbuf *m) +{ +} +#endif static uint16_t sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) @@ -278,8 +266,7 @@ sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (scatter_pkt != NULL) { if (rte_pktmbuf_chain(scatter_pkt, m) != 0) { - rte_mempool_put(rxq->refill_mb_pool, - scatter_pkt); + rte_pktmbuf_free(scatter_pkt); goto discard; } /* The packet to deliver */ @@ -289,7 +276,7 @@ sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (desc_flags & EFX_PKT_CONT) { /* The packet is scattered, more fragments to come */ scatter_pkt = m; - /* Futher fragments have no prefix */ + /* Further fragments have no prefix */ prefix_size = 0; continue; } @@ -346,6 +333,43 @@ sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) return rxq->pending - rxq->completed; } +static sfc_dp_rx_qdesc_status_t sfc_efx_rx_qdesc_status; +static int +sfc_efx_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) +{ + struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq); + + if (unlikely(offset > rxq->ptr_mask)) + return -EINVAL; + + /* + * Poll EvQ to derive up-to-date 'rxq->pending' figure; + * it is required for the queue to be running, but the + * check is omitted because API design assumes that it + * is the duty of the caller to satisfy all conditions + */ + SFC_ASSERT((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == + SFC_EFX_RXQ_FLAG_RUNNING); + sfc_ev_qpoll(rxq->evq); + + /* + * There is a handful of reserved entries in the ring, + * but an explicit check whether the offset points to + * a reserved entry is neglected since the two checks + * below rely on the figures which take the HW limits + * into account and thus if an entry is reserved, the + * checks will fail and UNAVAIL code will be returned + */ + + if (offset < (rxq->pending - rxq->completed)) + return RTE_ETH_RX_DESC_DONE; + + if (offset < (rxq->added - rxq->completed)) + return RTE_ETH_RX_DESC_AVAIL; + + return RTE_ETH_RX_DESC_UNAVAIL; +} + struct sfc_rxq * sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) { @@ -366,6 +390,19 @@ sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq) return rxq; } +static sfc_dp_rx_qsize_up_rings_t sfc_efx_rx_qsize_up_rings; +static int +sfc_efx_rx_qsize_up_rings(uint16_t nb_rx_desc, + unsigned int *rxq_entries, + unsigned int *evq_entries, + unsigned int *rxq_max_fill_level) +{ + *rxq_entries = nb_rx_desc; + *evq_entries = nb_rx_desc; + *rxq_max_fill_level = EFX_RXQ_LIMIT(*rxq_entries); + return 0; +} + static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate; static int sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id, @@ -399,6 +436,7 @@ sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id, rxq->ptr_mask = info->rxq_entries - 1; rxq->batch_max = info->batch_max; rxq->prefix_size = info->prefix_size; + rxq->max_fill_level = info->max_fill_level; rxq->refill_threshold = info->refill_threshold; rxq->buf_size = info->buf_size; rxq->refill_mb_pool = info->refill_mb_pool; @@ -488,6 +526,7 @@ struct sfc_dp_rx sfc_efx_rx = { .hw_fw_caps = 0, }, .features = SFC_DP_RX_FEAT_SCATTER, + .qsize_up_rings = sfc_efx_rx_qsize_up_rings, .qcreate = sfc_efx_rx_qcreate, .qdestroy = sfc_efx_rx_qdestroy, .qstart = sfc_efx_rx_qstart, @@ -495,6 +534,7 @@ struct sfc_dp_rx sfc_efx_rx = { .qpurge = sfc_efx_rx_qpurge, .supported_ptypes_get = sfc_efx_supported_ptypes_get, .qdesc_npending = sfc_efx_rx_qdesc_npending, + .qdesc_status = sfc_efx_rx_qdesc_status, .pkt_burst = sfc_efx_recv_pkts, }; @@ -526,6 +566,7 @@ sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) struct sfc_rxq *rxq; unsigned int retry_count; unsigned int wait_count; + int rc; rxq = sa->rxq_info[sw_index].rxq; SFC_ASSERT(rxq->state & SFC_RXQ_STARTED); @@ -538,8 +579,10 @@ sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) ((rxq->state & SFC_RXQ_FLUSHED) == 0) && (retry_count < SFC_RX_QFLUSH_ATTEMPTS); ++retry_count) { - if (efx_rx_qflush(rxq->common) != 0) { - rxq->state |= SFC_RXQ_FLUSH_FAILED; + rc = efx_rx_qflush(rxq->common); + if (rc != 0) { + rxq->state |= (rc == EALREADY) ? + SFC_RXQ_FLUSHED : SFC_RXQ_FLUSH_FAILED; break; } rxq->state &= ~SFC_RXQ_FLUSH_FAILED; @@ -565,7 +608,7 @@ sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) sfc_err(sa, "RxQ %u flush failed", sw_index); if (rxq->state & SFC_RXQ_FLUSHED) - sfc_info(sa, "RxQ %u flushed", sw_index); + sfc_notice(sa, "RxQ %u flushed", sw_index); } sa->dp_rx->qpurge(rxq->dp); @@ -574,7 +617,7 @@ sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index) static int sfc_rx_default_rxq_set_filter(struct sfc_adapter *sa, struct sfc_rxq *rxq) { - boolean_t rss = (sa->rss_channels > 1) ? B_TRUE : B_FALSE; + boolean_t rss = (sa->rss_channels > 0) ? B_TRUE : B_FALSE; struct sfc_port *port = &sa->port; int rc; @@ -624,6 +667,7 @@ retry: int sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) { + struct sfc_port *port = &sa->port; struct sfc_rxq_info *rxq_info; struct sfc_rxq *rxq; struct sfc_evq *evq; @@ -639,14 +683,14 @@ sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) evq = rxq->evq; - rc = sfc_ev_qstart(sa, evq->evq_index); + rc = sfc_ev_qstart(evq, sfc_evq_index_by_rxq_sw_index(sa, sw_index)); if (rc != 0) goto fail_ev_qstart; rc = efx_rx_qcreate(sa->nic, rxq->hw_index, 0, rxq_info->type, &rxq->mem, rxq_info->entries, - 0 /* not used on EF10 */, evq->common, - &rxq->common); + 0 /* not used on EF10 */, rxq_info->type_flags, + evq->common, &rxq->common); if (rc != 0) goto fail_rx_qcreate; @@ -658,7 +702,7 @@ sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index) rxq->state |= SFC_RXQ_STARTED; - if (sw_index == 0) { + if ((sw_index == 0) && !port->isolated) { rc = sfc_rx_default_rxq_set_filter(sa, rxq); if (rc != 0) goto fail_mac_filter_default_rxq_set; @@ -677,7 +721,7 @@ fail_dp_qstart: sfc_rx_qflush(sa, sw_index); fail_rx_qcreate: - sfc_ev_qstop(sa, evq->evq_index); + sfc_ev_qstop(evq); fail_ev_qstart: return rc; @@ -715,28 +759,97 @@ sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index) efx_rx_qdestroy(rxq->common); - sfc_ev_qstop(sa, rxq->evq->evq_index); + sfc_ev_qstop(rxq->evq); +} + +uint64_t +sfc_rx_get_dev_offload_caps(struct sfc_adapter *sa) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + uint64_t caps = 0; + + caps |= DEV_RX_OFFLOAD_JUMBO_FRAME; + caps |= DEV_RX_OFFLOAD_CRC_STRIP; + caps |= DEV_RX_OFFLOAD_IPV4_CKSUM; + caps |= DEV_RX_OFFLOAD_UDP_CKSUM; + caps |= DEV_RX_OFFLOAD_TCP_CKSUM; + + if (encp->enc_tunnel_encapsulations_supported && + (sa->dp_rx->features & SFC_DP_RX_FEAT_TUNNELS)) + caps |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; + + return caps; +} + +uint64_t +sfc_rx_get_queue_offload_caps(struct sfc_adapter *sa) +{ + uint64_t caps = 0; + + if (sa->dp_rx->features & SFC_DP_RX_FEAT_SCATTER) + caps |= DEV_RX_OFFLOAD_SCATTER; + + return caps; +} + +static void +sfc_rx_log_offloads(struct sfc_adapter *sa, const char *offload_group, + const char *verdict, uint64_t offloads) +{ + unsigned long long bit; + + while ((bit = __builtin_ffsll(offloads)) != 0) { + uint64_t flag = (1ULL << --bit); + + sfc_err(sa, "Rx %s offload %s %s", offload_group, + rte_eth_dev_rx_offload_name(flag), verdict); + + offloads &= ~flag; + } +} + +static boolean_t +sfc_rx_queue_offloads_mismatch(struct sfc_adapter *sa, uint64_t requested) +{ + uint64_t mandatory = sa->eth_dev->data->dev_conf.rxmode.offloads; + uint64_t supported = sfc_rx_get_dev_offload_caps(sa) | + sfc_rx_get_queue_offload_caps(sa); + uint64_t rejected = requested & ~supported; + uint64_t missing = (requested & mandatory) ^ mandatory; + boolean_t mismatch = B_FALSE; + + if (rejected) { + sfc_rx_log_offloads(sa, "queue", "is unsupported", rejected); + mismatch = B_TRUE; + } + + if (missing) { + sfc_rx_log_offloads(sa, "queue", "must be set", missing); + mismatch = B_TRUE; + } + + return mismatch; } static int -sfc_rx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_rx_desc, +sfc_rx_qcheck_conf(struct sfc_adapter *sa, unsigned int rxq_max_fill_level, const struct rte_eth_rxconf *rx_conf) { - const uint16_t rx_free_thresh_max = EFX_RXQ_LIMIT(nb_rx_desc); + uint64_t offloads_supported = sfc_rx_get_dev_offload_caps(sa) | + sfc_rx_get_queue_offload_caps(sa); int rc = 0; if (rx_conf->rx_thresh.pthresh != 0 || rx_conf->rx_thresh.hthresh != 0 || rx_conf->rx_thresh.wthresh != 0) { - sfc_err(sa, + sfc_warn(sa, "RxQ prefetch/host/writeback thresholds are not supported"); - rc = EINVAL; } - if (rx_conf->rx_free_thresh > rx_free_thresh_max) { + if (rx_conf->rx_free_thresh > rxq_max_fill_level) { sfc_err(sa, "RxQ free threshold too large: %u vs maximum %u", - rx_conf->rx_free_thresh, rx_free_thresh_max); + rx_conf->rx_free_thresh, rxq_max_fill_level); rc = EINVAL; } @@ -745,6 +858,17 @@ sfc_rx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_rx_desc, rc = EINVAL; } + if ((rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM) != + DEV_RX_OFFLOAD_CHECKSUM) + sfc_warn(sa, "Rx checksum offloads cannot be disabled - always on (IPv4/TCP/UDP)"); + + if ((offloads_supported & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM) && + (~rx_conf->offloads & DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM)) + sfc_warn(sa, "Rx outer IPv4 checksum offload cannot be disabled - always on"); + + if (sfc_rx_queue_offloads_mismatch(sa, rx_conf->offloads)) + rc = EINVAL; + return rc; } @@ -763,7 +887,7 @@ sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool) order = MIN(order, rte_bsf32(data_off)); - return 1u << (order - 1); + return 1u << order; } static uint16_t @@ -856,14 +980,25 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, { const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); int rc; + unsigned int rxq_entries; + unsigned int evq_entries; + unsigned int rxq_max_fill_level; uint16_t buf_size; struct sfc_rxq_info *rxq_info; - unsigned int evq_index; struct sfc_evq *evq; struct sfc_rxq *rxq; struct sfc_dp_rx_qcreate_info info; - rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf); + rc = sa->dp_rx->qsize_up_rings(nb_rx_desc, &rxq_entries, &evq_entries, + &rxq_max_fill_level); + if (rc != 0) + goto fail_size_up_rings; + SFC_ASSERT(rxq_entries >= EFX_RXQ_MINNDESCS); + SFC_ASSERT(rxq_entries <= EFX_RXQ_MAXNDESCS); + SFC_ASSERT(rxq_entries >= nb_rx_desc); + SFC_ASSERT(rxq_max_fill_level <= nb_rx_desc); + + rc = sfc_rx_qcheck_conf(sa, rxq_max_fill_level, rx_conf); if (rc != 0) goto fail_bad_conf; @@ -876,7 +1011,7 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, } if ((buf_size < sa->port.pdu + encp->enc_rx_prefix_size) && - !sa->eth_dev->data->dev_conf.rxmode.enable_scatter) { + (~rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)) { sfc_err(sa, "Rx scatter is disabled and RxQ %u mbuf pool " "object size is too small", sw_index); sfc_err(sa, "RxQ %u calculated Rx buffer size is %u vs " @@ -890,20 +1025,22 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, SFC_ASSERT(sw_index < sa->rxq_count); rxq_info = &sa->rxq_info[sw_index]; - SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries); - rxq_info->entries = nb_rx_desc; - rxq_info->type = - sa->eth_dev->data->dev_conf.rxmode.enable_scatter ? - EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT; + SFC_ASSERT(rxq_entries <= rxq_info->max_entries); + rxq_info->entries = rxq_entries; + rxq_info->type = EFX_RXQ_TYPE_DEFAULT; + rxq_info->type_flags = + (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) ? + EFX_RXQ_FLAG_SCATTER : EFX_RXQ_FLAG_NONE; - evq_index = sfc_evq_index_by_rxq_sw_index(sa, sw_index); + if ((encp->enc_tunnel_encapsulations_supported != 0) && + (sa->dp_rx->features & SFC_DP_RX_FEAT_TUNNELS)) + rxq_info->type_flags |= EFX_RXQ_FLAG_INNER_CLASSES; - rc = sfc_ev_qinit(sa, evq_index, rxq_info->entries, socket_id); + rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_RX, sw_index, + evq_entries, socket_id, &evq); if (rc != 0) goto fail_ev_qinit; - evq = sa->evq_info[evq_index].evq; - rc = ENOMEM; rxq = rte_zmalloc_socket("sfc-rxq", sizeof(*rxq), RTE_CACHE_LINE_SIZE, socket_id); @@ -914,7 +1051,8 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, rxq->evq = evq; rxq->hw_index = sw_index; - rxq->refill_threshold = rx_conf->rx_free_thresh; + rxq->refill_threshold = + RTE_MAX(rx_conf->rx_free_thresh, SFC_RX_REFILL_BULK); rxq->refill_mb_pool = mb_pool; rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries), @@ -924,20 +1062,27 @@ sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index, memset(&info, 0, sizeof(info)); info.refill_mb_pool = rxq->refill_mb_pool; + info.max_fill_level = rxq_max_fill_level; info.refill_threshold = rxq->refill_threshold; info.buf_size = buf_size; info.batch_max = encp->enc_rx_batch_max; info.prefix_size = encp->enc_rx_prefix_size; #if EFSYS_OPT_RX_SCALE - if (sa->hash_support == EFX_RX_HASH_AVAILABLE) + if (sa->hash_support == EFX_RX_HASH_AVAILABLE && sa->rss_channels > 0) info.flags |= SFC_RXQ_FLAG_RSS_HASH; #endif info.rxq_entries = rxq_info->entries; + info.rxq_hw_ring = rxq->mem.esm_base; + info.evq_entries = evq_entries; + info.evq_hw_ring = evq->mem.esm_base; + info.hw_index = rxq->hw_index; + info.mem_bar = sa->mem_bar.esb_base; + info.vi_window_shift = encp->enc_vi_window_shift; rc = sa->dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index, - &SFC_DEV_TO_PCI(sa->eth_dev)->addr, + &RTE_ETH_DEV_TO_PCI(sa->eth_dev)->addr, socket_id, &info, &rxq->dp); if (rc != 0) goto fail_dp_rx_qcreate; @@ -958,12 +1103,13 @@ fail_dma_alloc: rte_free(rxq); fail_rxq_alloc: - sfc_ev_qfini(sa, evq_index); + sfc_ev_qfini(evq); fail_ev_qinit: rxq_info->entries = 0; fail_bad_conf: +fail_size_up_rings: sfc_log_init(sa, "failed %d", rc); return rc; } @@ -988,6 +1134,10 @@ sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index) rxq_info->entries = 0; sfc_dma_free(sa, &rxq->mem); + + sfc_ev_qfini(rxq->evq); + rxq->evq = NULL; + rte_free(rxq); } @@ -1037,31 +1187,39 @@ sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types) } #endif +#if EFSYS_OPT_RX_SCALE static int sfc_rx_rss_config(struct sfc_adapter *sa) { int rc = 0; -#if EFSYS_OPT_RX_SCALE - if (sa->rss_channels > 1) { - rc = efx_rx_scale_mode_set(sa->nic, EFX_RX_HASHALG_TOEPLITZ, + if (sa->rss_channels > 0) { + rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + EFX_RX_HASHALG_TOEPLITZ, sa->rss_hash_types, B_TRUE); if (rc != 0) goto finish; - rc = efx_rx_scale_key_set(sa->nic, sa->rss_key, + rc = efx_rx_scale_key_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + sa->rss_key, sizeof(sa->rss_key)); if (rc != 0) goto finish; - rc = efx_rx_scale_tbl_set(sa->nic, sa->rss_tbl, - sizeof(sa->rss_tbl)); + rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT, + sa->rss_tbl, RTE_DIM(sa->rss_tbl)); } finish: -#endif return rc; } +#else +static int +sfc_rx_rss_config(__rte_unused struct sfc_adapter *sa) +{ + return 0; +} +#endif int sfc_rx_start(struct sfc_adapter *sa) @@ -1135,6 +1293,9 @@ sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index) static int sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode) { + uint64_t offloads_supported = sfc_rx_get_dev_offload_caps(sa) | + sfc_rx_get_queue_offload_caps(sa); + uint64_t offloads_rejected = rxmode->offloads & ~offloads_supported; int rc = 0; switch (rxmode->mq_mode) { @@ -1155,100 +1316,121 @@ sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode) rc = EINVAL; } - if (rxmode->header_split) { - sfc_err(sa, "Header split on Rx not supported"); - rc = EINVAL; - } - - if (rxmode->hw_vlan_filter) { - sfc_err(sa, "HW VLAN filtering not supported"); + if (offloads_rejected) { + sfc_rx_log_offloads(sa, "device", "is unsupported", + offloads_rejected); rc = EINVAL; } - if (rxmode->hw_vlan_strip) { - sfc_err(sa, "HW VLAN stripping not supported"); - rc = EINVAL; + if (~rxmode->offloads & DEV_RX_OFFLOAD_CRC_STRIP) { + sfc_warn(sa, "FCS stripping cannot be disabled - always on"); + rxmode->offloads |= DEV_RX_OFFLOAD_CRC_STRIP; + rxmode->hw_strip_crc = 1; } - if (rxmode->hw_vlan_extend) { - sfc_err(sa, - "Q-in-Q HW VLAN stripping not supported"); - rc = EINVAL; - } + return rc; +} - if (!rxmode->hw_strip_crc) { - sfc_warn(sa, - "FCS stripping control not supported - always stripped"); - rxmode->hw_strip_crc = 1; - } +/** + * Destroy excess queues that are no longer needed after reconfiguration + * or complete close. + */ +static void +sfc_rx_fini_queues(struct sfc_adapter *sa, unsigned int nb_rx_queues) +{ + int sw_index; - if (rxmode->enable_scatter && - (~sa->dp_rx->features & SFC_DP_RX_FEAT_SCATTER)) { - sfc_err(sa, "Rx scatter not supported by %s datapath", - sa->dp_rx->dp.name); - rc = EINVAL; - } + SFC_ASSERT(nb_rx_queues <= sa->rxq_count); - if (rxmode->enable_lro) { - sfc_err(sa, "LRO not supported"); - rc = EINVAL; + sw_index = sa->rxq_count; + while (--sw_index >= (int)nb_rx_queues) { + if (sa->rxq_info[sw_index].rxq != NULL) + sfc_rx_qfini(sa, sw_index); } - return rc; + sa->rxq_count = nb_rx_queues; } /** * Initialize Rx subsystem. * - * Called at device configuration stage when number of receive queues is + * Called at device (re)configuration stage when number of receive queues is * specified together with other device level receive configuration. * * It should be used to allocate NUMA-unaware resources. */ int -sfc_rx_init(struct sfc_adapter *sa) +sfc_rx_configure(struct sfc_adapter *sa) { struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf; - unsigned int sw_index; + const unsigned int nb_rx_queues = sa->eth_dev->data->nb_rx_queues; int rc; + sfc_log_init(sa, "nb_rx_queues=%u (old %u)", + nb_rx_queues, sa->rxq_count); + rc = sfc_rx_check_mode(sa, &dev_conf->rxmode); if (rc != 0) goto fail_check_mode; - sa->rxq_count = sa->eth_dev->data->nb_rx_queues; + if (nb_rx_queues == sa->rxq_count) + goto done; - rc = ENOMEM; - sa->rxq_info = rte_calloc_socket("sfc-rxqs", sa->rxq_count, - sizeof(struct sfc_rxq_info), 0, - sa->socket_id); - if (sa->rxq_info == NULL) - goto fail_rxqs_alloc; + if (sa->rxq_info == NULL) { + rc = ENOMEM; + sa->rxq_info = rte_calloc_socket("sfc-rxqs", nb_rx_queues, + sizeof(sa->rxq_info[0]), 0, + sa->socket_id); + if (sa->rxq_info == NULL) + goto fail_rxqs_alloc; + } else { + struct sfc_rxq_info *new_rxq_info; + + if (nb_rx_queues < sa->rxq_count) + sfc_rx_fini_queues(sa, nb_rx_queues); + + rc = ENOMEM; + new_rxq_info = + rte_realloc(sa->rxq_info, + nb_rx_queues * sizeof(sa->rxq_info[0]), 0); + if (new_rxq_info == NULL && nb_rx_queues > 0) + goto fail_rxqs_realloc; + + sa->rxq_info = new_rxq_info; + if (nb_rx_queues > sa->rxq_count) + memset(&sa->rxq_info[sa->rxq_count], 0, + (nb_rx_queues - sa->rxq_count) * + sizeof(sa->rxq_info[0])); + } - for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) { - rc = sfc_rx_qinit_info(sa, sw_index); + while (sa->rxq_count < nb_rx_queues) { + rc = sfc_rx_qinit_info(sa, sa->rxq_count); if (rc != 0) goto fail_rx_qinit_info; + + sa->rxq_count++; } #if EFSYS_OPT_RX_SCALE sa->rss_channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ? - MIN(sa->rxq_count, EFX_MAXRSS) : 1; + MIN(sa->rxq_count, EFX_MAXRSS) : 0; + + if (sa->rss_channels > 0) { + unsigned int sw_index; - if (sa->rss_channels > 1) { for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index) sa->rss_tbl[sw_index] = sw_index % sa->rss_channels; } #endif +done: return 0; fail_rx_qinit_info: - rte_free(sa->rxq_info); - sa->rxq_info = NULL; - +fail_rxqs_realloc: fail_rxqs_alloc: - sa->rxq_count = 0; + sfc_rx_close(sa); + fail_check_mode: sfc_log_init(sa, "failed %d", rc); return rc; @@ -1257,21 +1439,15 @@ fail_check_mode: /** * Shutdown Rx subsystem. * - * Called at device close stage, for example, before device - * reconfiguration or shutdown. + * Called at device close stage, for example, before device shutdown. */ void -sfc_rx_fini(struct sfc_adapter *sa) +sfc_rx_close(struct sfc_adapter *sa) { - unsigned int sw_index; + sfc_rx_fini_queues(sa, 0); - sw_index = sa->rxq_count; - while (sw_index-- > 0) { - if (sa->rxq_info[sw_index].rxq != NULL) - sfc_rx_qfini(sa, sw_index); - } + sa->rss_channels = 0; rte_free(sa->rxq_info); sa->rxq_info = NULL; - sa->rxq_count = 0; }