4 * Copyright (c) 2016-2017 Solarflare Communications Inc.
7 * This software was jointly developed between OKTET Labs (under contract
8 * for Solarflare) and Solarflare Communications, Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <rte_errno.h>
36 #include <rte_alarm.h>
48 sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id,
49 size_t len, int socket_id, efsys_mem_t *esmp)
51 const struct rte_memzone *mz;
53 sfc_log_init(sa, "name=%s id=%u len=%lu socket_id=%d",
54 name, id, len, socket_id);
56 mz = rte_eth_dma_zone_reserve(sa->eth_dev, name, id, len,
57 sysconf(_SC_PAGESIZE), socket_id);
59 sfc_err(sa, "cannot reserve DMA zone for %s:%u %#x@%d: %s",
60 name, (unsigned int)id, (unsigned int)len, socket_id,
61 rte_strerror(rte_errno));
65 esmp->esm_addr = mz->iova;
66 if (esmp->esm_addr == RTE_BAD_IOVA) {
67 (void)rte_memzone_free(mz);
72 esmp->esm_base = mz->addr;
78 sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp)
82 sfc_log_init(sa, "name=%s", esmp->esm_mz->name);
84 rc = rte_memzone_free(esmp->esm_mz);
86 sfc_err(sa, "rte_memzone_free(() failed: %d", rc);
88 memset(esmp, 0, sizeof(*esmp));
92 sfc_phy_cap_from_link_speeds(uint32_t speeds)
94 uint32_t phy_caps = 0;
96 if (~speeds & ETH_LINK_SPEED_FIXED) {
97 phy_caps |= (1 << EFX_PHY_CAP_AN);
99 * If no speeds are specified in the mask, any supported
102 if (speeds == ETH_LINK_SPEED_AUTONEG)
104 (1 << EFX_PHY_CAP_1000FDX) |
105 (1 << EFX_PHY_CAP_10000FDX) |
106 (1 << EFX_PHY_CAP_40000FDX);
108 if (speeds & ETH_LINK_SPEED_1G)
109 phy_caps |= (1 << EFX_PHY_CAP_1000FDX);
110 if (speeds & ETH_LINK_SPEED_10G)
111 phy_caps |= (1 << EFX_PHY_CAP_10000FDX);
112 if (speeds & ETH_LINK_SPEED_40G)
113 phy_caps |= (1 << EFX_PHY_CAP_40000FDX);
119 * Check requested device level configuration.
120 * Receive and transmit configuration is checked in corresponding
124 sfc_check_conf(struct sfc_adapter *sa)
126 const struct rte_eth_conf *conf = &sa->eth_dev->data->dev_conf;
129 sa->port.phy_adv_cap =
130 sfc_phy_cap_from_link_speeds(conf->link_speeds) &
131 sa->port.phy_adv_cap_mask;
132 if ((sa->port.phy_adv_cap & ~(1 << EFX_PHY_CAP_AN)) == 0) {
133 sfc_err(sa, "No link speeds from mask %#x are supported",
138 if (conf->lpbk_mode != 0) {
139 sfc_err(sa, "Loopback not supported");
143 if (conf->dcb_capability_en != 0) {
144 sfc_err(sa, "Priority-based flow control not supported");
148 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) {
149 sfc_err(sa, "Flow Director not supported");
153 if ((conf->intr_conf.lsc != 0) &&
154 (sa->intr.type != EFX_INTR_LINE) &&
155 (sa->intr.type != EFX_INTR_MESSAGE)) {
156 sfc_err(sa, "Link status change interrupt not supported");
160 if (conf->intr_conf.rxq != 0) {
161 sfc_err(sa, "Receive queue interrupt not supported");
169 * Find out maximum number of receive and transmit queues which could be
172 * NIC is kept initialized on success to allow other modules acquire
173 * defaults and capabilities.
176 sfc_estimate_resource_limits(struct sfc_adapter *sa)
178 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
179 efx_drv_limits_t limits;
181 uint32_t evq_allocated;
182 uint32_t rxq_allocated;
183 uint32_t txq_allocated;
185 memset(&limits, 0, sizeof(limits));
187 /* Request at least one Rx and Tx queue */
188 limits.edl_min_rxq_count = 1;
189 limits.edl_min_txq_count = 1;
190 /* Management event queue plus event queue for each Tx and Rx queue */
191 limits.edl_min_evq_count =
192 1 + limits.edl_min_rxq_count + limits.edl_min_txq_count;
194 /* Divide by number of functions to guarantee that all functions
195 * will get promised resources
197 /* FIXME Divide by number of functions (not 2) below */
198 limits.edl_max_evq_count = encp->enc_evq_limit / 2;
199 SFC_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count);
201 /* Split equally between receive and transmit */
202 limits.edl_max_rxq_count =
203 MIN(encp->enc_rxq_limit, (limits.edl_max_evq_count - 1) / 2);
204 SFC_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count);
206 limits.edl_max_txq_count =
207 MIN(encp->enc_txq_limit,
208 limits.edl_max_evq_count - 1 - limits.edl_max_rxq_count);
211 limits.edl_max_txq_count =
212 MIN(limits.edl_max_txq_count,
213 encp->enc_fw_assisted_tso_v2_n_contexts /
214 encp->enc_hw_pf_count);
216 SFC_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count);
218 /* Configure the minimum required resources needed for the
219 * driver to operate, and the maximum desired resources that the
220 * driver is capable of using.
222 efx_nic_set_drv_limits(sa->nic, &limits);
224 sfc_log_init(sa, "init nic");
225 rc = efx_nic_init(sa->nic);
229 /* Find resource dimensions assigned by firmware to this function */
230 rc = efx_nic_get_vi_pool(sa->nic, &evq_allocated, &rxq_allocated,
233 goto fail_get_vi_pool;
235 /* It still may allocate more than maximum, ensure limit */
236 evq_allocated = MIN(evq_allocated, limits.edl_max_evq_count);
237 rxq_allocated = MIN(rxq_allocated, limits.edl_max_rxq_count);
238 txq_allocated = MIN(txq_allocated, limits.edl_max_txq_count);
240 /* Subtract management EVQ not used for traffic */
241 SFC_ASSERT(evq_allocated > 0);
244 /* Right now we use separate EVQ for Rx and Tx */
245 sa->rxq_max = MIN(rxq_allocated, evq_allocated / 2);
246 sa->txq_max = MIN(txq_allocated, evq_allocated - sa->rxq_max);
248 /* Keep NIC initialized */
253 efx_nic_fini(sa->nic);
258 sfc_set_drv_limits(struct sfc_adapter *sa)
260 const struct rte_eth_dev_data *data = sa->eth_dev->data;
261 efx_drv_limits_t lim;
263 memset(&lim, 0, sizeof(lim));
265 /* Limits are strict since take into account initial estimation */
266 lim.edl_min_evq_count = lim.edl_max_evq_count =
267 1 + data->nb_rx_queues + data->nb_tx_queues;
268 lim.edl_min_rxq_count = lim.edl_max_rxq_count = data->nb_rx_queues;
269 lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues;
271 return efx_nic_set_drv_limits(sa->nic, &lim);
275 sfc_start(struct sfc_adapter *sa)
279 sfc_log_init(sa, "entry");
281 SFC_ASSERT(sfc_adapter_is_locked(sa));
284 case SFC_ADAPTER_CONFIGURED:
286 case SFC_ADAPTER_STARTED:
287 sfc_info(sa, "already started");
294 sa->state = SFC_ADAPTER_STARTING;
296 sfc_log_init(sa, "set resource limits");
297 rc = sfc_set_drv_limits(sa);
299 goto fail_set_drv_limits;
301 sfc_log_init(sa, "init nic");
302 rc = efx_nic_init(sa->nic);
306 rc = sfc_intr_start(sa);
308 goto fail_intr_start;
310 rc = sfc_ev_start(sa);
314 rc = sfc_port_start(sa);
316 goto fail_port_start;
318 rc = sfc_rx_start(sa);
322 rc = sfc_tx_start(sa);
326 rc = sfc_flow_start(sa);
328 goto fail_flows_insert;
330 sa->state = SFC_ADAPTER_STARTED;
331 sfc_log_init(sa, "done");
350 efx_nic_fini(sa->nic);
354 sa->state = SFC_ADAPTER_CONFIGURED;
356 sfc_log_init(sa, "failed %d", rc);
361 sfc_stop(struct sfc_adapter *sa)
363 sfc_log_init(sa, "entry");
365 SFC_ASSERT(sfc_adapter_is_locked(sa));
368 case SFC_ADAPTER_STARTED:
370 case SFC_ADAPTER_CONFIGURED:
371 sfc_info(sa, "already stopped");
374 sfc_err(sa, "stop in unexpected state %u", sa->state);
379 sa->state = SFC_ADAPTER_STOPPING;
387 efx_nic_fini(sa->nic);
389 sa->state = SFC_ADAPTER_CONFIGURED;
390 sfc_log_init(sa, "done");
394 sfc_restart(struct sfc_adapter *sa)
398 SFC_ASSERT(sfc_adapter_is_locked(sa));
400 if (sa->state != SFC_ADAPTER_STARTED)
407 sfc_err(sa, "restart failed");
413 sfc_restart_if_required(void *arg)
415 struct sfc_adapter *sa = arg;
417 /* If restart is scheduled, clear the flag and do it */
418 if (rte_atomic32_cmpset((volatile uint32_t *)&sa->restart_required,
420 sfc_adapter_lock(sa);
421 if (sa->state == SFC_ADAPTER_STARTED)
422 (void)sfc_restart(sa);
423 sfc_adapter_unlock(sa);
428 sfc_schedule_restart(struct sfc_adapter *sa)
432 /* Schedule restart alarm if it is not scheduled yet */
433 if (!rte_atomic32_test_and_set(&sa->restart_required))
436 rc = rte_eal_alarm_set(1, sfc_restart_if_required, sa);
438 sfc_warn(sa, "alarms are not supported, restart is pending");
440 sfc_err(sa, "cannot arm restart alarm (rc=%d)", rc);
442 sfc_info(sa, "restart scheduled");
446 sfc_configure(struct sfc_adapter *sa)
450 sfc_log_init(sa, "entry");
452 SFC_ASSERT(sfc_adapter_is_locked(sa));
454 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED ||
455 sa->state == SFC_ADAPTER_CONFIGURED);
456 sa->state = SFC_ADAPTER_CONFIGURING;
458 rc = sfc_check_conf(sa);
460 goto fail_check_conf;
462 rc = sfc_intr_configure(sa);
464 goto fail_intr_configure;
466 rc = sfc_port_configure(sa);
468 goto fail_port_configure;
470 rc = sfc_rx_configure(sa);
472 goto fail_rx_configure;
474 rc = sfc_tx_configure(sa);
476 goto fail_tx_configure;
478 sa->state = SFC_ADAPTER_CONFIGURED;
479 sfc_log_init(sa, "done");
493 sa->state = SFC_ADAPTER_INITIALIZED;
494 sfc_log_init(sa, "failed %d", rc);
499 sfc_close(struct sfc_adapter *sa)
501 sfc_log_init(sa, "entry");
503 SFC_ASSERT(sfc_adapter_is_locked(sa));
505 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED);
506 sa->state = SFC_ADAPTER_CLOSING;
513 sa->state = SFC_ADAPTER_INITIALIZED;
514 sfc_log_init(sa, "done");
518 sfc_mem_bar_init(struct sfc_adapter *sa)
520 struct rte_eth_dev *eth_dev = sa->eth_dev;
521 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
522 efsys_bar_t *ebp = &sa->mem_bar;
524 struct rte_mem_resource *res;
526 for (i = 0; i < RTE_DIM(pci_dev->mem_resource); i++) {
527 res = &pci_dev->mem_resource[i];
528 if ((res->len != 0) && (res->phys_addr != 0)) {
529 /* Found first memory BAR */
530 SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name);
532 ebp->esb_dev = pci_dev;
533 ebp->esb_base = res->addr;
542 sfc_mem_bar_fini(struct sfc_adapter *sa)
544 efsys_bar_t *ebp = &sa->mem_bar;
546 SFC_BAR_LOCK_DESTROY(ebp);
547 memset(ebp, 0, sizeof(*ebp));
550 #if EFSYS_OPT_RX_SCALE
552 * A fixed RSS key which has a property of being symmetric
553 * (symmetrical flows are distributed to the same CPU)
554 * and also known to give a uniform distribution
555 * (a good distribution of traffic between different CPUs)
557 static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = {
558 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
559 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
560 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
561 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
562 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
566 #if EFSYS_OPT_RX_SCALE
568 sfc_set_rss_defaults(struct sfc_adapter *sa)
572 rc = efx_intr_init(sa->nic, sa->intr.type, NULL);
576 rc = efx_ev_init(sa->nic);
580 rc = efx_rx_init(sa->nic);
584 rc = efx_rx_scale_default_support_get(sa->nic, &sa->rss_support);
586 goto fail_scale_support_get;
588 rc = efx_rx_hash_default_support_get(sa->nic, &sa->hash_support);
590 goto fail_hash_support_get;
592 efx_rx_fini(sa->nic);
593 efx_ev_fini(sa->nic);
594 efx_intr_fini(sa->nic);
596 sa->rss_hash_types = sfc_rte_to_efx_hash_type(SFC_RSS_OFFLOADS);
598 rte_memcpy(sa->rss_key, default_rss_key, sizeof(sa->rss_key));
602 fail_hash_support_get:
603 fail_scale_support_get:
605 efx_ev_fini(sa->nic);
608 efx_intr_fini(sa->nic);
615 sfc_set_rss_defaults(__rte_unused struct sfc_adapter *sa)
622 sfc_attach(struct sfc_adapter *sa)
624 const efx_nic_cfg_t *encp;
625 efx_nic_t *enp = sa->nic;
628 sfc_log_init(sa, "entry");
630 SFC_ASSERT(sfc_adapter_is_locked(sa));
632 efx_mcdi_new_epoch(enp);
634 sfc_log_init(sa, "reset nic");
635 rc = efx_nic_reset(enp);
639 encp = efx_nic_cfg_get(sa->nic);
641 if (sa->dp_tx->features & SFC_DP_TX_FEAT_TSO) {
642 sa->tso = encp->enc_fw_assisted_tso_v2_enabled;
645 "TSO support isn't available on this adapter");
648 sfc_log_init(sa, "estimate resource limits");
649 rc = sfc_estimate_resource_limits(sa);
651 goto fail_estimate_rsrc_limits;
653 sa->txq_max_entries = encp->enc_txq_max_ndescs;
654 SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries));
656 rc = sfc_intr_attach(sa);
658 goto fail_intr_attach;
660 rc = sfc_ev_attach(sa);
664 rc = sfc_port_attach(sa);
666 goto fail_port_attach;
668 rc = sfc_set_rss_defaults(sa);
670 goto fail_set_rss_defaults;
672 rc = sfc_filter_attach(sa);
674 goto fail_filter_attach;
676 sfc_log_init(sa, "fini nic");
681 sa->state = SFC_ADAPTER_INITIALIZED;
683 sfc_log_init(sa, "done");
687 fail_set_rss_defaults:
697 efx_nic_fini(sa->nic);
699 fail_estimate_rsrc_limits:
702 sfc_log_init(sa, "failed %d", rc);
707 sfc_detach(struct sfc_adapter *sa)
709 sfc_log_init(sa, "entry");
711 SFC_ASSERT(sfc_adapter_is_locked(sa));
715 sfc_filter_detach(sa);
720 sa->state = SFC_ADAPTER_UNINITIALIZED;
724 sfc_probe(struct sfc_adapter *sa)
726 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev);
730 sfc_log_init(sa, "entry");
732 SFC_ASSERT(sfc_adapter_is_locked(sa));
734 sa->socket_id = rte_socket_id();
735 rte_atomic32_init(&sa->restart_required);
737 sfc_log_init(sa, "init mem bar");
738 rc = sfc_mem_bar_init(sa);
740 goto fail_mem_bar_init;
742 sfc_log_init(sa, "get family");
743 rc = efx_family(pci_dev->id.vendor_id, pci_dev->id.device_id,
747 sfc_log_init(sa, "family is %u", sa->family);
749 sfc_log_init(sa, "create nic");
750 rte_spinlock_init(&sa->nic_lock);
751 rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa,
752 &sa->mem_bar, &sa->nic_lock, &enp);
754 goto fail_nic_create;
757 rc = sfc_mcdi_init(sa);
761 sfc_log_init(sa, "probe nic");
762 rc = efx_nic_probe(enp);
766 sfc_log_init(sa, "done");
773 sfc_log_init(sa, "destroy nic");
775 efx_nic_destroy(enp);
779 sfc_mem_bar_fini(sa);
782 sfc_log_init(sa, "failed %d", rc);
787 sfc_unprobe(struct sfc_adapter *sa)
789 efx_nic_t *enp = sa->nic;
791 sfc_log_init(sa, "entry");
793 SFC_ASSERT(sfc_adapter_is_locked(sa));
795 sfc_log_init(sa, "unprobe nic");
796 efx_nic_unprobe(enp);
801 * Make sure there is no pending alarm to restart since we are
802 * going to free device private which is passed as the callback
803 * opaque data. A new alarm cannot be scheduled since MCDI is
806 rte_eal_alarm_cancel(sfc_restart_if_required, sa);
808 sfc_log_init(sa, "destroy nic");
810 efx_nic_destroy(enp);
812 sfc_mem_bar_fini(sa);
815 sa->state = SFC_ADAPTER_UNINITIALIZED;