From dd45b8805b6607c2c167df508c231b21199ffffb Mon Sep 17 00:00:00 2001 From: Andrew Rybchenko Date: Thu, 24 Sep 2020 14:02:35 +0100 Subject: [PATCH] net/sfc: create virtual switch to enable VFs PF driver is responsible for vSwitch creation and vPorts allocation for VFs. Signed-off-by: Andrew Rybchenko Reviewed-by: Andy Moreton --- doc/guides/nics/sfc_efx.rst | 2 + doc/guides/rel_notes/release_20_11.rst | 6 + drivers/common/sfc_efx/efsys.h | 2 +- .../sfc_efx/rte_common_sfc_efx_version.map | 9 + drivers/net/sfc/meson.build | 1 + drivers/net/sfc/sfc.c | 36 ++++ drivers/net/sfc/sfc.h | 4 +- drivers/net/sfc/sfc_ethdev.c | 2 + drivers/net/sfc/sfc_sriov.c | 168 ++++++++++++++++++ drivers/net/sfc/sfc_sriov.h | 39 ++++ 10 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 drivers/net/sfc/sfc_sriov.c create mode 100644 drivers/net/sfc/sfc_sriov.h diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst index be1c2fe1d6..ab44ce66c8 100644 --- a/doc/guides/nics/sfc_efx.rst +++ b/doc/guides/nics/sfc_efx.rst @@ -71,6 +71,8 @@ SFC EFX PMD has support for: - Loopback +- SR-IOV PF + Non-supported Features ---------------------- diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst index c70a892125..1ae2ac0fcc 100644 --- a/doc/guides/rel_notes/release_20_11.rst +++ b/doc/guides/rel_notes/release_20_11.rst @@ -68,6 +68,12 @@ New Features * Added support for non-zero priorities for group 0 flows * Added support for VXLAN decap combined with VLAN pop +* **Updated Solarflare network PMD.** + + Updated the Solarflare ``sfc_efx`` driver with changes including: + + * Added SR-IOV PF support + * **Extended flow-perf application.** * Started supporting user order instead of bit mask: diff --git a/drivers/common/sfc_efx/efsys.h b/drivers/common/sfc_efx/efsys.h index 9ad7c82b86..8584cd1a40 100644 --- a/drivers/common/sfc_efx/efsys.h +++ b/drivers/common/sfc_efx/efsys.h @@ -159,7 +159,7 @@ prefetch_read_once(const volatile void *addr) #define EFSYS_OPT_FW_SUBVARIANT_AWARE 1 -#define EFSYS_OPT_EVB 0 +#define EFSYS_OPT_EVB 1 #define EFSYS_OPT_MCDI_PROXY_AUTH_SERVER 0 diff --git a/drivers/common/sfc_efx/rte_common_sfc_efx_version.map b/drivers/common/sfc_efx/rte_common_sfc_efx_version.map index 627469a025..fd95fd09e5 100644 --- a/drivers/common/sfc_efx/rte_common_sfc_efx_version.map +++ b/drivers/common/sfc_efx/rte_common_sfc_efx_version.map @@ -15,6 +15,15 @@ INTERNAL { efx_ev_qprime; efx_ev_usecs_to_ticks; + efx_evb_fini; + efx_evb_init; + efx_evb_vport_mac_set; + efx_evb_vport_reset; + efx_evb_vport_stats; + efx_evb_vport_vlan_set; + efx_evb_vswitch_create; + efx_evb_vswitch_destroy; + efx_evq_nbufs; efx_evq_size; diff --git a/drivers/net/sfc/meson.build b/drivers/net/sfc/meson.build index 26f0323738..1c6451938a 100644 --- a/drivers/net/sfc/meson.build +++ b/drivers/net/sfc/meson.build @@ -39,6 +39,7 @@ sources = files( 'sfc_kvargs.c', 'sfc.c', 'sfc_mcdi.c', + 'sfc_sriov.c', 'sfc_intr.c', 'sfc_ev.c', 'sfc_port.c', diff --git a/drivers/net/sfc/sfc.c b/drivers/net/sfc/sfc.c index 615e15af64..4f1fd0c695 100644 --- a/drivers/net/sfc/sfc.c +++ b/drivers/net/sfc/sfc.c @@ -436,7 +436,20 @@ sfc_start(struct sfc_adapter *sa) sa->state = SFC_ADAPTER_STARTING; + rc = 0; do { + /* + * FIXME Try to recreate vSwitch on start retry. + * vSwitch is absent after MC reboot like events and + * we should recreate it. May be we need proper + * indication instead of guessing. + */ + if (rc != 0) { + sfc_sriov_vswitch_destroy(sa); + rc = sfc_sriov_vswitch_create(sa); + if (rc != 0) + goto fail_sriov_vswitch_create; + } rc = sfc_try_start(sa); } while ((--start_tries > 0) && (rc == EIO || rc == EAGAIN || rc == ENOENT || rc == EINVAL)); @@ -449,6 +462,7 @@ sfc_start(struct sfc_adapter *sa) return 0; fail_try_start: +fail_sriov_vswitch_create: sa->state = SFC_ADAPTER_CONFIGURED; fail_bad_state: sfc_log_init(sa, "failed %d", rc); @@ -728,6 +742,10 @@ sfc_attach(struct sfc_adapter *sa) if (rc != 0) goto fail_nic_reset; + rc = sfc_sriov_attach(sa); + if (rc != 0) + goto fail_sriov_attach; + /* * Probed NIC is sufficient for tunnel init. * Initialize tunnel support to be able to use libefx @@ -810,11 +828,24 @@ sfc_attach(struct sfc_adapter *sa) sfc_flow_init(sa); + /* + * Create vSwitch to be able to use VFs when PF is not started yet + * as DPDK port. VFs should be able to talk to each other even + * if PF is down. + */ + rc = sfc_sriov_vswitch_create(sa); + if (rc != 0) + goto fail_sriov_vswitch_create; + sa->state = SFC_ADAPTER_INITIALIZED; sfc_log_init(sa, "done"); return 0; +fail_sriov_vswitch_create: + sfc_flow_fini(sa); + sfc_filter_detach(sa); + fail_filter_attach: sfc_rss_detach(sa); @@ -833,7 +864,9 @@ fail_intr_attach: fail_estimate_rsrc_limits: fail_tunnel_init: efx_tunnel_fini(sa->nic); + sfc_sriov_detach(sa); +fail_sriov_attach: fail_nic_reset: sfc_log_init(sa, "failed %d", rc); @@ -847,6 +880,8 @@ sfc_detach(struct sfc_adapter *sa) SFC_ASSERT(sfc_adapter_is_locked(sa)); + sfc_sriov_vswitch_destroy(sa); + sfc_flow_fini(sa); sfc_filter_detach(sa); @@ -855,6 +890,7 @@ sfc_detach(struct sfc_adapter *sa) sfc_ev_detach(sa); sfc_intr_detach(sa); efx_tunnel_fini(sa->nic); + sfc_sriov_detach(sa); sa->state = SFC_ADAPTER_UNINITIALIZED; } diff --git a/drivers/net/sfc/sfc.h b/drivers/net/sfc/sfc.h index 5ea29362e1..ecdd716256 100644 --- a/drivers/net/sfc/sfc.h +++ b/drivers/net/sfc/sfc.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * +* * Copyright(c) 2019-2020 Xilinx, Inc. * Copyright(c) 2016-2019 Solarflare Communications Inc. * @@ -26,6 +26,7 @@ #include "sfc_debug.h" #include "sfc_log.h" #include "sfc_filter.h" +#include "sfc_sriov.h" #ifdef __cplusplus extern "C" { @@ -226,6 +227,7 @@ struct sfc_adapter { rte_atomic32_t restart_required; struct sfc_efx_mcdi mcdi; + struct sfc_sriov sriov; struct sfc_intr intr; struct sfc_port port; struct sfc_filter filter; diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c index 602351bcc4..f41d0f5fe2 100644 --- a/drivers/net/sfc/sfc_ethdev.c +++ b/drivers/net/sfc/sfc_ethdev.c @@ -102,6 +102,8 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX; + dev_info->max_vfs = sa->sriov.num_vfs; + /* Autonegotiation may be disabled */ dev_info->speed_capa = ETH_LINK_SPEED_FIXED; if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_1000FDX)) diff --git a/drivers/net/sfc/sfc_sriov.c b/drivers/net/sfc/sfc_sriov.c new file mode 100644 index 0000000000..d8db22bdfd --- /dev/null +++ b/drivers/net/sfc/sfc_sriov.c @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2020 Xilinx, Inc. + * Copyright(c) 2019 Solarflare Communications Inc. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + */ + +#include +#include + +#include "sfc.h" +#include "sfc_log.h" + +#include "efx.h" + + +/* + * Check if a MAC address is already assigned to one of previously + * configured vPorts (either PF itself or one of already configured VFs). + * + * Typically the first vPort which corresponds to PF has globally + * administered unicast address, but it still could be locally + * administered if user assigned it or in the case of unconfigured NIC. + * So, it is safer to include it as well in uniqueness check. + */ +static bool +sriov_mac_addr_assigned(const efx_vport_config_t *vport_config, + unsigned int num, const uint8_t *mac_addr) +{ + unsigned int i; + + /* Check PF's MAC address as well as explained above */ + for (i = 0; i < num; ++i) { + if (memcmp(mac_addr, vport_config[i].evc_mac_addr, + sizeof(vport_config[i].evc_mac_addr)) == 0) + return true; + } + + return false; +} + +int +sfc_sriov_attach(struct sfc_adapter *sa) +{ + const struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); + struct sfc_sriov *sriov = &sa->sriov; + efx_vport_config_t *vport_config; + unsigned int i; + int rc; + + sfc_log_init(sa, "entry"); + + sriov->num_vfs = pci_dev->max_vfs; + if (sriov->num_vfs == 0) + goto done; + + vport_config = calloc(sriov->num_vfs + 1, sizeof(*vport_config)); + if (vport_config == NULL) { + rc = ENOMEM; + goto fail_alloc_vport_config; + } + + vport_config[0].evc_function = 0xffff; + vport_config[0].evc_vid = EFX_VF_VID_DEFAULT; + vport_config[0].evc_vlan_restrict = B_FALSE; + + for (i = 1; i <= sriov->num_vfs; ++i) { + vport_config[i].evc_function = i - 1; + vport_config[i].evc_vid = EFX_VF_VID_DEFAULT; + vport_config[i].evc_vlan_restrict = B_FALSE; + do { + rte_eth_random_addr(vport_config[i].evc_mac_addr); + } while (sriov_mac_addr_assigned(vport_config, i, + vport_config[i].evc_mac_addr)); + } + + sriov->vport_config = vport_config; + +done: + sfc_log_init(sa, "done"); + return 0; + +fail_alloc_vport_config: + sriov->num_vfs = 0; + return rc; +} + +void +sfc_sriov_detach(struct sfc_adapter *sa) +{ + struct sfc_sriov *sriov = &sa->sriov; + + sfc_log_init(sa, "entry"); + + free(sriov->vport_config); + sriov->vport_config = NULL; + sriov->num_vfs = 0; + + sfc_log_init(sa, "done"); +} + +int +sfc_sriov_vswitch_create(struct sfc_adapter *sa) +{ + struct sfc_sriov *sriov = &sa->sriov; + efx_vport_config_t *vport_config = sriov->vport_config; + int rc; + + sfc_log_init(sa, "entry"); + + if (sriov->num_vfs == 0) { + sfc_log_init(sa, "no VFs enabled"); + goto done; + } + + rc = efx_evb_init(sa->nic); + if (rc != 0) { + sfc_err(sa, "EVB init failed %d", rc); + goto fail_evb_init; + } + + RTE_BUILD_BUG_ON(sizeof(sa->port.default_mac_addr) != + sizeof(vport_config[0].evc_mac_addr)); + rte_ether_addr_copy(&sa->port.default_mac_addr, + (struct rte_ether_addr *)vport_config[0].evc_mac_addr); + + rc = efx_evb_vswitch_create(sa->nic, sriov->num_vfs + 1, + vport_config, &sriov->vswitch); + if (rc != 0) { + sfc_err(sa, "EVB vSwitch create failed %d", rc); + goto fail_evb_vswitch_create; + } + +done: + sfc_log_init(sa, "done"); + return 0; + +fail_evb_vswitch_create: + efx_evb_fini(sa->nic); + +fail_evb_init: + return rc; +} + +void +sfc_sriov_vswitch_destroy(struct sfc_adapter *sa) +{ + struct sfc_sriov *sriov = &sa->sriov; + int rc; + + sfc_log_init(sa, "entry"); + + if (sriov->num_vfs == 0) + goto done; + + rc = efx_evb_vswitch_destroy(sa->nic, sriov->vswitch); + if (rc != 0) + sfc_err(sa, "efx_evb_vswitch_destroy() failed %d", rc); + + sriov->vswitch = NULL; + + efx_evb_fini(sa->nic); + +done: + sfc_log_init(sa, "done"); +} diff --git a/drivers/net/sfc/sfc_sriov.h b/drivers/net/sfc/sfc_sriov.h new file mode 100644 index 0000000000..2e2e1c69f1 --- /dev/null +++ b/drivers/net/sfc/sfc_sriov.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2020 Xilinx, Inc. + * Copyright(c) 2019 Solarflare Communications Inc. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + */ + +#ifndef _SFC_SRIOV_H +#define _SFC_SRIOV_H + +#include "efx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct sfc_sriov { + /** Number of enabled virtual functions */ + unsigned int num_vfs; + /** PF and VFs vPorts configuration */ + efx_vport_config_t *vport_config; + /** vSwitch handle */ + efx_vswitch_t *vswitch; +}; + +struct sfc_adapter; + +int sfc_sriov_attach(struct sfc_adapter *sa); +void sfc_sriov_detach(struct sfc_adapter *sa); + +int sfc_sriov_vswitch_create(struct sfc_adapter *sa); +void sfc_sriov_vswitch_destroy(struct sfc_adapter *sa); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_SRIOV_H */ -- 2.20.1