From: Ivan Malov Date: Tue, 20 Oct 2020 09:13:23 +0000 (+0100) Subject: net/sfc: support concept of switch domains/ports X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=1e7fbdf0ba19;p=dpdk.git net/sfc: support concept of switch domains/ports A later patch will add support for RTE flow action PORT_ID to MAE backend. The driver has to ensure that such actions refer to RTE ethdev instances deployed on top of the same physical device. Also, the driver needs a means to find sibling RTE ethdev instances when parsing such actions. In order to solve these problems, add a switch infrastructure which allocates switch domains based on persistence of device serial number string across switch ports included in a domain. Explain mapping between RTE switch port IDs and MAE endpoints. Signed-off-by: Ivan Malov Signed-off-by: Andrew Rybchenko Reviewed-by: Andy Moreton --- diff --git a/drivers/net/sfc/meson.build b/drivers/net/sfc/meson.build index 7a893080cb..42b184c29e 100644 --- a/drivers/net/sfc/meson.build +++ b/drivers/net/sfc/meson.build @@ -47,6 +47,7 @@ sources = files( 'sfc_tx.c', 'sfc_tso.c', 'sfc_filter.c', + 'sfc_switch.c', 'sfc_mae.c', 'sfc_flow.c', 'sfc_dp.c', diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c index c0672083ec..107dd0f470 100644 --- a/drivers/net/sfc/sfc_ethdev.c +++ b/drivers/net/sfc/sfc_ethdev.c @@ -93,6 +93,7 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev); struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev); struct sfc_rss *rss = &sas->rss; + struct sfc_mae *mae = &sa->mae; uint64_t txq_offloads_def = 0; sfc_log_init(sa, "entry"); @@ -187,6 +188,12 @@ sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; + if (mae->status == SFC_MAE_STATUS_SUPPORTED) { + dev_info->switch_info.name = dev->device->driver->name; + dev_info->switch_info.domain_id = mae->switch_domain_id; + dev_info->switch_info.port_id = mae->switch_port_id; + } + return 0; } diff --git a/drivers/net/sfc/sfc_mae.c b/drivers/net/sfc/sfc_mae.c index a5800ae722..64cd6b0e9b 100644 --- a/drivers/net/sfc/sfc_mae.c +++ b/drivers/net/sfc/sfc_mae.c @@ -15,11 +15,24 @@ #include "sfc.h" #include "sfc_log.h" +#include "sfc_switch.h" + +static int +sfc_mae_assign_entity_mport(struct sfc_adapter *sa, + efx_mport_sel_t *mportp) +{ + const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + + return efx_mae_mport_by_pcie_function(encp->enc_pf, encp->enc_vf, + mportp); +} int sfc_mae_attach(struct sfc_adapter *sa) { + struct sfc_mae_switch_port_request switch_port_request = {0}; const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); + efx_mport_sel_t entity_mport; struct sfc_mae *mae = &sa->mae; efx_mae_limits_t limits; int rc; @@ -41,6 +54,25 @@ sfc_mae_attach(struct sfc_adapter *sa) if (rc != 0) goto fail_mae_get_limits; + sfc_log_init(sa, "assign entity MPORT"); + rc = sfc_mae_assign_entity_mport(sa, &entity_mport); + if (rc != 0) + goto fail_mae_assign_entity_mport; + + sfc_log_init(sa, "assign RTE switch domain"); + rc = sfc_mae_assign_switch_domain(sa, &mae->switch_domain_id); + if (rc != 0) + goto fail_mae_assign_switch_domain; + + sfc_log_init(sa, "assign RTE switch port"); + switch_port_request.type = SFC_MAE_SWITCH_PORT_INDEPENDENT; + switch_port_request.entity_mportp = &entity_mport; + rc = sfc_mae_assign_switch_port(mae->switch_domain_id, + &switch_port_request, + &mae->switch_port_id); + if (rc != 0) + goto fail_mae_assign_switch_port; + mae->status = SFC_MAE_STATUS_SUPPORTED; mae->nb_action_rule_prios_max = limits.eml_max_n_action_prios; TAILQ_INIT(&mae->action_sets); @@ -49,6 +81,9 @@ sfc_mae_attach(struct sfc_adapter *sa) return 0; +fail_mae_assign_switch_port: +fail_mae_assign_switch_domain: +fail_mae_assign_entity_mport: fail_mae_get_limits: efx_mae_fini(sa->nic); diff --git a/drivers/net/sfc/sfc_mae.h b/drivers/net/sfc/sfc_mae.h index 3c34d08f88..f92e62dcbe 100644 --- a/drivers/net/sfc/sfc_mae.h +++ b/drivers/net/sfc/sfc_mae.h @@ -12,6 +12,8 @@ #include +#include + #include "efx.h" #ifdef __cplusplus @@ -45,6 +47,10 @@ enum sfc_mae_status { }; struct sfc_mae { + /** Assigned switch domain identifier */ + uint16_t switch_domain_id; + /** Assigned switch port identifier */ + uint16_t switch_port_id; /** NIC support for MAE status */ enum sfc_mae_status status; /** Priority level limit for MAE action rules */ diff --git a/drivers/net/sfc/sfc_switch.c b/drivers/net/sfc/sfc_switch.c new file mode 100644 index 0000000000..395fc40263 --- /dev/null +++ b/drivers/net/sfc/sfc_switch.c @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2020 Xilinx, Inc. + * Copyright(c) 2019 Solarflare Communications Inc. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + */ + +#include + +#include +#include + +#include "efx.h" + +#include "sfc.h" +#include "sfc_log.h" +#include "sfc_switch.h" + +/** + * Switch port registry entry. + * + * Drivers aware of RTE switch domains also have to maintain RTE switch + * port IDs for RTE ethdev instances they operate. These IDs are supposed + * to stand for physical interconnect entities, in example, PCIe functions. + * + * In terms of MAE, a physical interconnect entity can be referred to using + * an MPORT selector, that is, a 32-bit value. RTE switch port IDs, in turn, + * are 16-bit values, so indirect mapping has to be maintained: + * + * +--------------------+ +---------------------------------------+ + * | RTE switch port ID | ------ | MAE switch port entry | + * +--------------------+ | --------------------- | + * | | + * | Entity (PCIe function) MPORT selector | + * | + | + * | Port type (independent/representor) | + * +---------------------------------------+ + * + * This mapping comprises a port type to ensure that RTE switch port ID + * of a represented entity and that of its representor are different in + * the case when the entity gets plugged into DPDK and not into a guest. + */ +struct sfc_mae_switch_port { + TAILQ_ENTRY(sfc_mae_switch_port) switch_domain_ports; + + /** Entity (PCIe function) MPORT selector */ + efx_mport_sel_t entity_mport; + /** Port type (independent/representor) */ + enum sfc_mae_switch_port_type type; + /** RTE switch port ID */ + uint16_t id; +}; + +TAILQ_HEAD(sfc_mae_switch_ports, sfc_mae_switch_port); + +/** + * Switch domain registry entry. + * + * Even if an RTE ethdev instance gets unplugged, the corresponding + * entry in the switch port registry will not be removed because the + * entity (PCIe function) MPORT is static and cannot change. If this + * RTE ethdev gets plugged back, the entry will be reused, and + * RTE switch port ID will be the same. + */ +struct sfc_mae_switch_domain { + TAILQ_ENTRY(sfc_mae_switch_domain) entries; + + /** HW switch ID */ + struct sfc_hw_switch_id *hw_switch_id; + /** The number of ports in the switch port registry */ + unsigned int nb_ports; + /** Switch port registry */ + struct sfc_mae_switch_ports ports; + /** RTE switch domain ID allocated for a group of devices */ + uint16_t id; +}; + +TAILQ_HEAD(sfc_mae_switch_domains, sfc_mae_switch_domain); + +/** + * MAE representation of RTE switch infrastructure. + * + * It is possible that an RTE flow API client tries to insert a rule + * referencing an RTE ethdev deployed on top of a different physical + * device (it may belong to the same vendor or not). This particular + * driver/engine cannot support this and has to turn down such rules. + * + * Technically, it's HW switch identifier which, if queried for each + * RTE ethdev instance, indicates relationship between the instances. + * In the meantime, RTE flow API clients also need to somehow figure + * out relationship between RTE ethdev instances in advance. + * + * The concept of RTE switch domains resolves this issue. The driver + * maintains a static list of switch domains which is easy to browse, + * and each RTE ethdev fills RTE switch parameters in device + * information structure which is made available to clients. + * + * Even if all RTE ethdev instances belonging to a switch domain get + * unplugged, the corresponding entry in the switch domain registry + * will not be removed because the corresponding HW switch exists + * regardless of its ports being plugged to DPDK or kept aside. + * If a port gets plugged back to DPDK, the corresponding + * RTE ethdev will indicate the same RTE switch domain ID. + */ +struct sfc_mae_switch { + /** A lock to protect the whole structure */ + rte_spinlock_t lock; + /** Switch domain registry */ + struct sfc_mae_switch_domains domains; +}; + +static struct sfc_mae_switch sfc_mae_switch = { + .lock = RTE_SPINLOCK_INITIALIZER, + .domains = TAILQ_HEAD_INITIALIZER(sfc_mae_switch.domains), +}; + + +/* This function expects to be called only when the lock is held */ +static struct sfc_mae_switch_domain * +sfc_mae_find_switch_domain_by_id(uint16_t switch_domain_id) +{ + struct sfc_mae_switch_domain *domain; + + SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); + + TAILQ_FOREACH(domain, &sfc_mae_switch.domains, entries) { + if (domain->id == switch_domain_id) + return domain; + } + + return NULL; +} + +/* This function expects to be called only when the lock is held */ +static struct sfc_mae_switch_domain * +sfc_mae_find_switch_domain_by_hw_switch_id(const struct sfc_hw_switch_id *id) +{ + struct sfc_mae_switch_domain *domain; + + SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); + + TAILQ_FOREACH(domain, &sfc_mae_switch.domains, entries) { + if (sfc_hw_switch_ids_equal(domain->hw_switch_id, id)) + return domain; + } + + return NULL; +} + +int +sfc_mae_assign_switch_domain(struct sfc_adapter *sa, + uint16_t *switch_domain_id) +{ + struct sfc_hw_switch_id *hw_switch_id; + struct sfc_mae_switch_domain *domain; + int rc; + + rte_spinlock_lock(&sfc_mae_switch.lock); + + rc = sfc_hw_switch_id_init(sa, &hw_switch_id); + if (rc != 0) + goto fail_hw_switch_id_init; + + domain = sfc_mae_find_switch_domain_by_hw_switch_id(hw_switch_id); + if (domain != NULL) { + sfc_hw_switch_id_fini(sa, hw_switch_id); + goto done; + } + + domain = rte_zmalloc("sfc_mae_switch_domain", sizeof(*domain), 0); + if (domain == NULL) { + rc = ENOMEM; + goto fail_mem_alloc; + } + + /* + * This code belongs to driver init path, that is, negation is + * done at the end of the path by sfc_eth_dev_init(). RTE APIs + * negate error codes, so drop negation here. + */ + rc = -rte_eth_switch_domain_alloc(&domain->id); + if (rc != 0) + goto fail_domain_alloc; + + domain->hw_switch_id = hw_switch_id; + + TAILQ_INIT(&domain->ports); + + TAILQ_INSERT_TAIL(&sfc_mae_switch.domains, domain, entries); + +done: + *switch_domain_id = domain->id; + + rte_spinlock_unlock(&sfc_mae_switch.lock); + + return 0; + +fail_domain_alloc: + rte_free(domain); + +fail_mem_alloc: + sfc_hw_switch_id_fini(sa, hw_switch_id); + rte_spinlock_unlock(&sfc_mae_switch.lock); + +fail_hw_switch_id_init: + return rc; +} + +/* This function expects to be called only when the lock is held */ +static struct sfc_mae_switch_port * +sfc_mae_find_switch_port_by_entity(const struct sfc_mae_switch_domain *domain, + const efx_mport_sel_t *entity_mportp, + enum sfc_mae_switch_port_type type) +{ + struct sfc_mae_switch_port *port; + + SFC_ASSERT(rte_spinlock_is_locked(&sfc_mae_switch.lock)); + + TAILQ_FOREACH(port, &domain->ports, switch_domain_ports) { + if (port->entity_mport.sel == entity_mportp->sel && + port->type == type) + return port; + } + + return NULL; +} + +int +sfc_mae_assign_switch_port(uint16_t switch_domain_id, + const struct sfc_mae_switch_port_request *req, + uint16_t *switch_port_id) +{ + struct sfc_mae_switch_domain *domain; + struct sfc_mae_switch_port *port; + int rc; + + rte_spinlock_lock(&sfc_mae_switch.lock); + + domain = sfc_mae_find_switch_domain_by_id(switch_domain_id); + if (domain == NULL) { + rc = EINVAL; + goto fail_find_switch_domain_by_id; + } + + port = sfc_mae_find_switch_port_by_entity(domain, req->entity_mportp, + req->type); + if (port != NULL) + goto done; + + port = rte_zmalloc("sfc_mae_switch_port", sizeof(*port), 0); + if (port == NULL) { + rc = ENOMEM; + goto fail_mem_alloc; + } + + port->entity_mport.sel = req->entity_mportp->sel; + port->type = req->type; + + port->id = (domain->nb_ports++); + + TAILQ_INSERT_TAIL(&domain->ports, port, switch_domain_ports); + +done: + *switch_port_id = port->id; + + rte_spinlock_unlock(&sfc_mae_switch.lock); + + return 0; + +fail_mem_alloc: +fail_find_switch_domain_by_id: + rte_spinlock_unlock(&sfc_mae_switch.lock); + return rc; +} diff --git a/drivers/net/sfc/sfc_switch.h b/drivers/net/sfc/sfc_switch.h new file mode 100644 index 0000000000..9845ac8801 --- /dev/null +++ b/drivers/net/sfc/sfc_switch.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2020 Xilinx, Inc. + * Copyright(c) 2019 Solarflare Communications Inc. + * + * This software was jointly developed between OKTET Labs (under contract + * for Solarflare) and Solarflare Communications, Inc. + */ + +#ifndef _SFC_SWITCH_H +#define _SFC_SWITCH_H + +#include + +#include "efx.h" + +#include "sfc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** Options for MAE switch port type */ +enum sfc_mae_switch_port_type { + /** + * The switch port is operated by a self-sufficient RTE ethdev + * and thus refers to its underlying PCIe function + */ + SFC_MAE_SWITCH_PORT_INDEPENDENT = 0, +}; + +struct sfc_mae_switch_port_request { + enum sfc_mae_switch_port_type type; + const efx_mport_sel_t *entity_mportp; +}; + +int sfc_mae_assign_switch_domain(struct sfc_adapter *sa, + uint16_t *switch_domain_id); + +int sfc_mae_assign_switch_port(uint16_t switch_domain_id, + const struct sfc_mae_switch_port_request *req, + uint16_t *switch_port_id); + +#ifdef __cplusplus +} +#endif +#endif /* _SFC_SWITCH_H */