net/cxgbe: add probe to initialize VF devices
authorKumar Sanghvi <kumaras@chelsio.com>
Sat, 10 Mar 2018 22:48:22 +0000 (04:18 +0530)
committerFerruh Yigit <ferruh.yigit@intel.com>
Fri, 30 Mar 2018 12:08:44 +0000 (14:08 +0200)
Add probe to initialize VF devices.  Separate init/de-init paths
for PF and VF.  Do firmware state initialization wrt VF and retrieve
various operational parameters by querying firmware.  Finally configure
and initialize ports.

Signed-off-by: Kumar Sanghvi <kumaras@chelsio.com>
Signed-off-by: Rahul Lakkireddy <rahul.lakkireddy@chelsio.com>
drivers/net/cxgbe/Makefile
drivers/net/cxgbe/base/adapter.h
drivers/net/cxgbe/base/t4_hw.c
drivers/net/cxgbe/cxgbe.h
drivers/net/cxgbe/cxgbe_main.c
drivers/net/cxgbe/cxgbevf_ethdev.c
drivers/net/cxgbe/cxgbevf_main.c [new file with mode: 0644]

index 66c445e..fe177ff 100644 (file)
@@ -82,6 +82,7 @@ VPATH += $(SRCDIR)/base
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbevf_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbe_main.c
+SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += cxgbevf_main.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += sge.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4_hw.c
 SRCS-$(CONFIG_RTE_LIBRTE_CXGBE_PMD) += t4vf_hw.c
index 1694028..6b2fc8b 100644 (file)
@@ -68,6 +68,7 @@ struct port_info {
        u8     port_type;               /* firmware port type */
        u8     mod_type;                /* firmware module type */
        u8     port_id;                 /* physical port ID */
+       u8     pidx;                    /* port index for this PF */
        u8     tx_chan;                 /* associated channel */
 
        u8     n_rx_qsets;              /* # of rx qsets */
index 436a995..b393894 100644 (file)
@@ -4488,7 +4488,7 @@ static void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl)
                lc->auto_fec = fec;
                pi->port_type = port_type;
                pi->mod_type = mod_type;
-               t4_os_portmod_changed(adapter, pi->port_id);
+               t4_os_portmod_changed(adapter, pi->pidx);
        }
        if (link_ok != lc->link_ok || speed != lc->speed ||
            fc != lc->fc || fec != lc->fec) { /* something changed */
index 555ef26..f8904ad 100644 (file)
@@ -54,6 +54,7 @@
                          ETH_RSS_NONFRAG_IPV6_UDP)
 
 int cxgbe_probe(struct adapter *adapter);
+int cxgbevf_probe(struct adapter *adapter);
 void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps);
 int cxgbe_up(struct adapter *adap);
 int cxgbe_down(struct port_info *pi);
@@ -68,5 +69,7 @@ void cfg_queues(struct rte_eth_dev *eth_dev);
 int cfg_queue_count(struct rte_eth_dev *eth_dev);
 int setup_rss(struct port_info *pi);
 void cxgbe_enable_rx_queues(struct port_info *pi);
+void print_port_info(struct adapter *adap);
+void print_adapter_info(struct adapter *adap);
 
 #endif /* _CXGBE_H_ */
index c126e42..feb494b 100644 (file)
@@ -370,7 +370,7 @@ static int init_rss(struct adapter *adap)
 /**
  * Dump basic information about the adapter.
  */
-static void print_adapter_info(struct adapter *adap)
+void print_adapter_info(struct adapter *adap)
 {
        /**
         * Hardware/Firmware/etc. Version/Revision IDs.
@@ -378,7 +378,7 @@ static void print_adapter_info(struct adapter *adap)
        t4_dump_version_info(adap);
 }
 
-static void print_port_info(struct adapter *adap)
+void print_port_info(struct adapter *adap)
 {
        int i;
        char buf[80];
@@ -917,7 +917,7 @@ int link_start(struct port_info *pi)
                        ret = 0;
                }
        }
-       if (ret == 0)
+       if (ret == 0 && is_pf4(adapter))
                ret = t4_link_l1cfg(adapter, adapter->mbox, pi->tx_chan,
                                    &pi->link_cfg);
        if (ret == 0) {
@@ -1200,7 +1200,8 @@ int cxgbe_up(struct adapter *adap)
 {
        enable_rx(adap, &adap->sge.fw_evtq);
        t4_sge_tx_monitor_start(adap);
-       t4_intr_enable(adap);
+       if (is_pf4(adap))
+               t4_intr_enable(adap);
        adap->flags |= FULL_INIT_DONE;
 
        /* TODO: deadman watchdog ?? */
@@ -1221,7 +1222,7 @@ int cxgbe_down(struct port_info *pi)
                return err;
        }
 
-       t4_reset_link_config(adapter, pi->port_id);
+       t4_reset_link_config(adapter, pi->pidx);
        return 0;
 }
 
@@ -1234,7 +1235,8 @@ void cxgbe_close(struct adapter *adapter)
        int i;
 
        if (adapter->flags & FULL_INIT_DONE) {
-               t4_intr_disable(adapter);
+               if (is_pf4(adapter))
+                       t4_intr_disable(adapter);
                t4_sge_tx_monitor_stop(adapter);
                t4_free_sge_resources(adapter);
                for_each_port(adapter, i) {
@@ -1252,7 +1254,7 @@ void cxgbe_close(struct adapter *adapter)
                adapter->flags &= ~FULL_INIT_DONE;
        }
 
-       if (adapter->flags & FW_OK)
+       if (is_pf4(adapter) && (adapter->flags & FW_OK))
                t4_fw_bye(adapter, adapter->mbox);
 }
 
@@ -1359,6 +1361,7 @@ allocate_mac:
                pi->adapter = adapter;
                pi->xact_addr_filt = -1;
                pi->port_id = i;
+               pi->pidx = i;
 
                pi->eth_dev->device = &adapter->pdev->device;
                pi->eth_dev->dev_ops = adapter->eth_dev->dev_ops;
index 7d5b8c0..3b1deac 100644 (file)
@@ -59,12 +59,73 @@ static const struct eth_dev_ops cxgbevf_eth_dev_ops = {
  */
 static int eth_cxgbevf_dev_init(struct rte_eth_dev *eth_dev)
 {
+       struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
+       struct rte_pci_device *pci_dev;
+       char name[RTE_ETH_NAME_MAX_LEN];
+       struct adapter *adapter = NULL;
+       int err = 0;
+
        CXGBE_FUNC_TRACE();
 
        eth_dev->dev_ops = &cxgbevf_eth_dev_ops;
+       eth_dev->rx_pkt_burst = NULL;
+       eth_dev->tx_pkt_burst = NULL;
+       pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+
+       /* for secondary processes, we attach to ethdevs allocated by primary
+        * and do minimal initialization.
+        */
+       if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+               int i;
+
+               for (i = 1; i < MAX_NPORTS; i++) {
+                       struct rte_eth_dev *rest_eth_dev;
+                       char namei[RTE_ETH_NAME_MAX_LEN];
+
+                       snprintf(namei, sizeof(namei), "%s_%d",
+                                pci_dev->device.name, i);
+                       rest_eth_dev = rte_eth_dev_attach_secondary(namei);
+                       if (rest_eth_dev) {
+                               rest_eth_dev->device = &pci_dev->device;
+                               rest_eth_dev->dev_ops =
+                                       eth_dev->dev_ops;
+                               rest_eth_dev->rx_pkt_burst =
+                                       eth_dev->rx_pkt_burst;
+                               rest_eth_dev->tx_pkt_burst =
+                                       eth_dev->tx_pkt_burst;
+                       }
+               }
+               return 0;
+       }
+
+       snprintf(name, sizeof(name), "cxgbevfadapter%d",
+                eth_dev->data->port_id);
+       adapter = rte_zmalloc(name, sizeof(*adapter), 0);
+       if (!adapter)
+               return -1;
+
+       adapter->use_unpacked_mode = 1;
+       adapter->regs = (void *)pci_dev->mem_resource[0].addr;
+       if (!adapter->regs) {
+               dev_err(adapter, "%s: cannot map device registers\n", __func__);
+               err = -ENOMEM;
+               goto out_free_adapter;
+       }
+       adapter->pdev = pci_dev;
+       adapter->eth_dev = eth_dev;
+       pi->adapter = adapter;
+       err = cxgbevf_probe(adapter);
+       if (err) {
+               dev_err(adapter, "%s: cxgbevf probe failed with err %d\n",
+                       __func__, err);
+               goto out_free_adapter;
+       }
+
+       return 0;
 
-       /* XXX: Do probe */
-       return -EIO;
+out_free_adapter:
+       rte_free(adapter);
+       return err;
 }
 
 static int eth_cxgbevf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
diff --git a/drivers/net/cxgbe/cxgbevf_main.c b/drivers/net/cxgbe/cxgbevf_main.c
new file mode 100644 (file)
index 0000000..0624267
--- /dev/null
@@ -0,0 +1,280 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Chelsio Communications.
+ * All rights reserved.
+ */
+
+#include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
+#include <rte_malloc.h>
+
+#include "common.h"
+#include "t4_regs.h"
+#include "t4_msg.h"
+#include "cxgbe.h"
+
+/*
+ * Figure out how many Ports and Queue Sets we can support.  This depends on
+ * knowing our Virtual Function Resources and may be called a second time if
+ * we fall back from MSI-X to MSI Interrupt Mode.
+ */
+static void size_nports_qsets(struct adapter *adapter)
+{
+       struct vf_resources *vfres = &adapter->params.vfres;
+       unsigned int ethqsets, pmask_nports;
+
+       /*
+        * The number of "ports" which we support is equal to the number of
+        * Virtual Interfaces with which we've been provisioned.
+        */
+       adapter->params.nports = vfres->nvi;
+       if (adapter->params.nports > MAX_NPORTS) {
+               dev_warn(adapter->pdev_dev, "only using %d of %d maximum"
+                        " allowed virtual interfaces\n", MAX_NPORTS,
+                        adapter->params.nports);
+               adapter->params.nports = MAX_NPORTS;
+       }
+
+       /*
+        * We may have been provisioned with more VIs than the number of
+        * ports we're allowed to access (our Port Access Rights Mask).
+        * This is obviously a configuration conflict but we don't want to
+        * do anything silly just because of that.
+        */
+       pmask_nports = hweight32(adapter->params.vfres.pmask);
+       if (pmask_nports < adapter->params.nports) {
+               dev_warn(adapter->pdev_dev, "only using %d of %d provissioned"
+                        " virtual interfaces; limited by Port Access Rights"
+                        " mask %#x\n", pmask_nports, adapter->params.nports,
+                        adapter->params.vfres.pmask);
+               adapter->params.nports = pmask_nports;
+       }
+
+       /*
+        * We need to reserve an Ingress Queue for the Asynchronous Firmware
+        * Event Queue.
+        *
+        * For each Queue Set, we'll need the ability to allocate two Egress
+        * Contexts -- one for the Ingress Queue Free List and one for the TX
+        * Ethernet Queue.
+        */
+       ethqsets = vfres->niqflint - 1;
+       if (vfres->nethctrl != ethqsets)
+               ethqsets = min(vfres->nethctrl, ethqsets);
+       if (vfres->neq < ethqsets * 2)
+               ethqsets = vfres->neq / 2;
+       if (ethqsets > MAX_ETH_QSETS)
+               ethqsets = MAX_ETH_QSETS;
+       adapter->sge.max_ethqsets = ethqsets;
+
+       if (adapter->sge.max_ethqsets < adapter->params.nports) {
+               dev_warn(adapter->pdev_dev, "only using %d of %d available"
+                        " virtual interfaces (too few Queue Sets)\n",
+                        adapter->sge.max_ethqsets, adapter->params.nports);
+               adapter->params.nports = adapter->sge.max_ethqsets;
+       }
+}
+
+static int adap_init0vf(struct adapter *adapter)
+{
+       u32 param, val = 0;
+       int err;
+
+       err = t4vf_fw_reset(adapter);
+       if (err < 0) {
+               dev_err(adapter->pdev_dev, "FW reset failed: err=%d\n", err);
+               return err;
+       }
+
+       /*
+        * Grab basic operational parameters.  These will predominantly have
+        * been set up by the Physical Function Driver or will be hard coded
+        * into the adapter.  We just have to live with them ...  Note that
+        * we _must_ get our VPD parameters before our SGE parameters because
+        * we need to know the adapter's core clock from the VPD in order to
+        * properly decode the SGE Timer Values.
+        */
+       err = t4vf_get_dev_params(adapter);
+       if (err) {
+               dev_err(adapter->pdev_dev, "unable to retrieve adapter"
+                       " device parameters: err=%d\n", err);
+               return err;
+       }
+
+       err = t4vf_get_vpd_params(adapter);
+       if (err) {
+               dev_err(adapter->pdev_dev, "unable to retrieve adapter"
+                       " VPD parameters: err=%d\n", err);
+               return err;
+       }
+
+       adapter->pf = t4vf_get_pf_from_vf(adapter);
+
+       /* If we're running on newer firmware, let it know that we're
+        * prepared to deal with encapsulated CPL messages.  Older
+        * firmware won't understand this and we'll just get
+        * unencapsulated messages ...
+        */
+       param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) |
+               V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP);
+       val = 1;
+       t4vf_set_params(adapter, 1, &param, &val);
+
+       /*
+        * Grab our Virtual Interface resource allocation, extract the
+        * features that we're interested in and do a bit of sanity testing on
+        * what we discover.
+        */
+       err = t4vf_get_vfres(adapter);
+       if (err) {
+               dev_err(adapter->pdev_dev, "unable to get virtual interface"
+                       " resources: err=%d\n", err);
+               return err;
+       }
+
+       /*
+        * Check for various parameter sanity issues.
+        */
+       if (adapter->params.vfres.pmask == 0) {
+               dev_err(adapter->pdev_dev, "no port access configured\n"
+                       "usable!\n");
+               return -EINVAL;
+       }
+       if (adapter->params.vfres.nvi == 0) {
+               dev_err(adapter->pdev_dev, "no virtual interfaces configured/"
+                       "usable!\n");
+               return -EINVAL;
+       }
+
+       /*
+        * Initialize nports and max_ethqsets now that we have our Virtual
+        * Function Resources.
+        */
+       size_nports_qsets(adapter);
+       adapter->flags |= FW_OK;
+       return 0;
+}
+
+int cxgbevf_probe(struct adapter *adapter)
+{
+       struct port_info *pi;
+       unsigned int pmask;
+       int err = 0;
+       int i;
+
+       t4_os_lock_init(&adapter->mbox_lock);
+       TAILQ_INIT(&adapter->mbox_list);
+       err = t4vf_prep_adapter(adapter);
+       if (err)
+               return err;
+
+       if (!is_t4(adapter->params.chip)) {
+               adapter->bar2 = (void *)adapter->pdev->mem_resource[2].addr;
+               if (!adapter->bar2) {
+                       dev_err(adapter, "cannot map device bar2 region\n");
+                       err = -ENOMEM;
+                       return err;
+               }
+       }
+
+       err = adap_init0vf(adapter);
+       if (err) {
+               dev_err(adapter, "%s: Adapter initialization failed, error %d\n",
+                               __func__, err);
+               goto out_free;
+       }
+
+       pmask = adapter->params.vfres.pmask;
+       for_each_port(adapter, i) {
+               const unsigned int numa_node = rte_socket_id();
+               char name[RTE_ETH_NAME_MAX_LEN];
+               struct rte_eth_dev *eth_dev;
+               int port_id;
+
+               if (pmask == 0)
+                       break;
+               port_id = ffs(pmask) - 1;
+               pmask &= ~(1 << port_id);
+
+               snprintf(name, sizeof(name), "%s_%d",
+                        adapter->pdev->device.name, i);
+
+               if (i == 0) {
+                       /* First port is already allocated by DPDK */
+                       eth_dev = adapter->eth_dev;
+                       goto allocate_mac;
+               }
+
+               /*
+                * now do all data allocation - for eth_dev structure,
+                * and internal (private) data for the remaining ports
+                */
+
+               /* reserve an ethdev entry */
+               eth_dev = rte_eth_dev_allocate(name);
+               if (!eth_dev) {
+                       err = -ENOMEM;
+                       goto out_free;
+               }
+               eth_dev->data->dev_private =
+                       rte_zmalloc_socket(name, sizeof(struct port_info),
+                                          RTE_CACHE_LINE_SIZE, numa_node);
+               if (!eth_dev->data->dev_private)
+                       goto out_free;
+
+allocate_mac:
+               pi = (struct port_info *)eth_dev->data->dev_private;
+               adapter->port[i] = pi;
+               pi->eth_dev = eth_dev;
+               pi->adapter = adapter;
+               pi->xact_addr_filt = -1;
+               pi->port_id = port_id;
+               pi->pidx = i;
+
+               pi->eth_dev->device = &adapter->pdev->device;
+               pi->eth_dev->dev_ops = adapter->eth_dev->dev_ops;
+               pi->eth_dev->tx_pkt_burst = adapter->eth_dev->tx_pkt_burst;
+               pi->eth_dev->rx_pkt_burst = adapter->eth_dev->rx_pkt_burst;
+
+               rte_eth_copy_pci_info(pi->eth_dev, adapter->pdev);
+               pi->eth_dev->data->mac_addrs = rte_zmalloc(name,
+                                                          ETHER_ADDR_LEN, 0);
+               if (!pi->eth_dev->data->mac_addrs) {
+                       dev_err(adapter, "%s: Mem allocation failed for storing mac addr, aborting\n",
+                               __func__);
+                       err = -ENOMEM;
+                       goto out_free;
+               }
+       }
+
+       if (adapter->flags & FW_OK) {
+               err = t4vf_port_init(adapter);
+               if (err) {
+                       dev_err(adapter, "%s: t4_port_init failed with err %d\n",
+                               __func__, err);
+                       goto out_free;
+               }
+       }
+
+       cfg_queues(adapter->eth_dev);
+       print_adapter_info(adapter);
+       print_port_info(adapter);
+
+       return 0;
+
+out_free:
+       for_each_port(adapter, i) {
+               pi = adap2pinfo(adapter, i);
+               if (pi->viid != 0)
+                       t4_free_vi(adapter, adapter->mbox, adapter->pf,
+                                  0, pi->viid);
+               /* Skip first port since it'll be de-allocated by DPDK */
+               if (i == 0)
+                       continue;
+               if (pi->eth_dev) {
+                       if (pi->eth_dev->data->dev_private)
+                               rte_free(pi->eth_dev->data->dev_private);
+                       rte_eth_dev_release_port(pi->eth_dev);
+               }
+       }
+       return -err;
+}