drivers/net/nfp/nfp_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2014-2021 Netronome Systems, Inc.
   3  * All rights reserved.
   4  *
   5  * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.
   6  */
   7
   8 /*
   9  * vim:shiftwidth=8:noexpandtab
  10  *
  11  * @file dpdk/pmd/nfp_ethdev.c
  12  *
  13  * Netronome vNIC DPDK Poll-Mode Driver: Main entry point
  14  */
  15
  16 #include <rte_common.h>
  17 #include <ethdev_driver.h>
  18 #include <ethdev_pci.h>
  19 #include <rte_dev.h>
  20 #include <rte_ether.h>
  21 #include <rte_malloc.h>
  22 #include <rte_memzone.h>
  23 #include <rte_mempool.h>
  24 #include <rte_service_component.h>
  25 #include <rte_alarm.h>
  26 #include "eal_firmware.h"
  27
  28 #include "nfpcore/nfp_cpp.h"
  29 #include "nfpcore/nfp_nffw.h"
  30 #include "nfpcore/nfp_hwinfo.h"
  31 #include "nfpcore/nfp_mip.h"
  32 #include "nfpcore/nfp_rtsym.h"
  33 #include "nfpcore/nfp_nsp.h"
  34
  35 #include "nfp_common.h"
  36 #include "nfp_rxtx.h"
  37 #include "nfp_logs.h"
  38 #include "nfp_ctrl.h"
  39 #include "nfp_cpp_bridge.h"
  40
  41
  42 static int nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port);
  43 static int nfp_net_start(struct rte_eth_dev *dev);
  44 static int nfp_net_stop(struct rte_eth_dev *dev);
  45 static int nfp_net_set_link_up(struct rte_eth_dev *dev);
  46 static int nfp_net_set_link_down(struct rte_eth_dev *dev);
  47 static int nfp_net_close(struct rte_eth_dev *dev);
  48 static int nfp_net_init(struct rte_eth_dev *eth_dev);
  49 static int nfp_fw_upload(struct rte_pci_device *dev,
  50                          struct nfp_nsp *nsp, char *card);
  51 static int nfp_fw_setup(struct rte_pci_device *dev,
  52                         struct nfp_cpp *cpp,
  53                         struct nfp_eth_table *nfp_eth_table,
  54                         struct nfp_hwinfo *hwinfo);
  55 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev);
  56 static int nfp_pf_init(struct rte_pci_device *pci_dev);
  57 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev);
  58 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
  59                             struct rte_pci_device *dev);
  60 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev);
  61 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev);
  62
  63 static int
  64 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port)
  65 {
  66         struct nfp_eth_table *nfp_eth_table;
  67         struct nfp_net_hw *hw = NULL;
  68
  69         /* Grab a pointer to the correct physical port */
  70         hw = pf_dev->ports[port];
  71
  72         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
  73
  74         nfp_eth_copy_mac((uint8_t *)&hw->mac_addr,
  75                          (uint8_t *)&nfp_eth_table->ports[port].mac_addr);
  76
  77         free(nfp_eth_table);
  78         return 0;
  79 }
  80
  81 static int
  82 nfp_net_start(struct rte_eth_dev *dev)
  83 {
  84         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
  85         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
  86         uint32_t new_ctrl, update = 0;
  87         struct nfp_net_hw *hw;
  88         struct nfp_pf_dev *pf_dev;
  89         struct rte_eth_conf *dev_conf;
  90         struct rte_eth_rxmode *rxmode;
  91         uint32_t intr_vector;
  92         int ret;
  93
  94         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  95         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
  96
  97         PMD_INIT_LOG(DEBUG, "Start");
  98
  99         /* Disabling queues just in case... */
 100         nfp_net_disable_queues(dev);
 101
 102         /* Enabling the required queues in the device */
 103         nfp_net_enable_queues(dev);
 104
 105         /* check and configure queue intr-vector mapping */
 106         if (dev->data->dev_conf.intr_conf.rxq != 0) {
 107                 if (pf_dev->multiport) {
 108                         PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported "
 109                                           "with NFP multiport PF");
 110                                 return -EINVAL;
 111                 }
 112                 if (rte_intr_type_get(intr_handle) ==
 113                                                 RTE_INTR_HANDLE_UIO) {
 114                         /*
 115                          * Better not to share LSC with RX interrupts.
 116                          * Unregistering LSC interrupt handler
 117                          */
 118                         rte_intr_callback_unregister(pci_dev->intr_handle,
 119                                 nfp_net_dev_interrupt_handler, (void *)dev);
 120
 121                         if (dev->data->nb_rx_queues > 1) {
 122                                 PMD_INIT_LOG(ERR, "PMD rx interrupt only "
 123                                              "supports 1 queue with UIO");
 124                                 return -EIO;
 125                         }
 126                 }
 127                 intr_vector = dev->data->nb_rx_queues;
 128                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 129                         return -1;
 130
 131                 nfp_configure_rx_interrupt(dev, intr_handle);
 132                 update = NFP_NET_CFG_UPDATE_MSIX;
 133         }
 134
 135         rte_intr_enable(intr_handle);
 136
 137         new_ctrl = nfp_check_offloads(dev);
 138
 139         /* Writing configuration parameters in the device */
 140         nfp_net_params_setup(hw);
 141
 142         dev_conf = &dev->data->dev_conf;
 143         rxmode = &dev_conf->rxmode;
 144
 145         if (rxmode->mq_mode & RTE_ETH_MQ_RX_RSS) {
 146                 nfp_net_rss_config_default(dev);
 147                 update |= NFP_NET_CFG_UPDATE_RSS;
 148                 new_ctrl |= NFP_NET_CFG_CTRL_RSS;
 149         }
 150
 151         /* Enable device */
 152         new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
 153
 154         update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING;
 155
 156         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 157                 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;
 158
 159         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl);
 160         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 161                 return -EIO;
 162
 163         /*
 164          * Allocating rte mbufs for configured rx queues.
 165          * This requires queues being enabled before
 166          */
 167         if (nfp_net_rx_freelist_setup(dev) < 0) {
 168                 ret = -ENOMEM;
 169                 goto error;
 170         }
 171
 172         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 173                 /* Configure the physical port up */
 174                 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 175         else
 176                 nfp_eth_set_configured(dev->process_private,
 177                                        hw->nfp_idx, 1);
 178
 179         hw->ctrl = new_ctrl;
 180
 181         return 0;
 182
 183 error:
 184         /*
 185          * An error returned by this function should mean the app
 186          * exiting and then the system releasing all the memory
 187          * allocated even memory coming from hugepages.
 188          *
 189          * The device could be enabled at this point with some queues
 190          * ready for getting packets. This is true if the call to
 191          * nfp_net_rx_freelist_setup() succeeds for some queues but
 192          * fails for subsequent queues.
 193          *
 194          * This should make the app exiting but better if we tell the
 195          * device first.
 196          */
 197         nfp_net_disable_queues(dev);
 198
 199         return ret;
 200 }
 201
 202 /* Stop device: disable rx and tx functions to allow for reconfiguring. */
 203 static int
 204 nfp_net_stop(struct rte_eth_dev *dev)
 205 {
 206         int i;
 207         struct nfp_net_hw *hw;
 208         struct nfp_net_txq *this_tx_q;
 209         struct nfp_net_rxq *this_rx_q;
 210
 211         PMD_INIT_LOG(DEBUG, "Stop");
 212
 213         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 214
 215         nfp_net_disable_queues(dev);
 216
 217         /* Clear queues */
 218         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 219                 this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i];
 220                 nfp_net_reset_tx_queue(this_tx_q);
 221         }
 222
 223         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 224                 this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i];
 225                 nfp_net_reset_rx_queue(this_rx_q);
 226         }
 227
 228         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 229                 /* Configure the physical port down */
 230                 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 231         else
 232                 nfp_eth_set_configured(dev->process_private,
 233                                        hw->nfp_idx, 0);
 234
 235         return 0;
 236 }
 237
 238 /* Set the link up. */
 239 static int
 240 nfp_net_set_link_up(struct rte_eth_dev *dev)
 241 {
 242         struct nfp_net_hw *hw;
 243
 244         PMD_DRV_LOG(DEBUG, "Set link up");
 245
 246         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 247
 248         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 249                 /* Configure the physical port down */
 250                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 251         else
 252                 return nfp_eth_set_configured(dev->process_private,
 253                                               hw->nfp_idx, 1);
 254 }
 255
 256 /* Set the link down. */
 257 static int
 258 nfp_net_set_link_down(struct rte_eth_dev *dev)
 259 {
 260         struct nfp_net_hw *hw;
 261
 262         PMD_DRV_LOG(DEBUG, "Set link down");
 263
 264         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 265
 266         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 267                 /* Configure the physical port down */
 268                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 269         else
 270                 return nfp_eth_set_configured(dev->process_private,
 271                                               hw->nfp_idx, 0);
 272 }
 273
 274 /* Reset and stop device. The device can not be restarted. */
 275 static int
 276 nfp_net_close(struct rte_eth_dev *dev)
 277 {
 278         struct nfp_net_hw *hw;
 279         struct rte_pci_device *pci_dev;
 280         struct nfp_pf_dev *pf_dev;
 281         struct nfp_net_txq *this_tx_q;
 282         struct nfp_net_rxq *this_rx_q;
 283         int i;
 284
 285         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 286                 return 0;
 287
 288         PMD_INIT_LOG(DEBUG, "Close");
 289
 290         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 291         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 292         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 293
 294         /*
 295          * We assume that the DPDK application is stopping all the
 296          * threads/queues before calling the device close function.
 297          */
 298
 299         nfp_net_disable_queues(dev);
 300
 301         /* Clear queues */
 302         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 303                 this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i];
 304                 nfp_net_reset_tx_queue(this_tx_q);
 305         }
 306
 307         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 308                 this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i];
 309                 nfp_net_reset_rx_queue(this_rx_q);
 310         }
 311
 312         /* Cancel possible impending LSC work here before releasing the port*/
 313         rte_eal_alarm_cancel(nfp_net_dev_interrupt_delayed_handler,
 314                              (void *)dev);
 315
 316         /* Only free PF resources after all physical ports have been closed */
 317         /* Mark this port as unused and free device priv resources*/
 318         nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff);
 319         pf_dev->ports[hw->idx] = NULL;
 320         rte_eth_dev_release_port(dev);
 321
 322         for (i = 0; i < pf_dev->total_phyports; i++) {
 323                 /* Check to see if ports are still in use */
 324                 if (pf_dev->ports[i])
 325                         return 0;
 326         }
 327
 328         /* Now it is safe to free all PF resources */
 329         PMD_INIT_LOG(INFO, "Freeing PF resources");
 330         nfp_cpp_area_free(pf_dev->ctrl_area);
 331         nfp_cpp_area_free(pf_dev->hwqueues_area);
 332         free(pf_dev->hwinfo);
 333         free(pf_dev->sym_tbl);
 334         nfp_cpp_free(pf_dev->cpp);
 335         rte_free(pf_dev);
 336
 337         rte_intr_disable(pci_dev->intr_handle);
 338
 339         /* unregister callback func from eal lib */
 340         rte_intr_callback_unregister(pci_dev->intr_handle,
 341                                      nfp_net_dev_interrupt_handler,
 342                                      (void *)dev);
 343
 344         /*
 345          * The ixgbe PMD driver disables the pcie master on the
 346          * device. The i40e does not...
 347          */
 348
 349         return 0;
 350 }
 351
 352 /* Initialise and register driver with DPDK Application */
 353 static const struct eth_dev_ops nfp_net_eth_dev_ops = {
 354         .dev_configure          = nfp_net_configure,
 355         .dev_start              = nfp_net_start,
 356         .dev_stop               = nfp_net_stop,
 357         .dev_set_link_up        = nfp_net_set_link_up,
 358         .dev_set_link_down      = nfp_net_set_link_down,
 359         .dev_close              = nfp_net_close,
 360         .promiscuous_enable     = nfp_net_promisc_enable,
 361         .promiscuous_disable    = nfp_net_promisc_disable,
 362         .link_update            = nfp_net_link_update,
 363         .stats_get              = nfp_net_stats_get,
 364         .stats_reset            = nfp_net_stats_reset,
 365         .dev_infos_get          = nfp_net_infos_get,
 366         .dev_supported_ptypes_get = nfp_net_supported_ptypes_get,
 367         .mtu_set                = nfp_net_dev_mtu_set,
 368         .mac_addr_set           = nfp_set_mac_addr,
 369         .vlan_offload_set       = nfp_net_vlan_offload_set,
 370         .reta_update            = nfp_net_reta_update,
 371         .reta_query             = nfp_net_reta_query,
 372         .rss_hash_update        = nfp_net_rss_hash_update,
 373         .rss_hash_conf_get      = nfp_net_rss_hash_conf_get,
 374         .rx_queue_setup         = nfp_net_rx_queue_setup,
 375         .rx_queue_release       = nfp_net_rx_queue_release,
 376         .tx_queue_setup         = nfp_net_tx_queue_setup,
 377         .tx_queue_release       = nfp_net_tx_queue_release,
 378         .rx_queue_intr_enable   = nfp_rx_queue_intr_enable,
 379         .rx_queue_intr_disable  = nfp_rx_queue_intr_disable,
 380 };
 381
 382 static int
 383 nfp_net_init(struct rte_eth_dev *eth_dev)
 384 {
 385         struct rte_pci_device *pci_dev;
 386         struct nfp_pf_dev *pf_dev;
 387         struct nfp_net_hw *hw;
 388         struct rte_ether_addr *tmp_ether_addr;
 389
 390         uint64_t tx_bar_off = 0, rx_bar_off = 0;
 391         uint32_t start_q;
 392         int stride = 4;
 393         int port = 0;
 394         int err;
 395
 396         PMD_INIT_FUNC_TRACE();
 397
 398         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 399
 400         /* Use backpointer here to the PF of this eth_dev */
 401         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private);
 402
 403         /* NFP can not handle DMA addresses requiring more than 40 bits */
 404         if (rte_mem_check_dma_mask(40)) {
 405                 RTE_LOG(ERR, PMD, "device %s can not be used:",
 406                                    pci_dev->device.name);
 407                 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n");
 408                 return -ENODEV;
 409         };
 410
 411         port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx;
 412         if (port < 0 || port > 7) {
 413                 PMD_DRV_LOG(ERR, "Port value is wrong");
 414                 return -ENODEV;
 415         }
 416
 417         /* Use PF array of physical ports to get pointer to
 418          * this specific port
 419          */
 420         hw = pf_dev->ports[port];
 421
 422         PMD_INIT_LOG(DEBUG, "Working with physical port number: %d, "
 423                             "NFP internal port number: %d",
 424                             port, hw->nfp_idx);
 425
 426         eth_dev->dev_ops = &nfp_net_eth_dev_ops;
 427         eth_dev->rx_queue_count = nfp_net_rx_queue_count;
 428         eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
 429         eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
 430
 431         /* For secondary processes, the primary has done all the work */
 432         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 433                 return 0;
 434
 435         rte_eth_copy_pci_info(eth_dev, pci_dev);
 436
 437         hw->device_id = pci_dev->id.device_id;
 438         hw->vendor_id = pci_dev->id.vendor_id;
 439         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 440         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 441
 442         PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u",
 443                      pci_dev->id.vendor_id, pci_dev->id.device_id,
 444                      pci_dev->addr.domain, pci_dev->addr.bus,
 445                      pci_dev->addr.devid, pci_dev->addr.function);
 446
 447         hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr;
 448         if (hw->ctrl_bar == NULL) {
 449                 PMD_DRV_LOG(ERR,
 450                         "hw->ctrl_bar is NULL. BAR0 not configured");
 451                 return -ENODEV;
 452         }
 453
 454         if (port == 0) {
 455                 hw->ctrl_bar = pf_dev->ctrl_bar;
 456         } else {
 457                 if (!pf_dev->ctrl_bar)
 458                         return -ENODEV;
 459                 /* Use port offset in pf ctrl_bar for this
 460                  * ports control bar
 461                  */
 462                 hw->ctrl_bar = pf_dev->ctrl_bar +
 463                                (port * NFP_PF_CSR_SLICE_SIZE);
 464         }
 465
 466         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar);
 467
 468         hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS);
 469         hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS);
 470
 471         /* Work out where in the BAR the queues start. */
 472         switch (pci_dev->id.device_id) {
 473         case PCI_DEVICE_ID_NFP4000_PF_NIC:
 474         case PCI_DEVICE_ID_NFP6000_PF_NIC:
 475                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
 476                 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
 477                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
 478                 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
 479                 break;
 480         default:
 481                 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
 482                 err = -ENODEV;
 483                 goto dev_err_ctrl_map;
 484         }
 485
 486         PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off);
 487         PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off);
 488
 489         hw->tx_bar = pf_dev->hw_queues + tx_bar_off;
 490         hw->rx_bar = pf_dev->hw_queues + rx_bar_off;
 491         eth_dev->data->dev_private = hw;
 492
 493         PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p",
 494                      hw->ctrl_bar, hw->tx_bar, hw->rx_bar);
 495
 496         nfp_net_cfg_queue_setup(hw);
 497
 498         /* Get some of the read-only fields from the config BAR */
 499         hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION);
 500         hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP);
 501         hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU);
 502         hw->mtu = RTE_ETHER_MTU;
 503
 504         /* VLAN insertion is incompatible with LSOv2 */
 505         if (hw->cap & NFP_NET_CFG_CTRL_LSO2)
 506                 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN;
 507
 508         if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2)
 509                 hw->rx_offset = NFP_NET_RX_OFFSET;
 510         else
 511                 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR);
 512
 513         PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d",
 514                            NFD_CFG_MAJOR_VERSION_of(hw->ver),
 515                            NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu);
 516
 517         PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap,
 518                      hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "",
 519                      hw->cap & NFP_NET_CFG_CTRL_L2BC    ? "L2BCFILT " : "",
 520                      hw->cap & NFP_NET_CFG_CTRL_L2MC    ? "L2MCFILT " : "",
 521                      hw->cap & NFP_NET_CFG_CTRL_RXCSUM  ? "RXCSUM "  : "",
 522                      hw->cap & NFP_NET_CFG_CTRL_TXCSUM  ? "TXCSUM "  : "",
 523                      hw->cap & NFP_NET_CFG_CTRL_RXVLAN  ? "RXVLAN "  : "",
 524                      hw->cap & NFP_NET_CFG_CTRL_TXVLAN  ? "TXVLAN "  : "",
 525                      hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "",
 526                      hw->cap & NFP_NET_CFG_CTRL_GATHER  ? "GATHER "  : "",
 527                      hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR "  : "",
 528                      hw->cap & NFP_NET_CFG_CTRL_LSO     ? "TSO "     : "",
 529                      hw->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSOv2 "     : "",
 530                      hw->cap & NFP_NET_CFG_CTRL_RSS     ? "RSS "     : "",
 531                      hw->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSSv2 "     : "");
 532
 533         hw->ctrl = 0;
 534
 535         hw->stride_rx = stride;
 536         hw->stride_tx = stride;
 537
 538         PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u",
 539                      hw->max_rx_queues, hw->max_tx_queues);
 540
 541         /* Initializing spinlock for reconfigs */
 542         rte_spinlock_init(&hw->reconfig_lock);
 543
 544         /* Allocating memory for mac addr */
 545         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
 546                                                RTE_ETHER_ADDR_LEN, 0);
 547         if (eth_dev->data->mac_addrs == NULL) {
 548                 PMD_INIT_LOG(ERR, "Failed to space for MAC address");
 549                 err = -ENOMEM;
 550                 goto dev_err_queues_map;
 551         }
 552
 553         nfp_net_pf_read_mac(pf_dev, port);
 554         nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
 555
 556         tmp_ether_addr = (struct rte_ether_addr *)&hw->mac_addr;
 557         if (!rte_is_valid_assigned_ether_addr(tmp_ether_addr)) {
 558                 PMD_INIT_LOG(INFO, "Using random mac address for port %d",
 559                                    port);
 560                 /* Using random mac addresses for VFs */
 561                 rte_eth_random_addr(&hw->mac_addr[0]);
 562                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
 563         }
 564
 565         /* Copying mac address to DPDK eth_dev struct */
 566         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
 567                         &eth_dev->data->mac_addrs[0]);
 568
 569         if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 570                 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR;
 571
 572         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 573
 574         PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x "
 575                      "mac=" RTE_ETHER_ADDR_PRT_FMT,
 576                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 577                      pci_dev->id.device_id,
 578                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
 579                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
 580
 581         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 582                 /* Registering LSC interrupt handler */
 583                 rte_intr_callback_register(pci_dev->intr_handle,
 584                                            nfp_net_dev_interrupt_handler,
 585                                            (void *)eth_dev);
 586                 /* Telling the firmware about the LSC interrupt entry */
 587                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
 588                 /* Recording current stats counters values */
 589                 nfp_net_stats_reset(eth_dev);
 590         }
 591
 592         return 0;
 593
 594 dev_err_queues_map:
 595                 nfp_cpp_area_free(hw->hwqueues_area);
 596 dev_err_ctrl_map:
 597                 nfp_cpp_area_free(hw->ctrl_area);
 598
 599         return err;
 600 }
 601
 602 #define DEFAULT_FW_PATH       "/lib/firmware/netronome"
 603
 604 static int
 605 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 606 {
 607         struct nfp_cpp *cpp = nsp->cpp;
 608         void *fw_buf;
 609         char fw_name[125];
 610         char serial[40];
 611         size_t fsize;
 612
 613         /* Looking for firmware file in order of priority */
 614
 615         /* First try to find a firmware image specific for this device */
 616         snprintf(serial, sizeof(serial),
 617                         "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x",
 618                 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3],
 619                 cpp->serial[4], cpp->serial[5], cpp->interface >> 8,
 620                 cpp->interface & 0xff);
 621
 622         snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 623                         serial);
 624
 625         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
 626         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
 627                 goto load_fw;
 628         /* Then try the PCI name */
 629         snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 630                         dev->device.name);
 631
 632         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
 633         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
 634                 goto load_fw;
 635
 636         /* Finally try the card type and media */
 637         snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 638         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
 639         if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 640                 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 641                 return -ENOENT;
 642         }
 643
 644 load_fw:
 645         PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
 646                 fw_name, fsize);
 647         PMD_DRV_LOG(INFO, "Uploading the firmware ...");
 648         nfp_nsp_load_fw(nsp, fw_buf, fsize);
 649         PMD_DRV_LOG(INFO, "Done");
 650
 651         free(fw_buf);
 652
 653         return 0;
 654 }
 655
 656 static int
 657 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp,
 658              struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo)
 659 {
 660         struct nfp_nsp *nsp;
 661         const char *nfp_fw_model;
 662         char card_desc[100];
 663         int err = 0;
 664
 665         nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno");
 666
 667         if (nfp_fw_model) {
 668                 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model);
 669         } else {
 670                 PMD_DRV_LOG(ERR, "firmware model NOT found");
 671                 return -EIO;
 672         }
 673
 674         if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) {
 675                 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u",
 676                        nfp_eth_table->count);
 677                 return -EIO;
 678         }
 679
 680         PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports",
 681                            nfp_eth_table->count);
 682
 683         PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed);
 684
 685         snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw",
 686                         nfp_fw_model, nfp_eth_table->count,
 687                         nfp_eth_table->ports[0].speed / 1000);
 688
 689         nsp = nfp_nsp_open(cpp);
 690         if (!nsp) {
 691                 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle");
 692                 return -EIO;
 693         }
 694
 695         nfp_nsp_device_soft_reset(nsp);
 696         err = nfp_fw_upload(dev, nsp, card_desc);
 697
 698         nfp_nsp_close(nsp);
 699         return err;
 700 }
 701
 702 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev)
 703 {
 704         struct nfp_net_hw *hw;
 705         struct rte_eth_dev *eth_dev;
 706         struct nfp_eth_table *nfp_eth_table = NULL;
 707         int ret = 0;
 708         int i;
 709
 710         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
 711         if (!nfp_eth_table) {
 712                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
 713                 ret = -EIO;
 714                 goto error;
 715         }
 716
 717         /* Loop through all physical ports on PF */
 718         for (i = 0; i < pf_dev->total_phyports; i++) {
 719                 const unsigned int numa_node = rte_socket_id();
 720                 char port_name[RTE_ETH_NAME_MAX_LEN];
 721
 722                 snprintf(port_name, sizeof(port_name), "%s_port%d",
 723                          pf_dev->pci_dev->device.name, i);
 724
 725                 /* Allocate a eth_dev for this phyport */
 726                 eth_dev = rte_eth_dev_allocate(port_name);
 727                 if (!eth_dev) {
 728                         ret = -ENODEV;
 729                         goto port_cleanup;
 730                 }
 731
 732                 /* Allocate memory for this phyport */
 733                 eth_dev->data->dev_private =
 734                         rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw),
 735                                            RTE_CACHE_LINE_SIZE, numa_node);
 736                 if (!eth_dev->data->dev_private) {
 737                         ret = -ENOMEM;
 738                         rte_eth_dev_release_port(eth_dev);
 739                         goto port_cleanup;
 740                 }
 741
 742                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 743
 744                 /* Add this device to the PF's array of physical ports */
 745                 pf_dev->ports[i] = hw;
 746
 747                 hw->pf_dev = pf_dev;
 748                 hw->cpp = pf_dev->cpp;
 749                 hw->eth_dev = eth_dev;
 750                 hw->idx = i;
 751                 hw->nfp_idx = nfp_eth_table->ports[i].index;
 752                 hw->is_phyport = true;
 753
 754                 eth_dev->device = &pf_dev->pci_dev->device;
 755
 756                 /* ctrl/tx/rx BAR mappings and remaining init happens in
 757                  * nfp_net_init
 758                  */
 759                 ret = nfp_net_init(eth_dev);
 760
 761                 if (ret) {
 762                         ret = -ENODEV;
 763                         goto port_cleanup;
 764                 }
 765
 766                 rte_eth_dev_probing_finish(eth_dev);
 767
 768         } /* End loop, all ports on this PF */
 769         ret = 0;
 770         goto eth_table_cleanup;
 771
 772 port_cleanup:
 773         for (i = 0; i < pf_dev->total_phyports; i++) {
 774                 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) {
 775                         struct rte_eth_dev *tmp_dev;
 776                         tmp_dev = pf_dev->ports[i]->eth_dev;
 777                         rte_eth_dev_release_port(tmp_dev);
 778                         pf_dev->ports[i] = NULL;
 779                 }
 780         }
 781 eth_table_cleanup:
 782         free(nfp_eth_table);
 783 error:
 784         return ret;
 785 }
 786
 787 static int nfp_pf_init(struct rte_pci_device *pci_dev)
 788 {
 789         struct nfp_pf_dev *pf_dev = NULL;
 790         struct nfp_cpp *cpp;
 791         struct nfp_hwinfo *hwinfo;
 792         struct nfp_rtsym_table *sym_tbl;
 793         struct nfp_eth_table *nfp_eth_table = NULL;
 794         char name[RTE_ETH_NAME_MAX_LEN];
 795         int total_ports;
 796         int ret = -ENODEV;
 797         int err;
 798
 799         if (!pci_dev)
 800                 return ret;
 801
 802         /*
 803          * When device bound to UIO, the device could be used, by mistake,
 804          * by two DPDK apps, and the UIO driver does not avoid it. This
 805          * could lead to a serious problem when configuring the NFP CPP
 806          * interface. Here we avoid this telling to the CPP init code to
 807          * use a lock file if UIO is being used.
 808          */
 809         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
 810                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
 811         else
 812                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
 813
 814         if (!cpp) {
 815                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
 816                 ret = -EIO;
 817                 goto error;
 818         }
 819
 820         hwinfo = nfp_hwinfo_read(cpp);
 821         if (!hwinfo) {
 822                 PMD_INIT_LOG(ERR, "Error reading hwinfo table");
 823                 ret = -EIO;
 824                 goto error;
 825         }
 826
 827         nfp_eth_table = nfp_eth_read_ports(cpp);
 828         if (!nfp_eth_table) {
 829                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
 830                 ret = -EIO;
 831                 goto hwinfo_cleanup;
 832         }
 833
 834         if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) {
 835                 PMD_INIT_LOG(ERR, "Error when uploading firmware");
 836                 ret = -EIO;
 837                 goto eth_table_cleanup;
 838         }
 839
 840         /* Now the symbol table should be there */
 841         sym_tbl = nfp_rtsym_table_read(cpp);
 842         if (!sym_tbl) {
 843                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
 844                                 " symbol table");
 845                 ret = -EIO;
 846                 goto eth_table_cleanup;
 847         }
 848
 849         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
 850         if (total_ports != (int)nfp_eth_table->count) {
 851                 PMD_DRV_LOG(ERR, "Inconsistent number of ports");
 852                 ret = -EIO;
 853                 goto sym_tbl_cleanup;
 854         }
 855
 856         PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports);
 857
 858         if (total_ports <= 0 || total_ports > 8) {
 859                 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
 860                 ret = -ENODEV;
 861                 goto sym_tbl_cleanup;
 862         }
 863         /* Allocate memory for the PF "device" */
 864         snprintf(name, sizeof(name), "nfp_pf%d", 0);
 865         pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
 866         if (!pf_dev) {
 867                 ret = -ENOMEM;
 868                 goto sym_tbl_cleanup;
 869         }
 870
 871         /* Populate the newly created PF device */
 872         pf_dev->cpp = cpp;
 873         pf_dev->hwinfo = hwinfo;
 874         pf_dev->sym_tbl = sym_tbl;
 875         pf_dev->total_phyports = total_ports;
 876
 877         if (total_ports > 1)
 878                 pf_dev->multiport = true;
 879
 880         pf_dev->pci_dev = pci_dev;
 881
 882         /* Map the symbol table */
 883         pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0",
 884                                      pf_dev->total_phyports * 32768,
 885                                      &pf_dev->ctrl_area);
 886         if (!pf_dev->ctrl_bar) {
 887                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar");
 888                 ret = -EIO;
 889                 goto pf_cleanup;
 890         }
 891
 892         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar);
 893
 894         /* configure access to tx/rx vNIC BARs */
 895         pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0,
 896                                               NFP_PCIE_QUEUE(0),
 897                                               NFP_QCP_QUEUE_AREA_SZ,
 898                                               &pf_dev->hwqueues_area);
 899         if (!pf_dev->hw_queues) {
 900                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
 901                 ret = -EIO;
 902                 goto ctrl_area_cleanup;
 903         }
 904
 905         PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues);
 906
 907         /* Initialize and prep physical ports now
 908          * This will loop through all physical ports
 909          */
 910         ret = nfp_init_phyports(pf_dev);
 911         if (ret) {
 912                 PMD_INIT_LOG(ERR, "Could not create physical ports");
 913                 goto hwqueues_cleanup;
 914         }
 915
 916         /* register the CPP bridge service here for primary use */
 917         nfp_register_cpp_service(pf_dev->cpp);
 918
 919         return 0;
 920
 921 hwqueues_cleanup:
 922         nfp_cpp_area_free(pf_dev->hwqueues_area);
 923 ctrl_area_cleanup:
 924         nfp_cpp_area_free(pf_dev->ctrl_area);
 925 pf_cleanup:
 926         rte_free(pf_dev);
 927 sym_tbl_cleanup:
 928         free(sym_tbl);
 929 eth_table_cleanup:
 930         free(nfp_eth_table);
 931 hwinfo_cleanup:
 932         free(hwinfo);
 933 error:
 934         return ret;
 935 }
 936
 937 /*
 938  * When attaching to the NFP4000/6000 PF on a secondary process there
 939  * is no need to initialise the PF again. Only minimal work is required
 940  * here
 941  */
 942 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
 943 {
 944         struct nfp_cpp *cpp;
 945         struct nfp_rtsym_table *sym_tbl;
 946         int total_ports;
 947         int i;
 948         int err;
 949
 950         if (!pci_dev)
 951                 return -ENODEV;
 952
 953         /*
 954          * When device bound to UIO, the device could be used, by mistake,
 955          * by two DPDK apps, and the UIO driver does not avoid it. This
 956          * could lead to a serious problem when configuring the NFP CPP
 957          * interface. Here we avoid this telling to the CPP init code to
 958          * use a lock file if UIO is being used.
 959          */
 960         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
 961                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
 962         else
 963                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
 964
 965         if (!cpp) {
 966                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
 967                 return -EIO;
 968         }
 969
 970         /*
 971          * We don't have access to the PF created in the primary process
 972          * here so we have to read the number of ports from firmware
 973          */
 974         sym_tbl = nfp_rtsym_table_read(cpp);
 975         if (!sym_tbl) {
 976                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
 977                                 " symbol table");
 978                 return -EIO;
 979         }
 980
 981         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
 982
 983         for (i = 0; i < total_ports; i++) {
 984                 struct rte_eth_dev *eth_dev;
 985                 char port_name[RTE_ETH_NAME_MAX_LEN];
 986
 987                 snprintf(port_name, sizeof(port_name), "%s_port%d",
 988                          pci_dev->device.name, i);
 989
 990                 PMD_DRV_LOG(DEBUG, "Secondary attaching to port %s",
 991                     port_name);
 992                 eth_dev = rte_eth_dev_attach_secondary(port_name);
 993                 if (!eth_dev) {
 994                         RTE_LOG(ERR, EAL,
 995                         "secondary process attach failed, "
 996                         "ethdev doesn't exist");
 997                         return -ENODEV;
 998                 }
 999                 eth_dev->process_private = cpp;
1000                 eth_dev->dev_ops = &nfp_net_eth_dev_ops;
1001                 eth_dev->rx_queue_count = nfp_net_rx_queue_count;
1002                 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
1003                 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
1004                 rte_eth_dev_probing_finish(eth_dev);
1005         }
1006
1007         /* Register the CPP bridge service for the secondary too */
1008         nfp_register_cpp_service(cpp);
1009
1010         return 0;
1011 }
1012
1013 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1014                             struct rte_pci_device *dev)
1015 {
1016         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
1017                 return nfp_pf_init(dev);
1018         else
1019                 return nfp_pf_secondary_init(dev);
1020 }
1021
1022 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = {
1023         {
1024                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
1025                                PCI_DEVICE_ID_NFP4000_PF_NIC)
1026         },
1027         {
1028                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
1029                                PCI_DEVICE_ID_NFP6000_PF_NIC)
1030         },
1031         {
1032                 .vendor_id = 0,
1033         },
1034 };
1035
1036 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev)
1037 {
1038         struct rte_pci_device *pci_dev;
1039         uint16_t port_id;
1040
1041         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1042
1043         /* Free up all physical ports under PF */
1044         RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
1045                 rte_eth_dev_close(port_id);
1046         /*
1047          * Ports can be closed and freed but hotplugging is not
1048          * currently supported
1049          */
1050         return -ENOTSUP;
1051 }
1052
1053 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev)
1054 {
1055         return rte_eth_dev_pci_generic_remove(pci_dev, nfp_pci_uninit);
1056 }
1057
1058 static struct rte_pci_driver rte_nfp_net_pf_pmd = {
1059         .id_table = pci_id_nfp_pf_net_map,
1060         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1061         .probe = nfp_pf_pci_probe,
1062         .remove = eth_nfp_pci_remove,
1063 };
1064
1065 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd);
1066 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map);
1067 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio");
1068 /*
1069  * Local variables:
1070  * c-file-style: "Linux"
1071  * indent-tabs-mode: t
1072  * End:
1073  */