drivers/net/nfp/nfp_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2014-2021 Netronome Systems, Inc.
   3  * All rights reserved.
   4  *
   5  * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.
   6  */
   7
   8 /*
   9  * vim:shiftwidth=8:noexpandtab
  10  *
  11  * @file dpdk/pmd/nfp_ethdev.c
  12  *
  13  * Netronome vNIC DPDK Poll-Mode Driver: Main entry point
  14  */
  15
  16 #include <rte_common.h>
  17 #include <ethdev_driver.h>
  18 #include <ethdev_pci.h>
  19 #include <rte_dev.h>
  20 #include <rte_ether.h>
  21 #include <rte_malloc.h>
  22 #include <rte_memzone.h>
  23 #include <rte_mempool.h>
  24 #include <rte_service_component.h>
  25 #include <rte_alarm.h>
  26 #include "eal_firmware.h"
  27
  28 #include "nfpcore/nfp_cpp.h"
  29 #include "nfpcore/nfp_nffw.h"
  30 #include "nfpcore/nfp_hwinfo.h"
  31 #include "nfpcore/nfp_mip.h"
  32 #include "nfpcore/nfp_rtsym.h"
  33 #include "nfpcore/nfp_nsp.h"
  34
  35 #include "nfp_common.h"
  36 #include "nfp_rxtx.h"
  37 #include "nfp_logs.h"
  38 #include "nfp_ctrl.h"
  39 #include "nfp_cpp_bridge.h"
  40
  41
  42 static int nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port);
  43 static int nfp_net_start(struct rte_eth_dev *dev);
  44 static int nfp_net_stop(struct rte_eth_dev *dev);
  45 static int nfp_net_set_link_up(struct rte_eth_dev *dev);
  46 static int nfp_net_set_link_down(struct rte_eth_dev *dev);
  47 static int nfp_net_close(struct rte_eth_dev *dev);
  48 static int nfp_net_init(struct rte_eth_dev *eth_dev);
  49 static int nfp_fw_upload(struct rte_pci_device *dev,
  50                          struct nfp_nsp *nsp, char *card);
  51 static int nfp_fw_setup(struct rte_pci_device *dev,
  52                         struct nfp_cpp *cpp,
  53                         struct nfp_eth_table *nfp_eth_table,
  54                         struct nfp_hwinfo *hwinfo);
  55 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev);
  56 static int nfp_pf_init(struct rte_pci_device *pci_dev);
  57 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev);
  58 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
  59                             struct rte_pci_device *dev);
  60 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev);
  61 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev);
  62
  63 static int
  64 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port)
  65 {
  66         struct nfp_eth_table *nfp_eth_table;
  67         struct nfp_net_hw *hw = NULL;
  68
  69         /* Grab a pointer to the correct physical port */
  70         hw = pf_dev->ports[port];
  71
  72         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
  73
  74         nfp_eth_copy_mac((uint8_t *)&hw->mac_addr,
  75                          (uint8_t *)&nfp_eth_table->ports[port].mac_addr);
  76
  77         free(nfp_eth_table);
  78         return 0;
  79 }
  80
  81 static int
  82 nfp_net_start(struct rte_eth_dev *dev)
  83 {
  84         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
  85         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
  86         uint32_t new_ctrl, update = 0;
  87         struct nfp_net_hw *hw;
  88         struct nfp_pf_dev *pf_dev;
  89         struct rte_eth_conf *dev_conf;
  90         struct rte_eth_rxmode *rxmode;
  91         uint32_t intr_vector;
  92         int ret;
  93
  94         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  95         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
  96
  97         PMD_INIT_LOG(DEBUG, "Start");
  98
  99         /* Disabling queues just in case... */
 100         nfp_net_disable_queues(dev);
 101
 102         /* Enabling the required queues in the device */
 103         nfp_net_enable_queues(dev);
 104
 105         /* check and configure queue intr-vector mapping */
 106         if (dev->data->dev_conf.intr_conf.rxq != 0) {
 107                 if (pf_dev->multiport) {
 108                         PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported "
 109                                           "with NFP multiport PF");
 110                                 return -EINVAL;
 111                 }
 112                 if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 113                         /*
 114                          * Better not to share LSC with RX interrupts.
 115                          * Unregistering LSC interrupt handler
 116                          */
 117                         rte_intr_callback_unregister(&pci_dev->intr_handle,
 118                                 nfp_net_dev_interrupt_handler, (void *)dev);
 119
 120                         if (dev->data->nb_rx_queues > 1) {
 121                                 PMD_INIT_LOG(ERR, "PMD rx interrupt only "
 122                                              "supports 1 queue with UIO");
 123                                 return -EIO;
 124                         }
 125                 }
 126                 intr_vector = dev->data->nb_rx_queues;
 127                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 128                         return -1;
 129
 130                 nfp_configure_rx_interrupt(dev, intr_handle);
 131                 update = NFP_NET_CFG_UPDATE_MSIX;
 132         }
 133
 134         rte_intr_enable(intr_handle);
 135
 136         new_ctrl = nfp_check_offloads(dev);
 137
 138         /* Writing configuration parameters in the device */
 139         nfp_net_params_setup(hw);
 140
 141         dev_conf = &dev->data->dev_conf;
 142         rxmode = &dev_conf->rxmode;
 143
 144         if (rxmode->mq_mode & ETH_MQ_RX_RSS) {
 145                 nfp_net_rss_config_default(dev);
 146                 update |= NFP_NET_CFG_UPDATE_RSS;
 147                 new_ctrl |= NFP_NET_CFG_CTRL_RSS;
 148         }
 149
 150         /* Enable device */
 151         new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
 152
 153         update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING;
 154
 155         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 156                 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;
 157
 158         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl);
 159         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 160                 return -EIO;
 161
 162         /*
 163          * Allocating rte mbufs for configured rx queues.
 164          * This requires queues being enabled before
 165          */
 166         if (nfp_net_rx_freelist_setup(dev) < 0) {
 167                 ret = -ENOMEM;
 168                 goto error;
 169         }
 170
 171         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 172                 /* Configure the physical port up */
 173                 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 174         else
 175                 nfp_eth_set_configured(dev->process_private,
 176                                        hw->nfp_idx, 1);
 177
 178         hw->ctrl = new_ctrl;
 179
 180         return 0;
 181
 182 error:
 183         /*
 184          * An error returned by this function should mean the app
 185          * exiting and then the system releasing all the memory
 186          * allocated even memory coming from hugepages.
 187          *
 188          * The device could be enabled at this point with some queues
 189          * ready for getting packets. This is true if the call to
 190          * nfp_net_rx_freelist_setup() succeeds for some queues but
 191          * fails for subsequent queues.
 192          *
 193          * This should make the app exiting but better if we tell the
 194          * device first.
 195          */
 196         nfp_net_disable_queues(dev);
 197
 198         return ret;
 199 }
 200
 201 /* Stop device: disable rx and tx functions to allow for reconfiguring. */
 202 static int
 203 nfp_net_stop(struct rte_eth_dev *dev)
 204 {
 205         int i;
 206         struct nfp_net_hw *hw;
 207         struct nfp_net_txq *this_tx_q;
 208         struct nfp_net_rxq *this_rx_q;
 209
 210         PMD_INIT_LOG(DEBUG, "Stop");
 211
 212         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 213
 214         nfp_net_disable_queues(dev);
 215
 216         /* Clear queues */
 217         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 218                 this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i];
 219                 nfp_net_reset_tx_queue(this_tx_q);
 220         }
 221
 222         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 223                 this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i];
 224                 nfp_net_reset_rx_queue(this_rx_q);
 225         }
 226
 227         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 228                 /* Configure the physical port down */
 229                 nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 230         else
 231                 nfp_eth_set_configured(dev->process_private,
 232                                        hw->nfp_idx, 0);
 233
 234         return 0;
 235 }
 236
 237 /* Set the link up. */
 238 static int
 239 nfp_net_set_link_up(struct rte_eth_dev *dev)
 240 {
 241         struct nfp_net_hw *hw;
 242
 243         PMD_DRV_LOG(DEBUG, "Set link up");
 244
 245         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 246
 247         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 248                 /* Configure the physical port down */
 249                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 250         else
 251                 return nfp_eth_set_configured(dev->process_private,
 252                                               hw->nfp_idx, 1);
 253 }
 254
 255 /* Set the link down. */
 256 static int
 257 nfp_net_set_link_down(struct rte_eth_dev *dev)
 258 {
 259         struct nfp_net_hw *hw;
 260
 261         PMD_DRV_LOG(DEBUG, "Set link down");
 262
 263         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 264
 265         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 266                 /* Configure the physical port down */
 267                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 268         else
 269                 return nfp_eth_set_configured(dev->process_private,
 270                                               hw->nfp_idx, 0);
 271 }
 272
 273 /* Reset and stop device. The device can not be restarted. */
 274 static int
 275 nfp_net_close(struct rte_eth_dev *dev)
 276 {
 277         struct nfp_net_hw *hw;
 278         struct rte_pci_device *pci_dev;
 279         struct nfp_pf_dev *pf_dev;
 280         struct nfp_net_txq *this_tx_q;
 281         struct nfp_net_rxq *this_rx_q;
 282         int i;
 283
 284         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 285                 return 0;
 286
 287         PMD_INIT_LOG(DEBUG, "Close");
 288
 289         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 290         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 291         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 292
 293         /*
 294          * We assume that the DPDK application is stopping all the
 295          * threads/queues before calling the device close function.
 296          */
 297
 298         nfp_net_disable_queues(dev);
 299
 300         /* Clear queues */
 301         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 302                 this_tx_q = (struct nfp_net_txq *)dev->data->tx_queues[i];
 303                 nfp_net_reset_tx_queue(this_tx_q);
 304         }
 305
 306         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 307                 this_rx_q = (struct nfp_net_rxq *)dev->data->rx_queues[i];
 308                 nfp_net_reset_rx_queue(this_rx_q);
 309         }
 310
 311         /* Cancel possible impending LSC work here before releasing the port*/
 312         rte_eal_alarm_cancel(nfp_net_dev_interrupt_delayed_handler,
 313                              (void *)dev);
 314
 315         /* Only free PF resources after all physical ports have been closed */
 316         /* Mark this port as unused and free device priv resources*/
 317         nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff);
 318         pf_dev->ports[hw->idx] = NULL;
 319         rte_eth_dev_release_port(dev);
 320
 321         for (i = 0; i < pf_dev->total_phyports; i++) {
 322                 /* Check to see if ports are still in use */
 323                 if (pf_dev->ports[i])
 324                         return 0;
 325         }
 326
 327         /* Now it is safe to free all PF resources */
 328         PMD_INIT_LOG(INFO, "Freeing PF resources");
 329         nfp_cpp_area_free(pf_dev->ctrl_area);
 330         nfp_cpp_area_free(pf_dev->hwqueues_area);
 331         free(pf_dev->hwinfo);
 332         free(pf_dev->sym_tbl);
 333         nfp_cpp_free(pf_dev->cpp);
 334         rte_free(pf_dev);
 335
 336         rte_intr_disable(&pci_dev->intr_handle);
 337
 338         /* unregister callback func from eal lib */
 339         rte_intr_callback_unregister(&pci_dev->intr_handle,
 340                                      nfp_net_dev_interrupt_handler,
 341                                      (void *)dev);
 342
 343         /*
 344          * The ixgbe PMD driver disables the pcie master on the
 345          * device. The i40e does not...
 346          */
 347
 348         return 0;
 349 }
 350
 351 /* Initialise and register driver with DPDK Application */
 352 static const struct eth_dev_ops nfp_net_eth_dev_ops = {
 353         .dev_configure          = nfp_net_configure,
 354         .dev_start              = nfp_net_start,
 355         .dev_stop               = nfp_net_stop,
 356         .dev_set_link_up        = nfp_net_set_link_up,
 357         .dev_set_link_down      = nfp_net_set_link_down,
 358         .dev_close              = nfp_net_close,
 359         .promiscuous_enable     = nfp_net_promisc_enable,
 360         .promiscuous_disable    = nfp_net_promisc_disable,
 361         .link_update            = nfp_net_link_update,
 362         .stats_get              = nfp_net_stats_get,
 363         .stats_reset            = nfp_net_stats_reset,
 364         .dev_infos_get          = nfp_net_infos_get,
 365         .dev_supported_ptypes_get = nfp_net_supported_ptypes_get,
 366         .mtu_set                = nfp_net_dev_mtu_set,
 367         .mac_addr_set           = nfp_set_mac_addr,
 368         .vlan_offload_set       = nfp_net_vlan_offload_set,
 369         .reta_update            = nfp_net_reta_update,
 370         .reta_query             = nfp_net_reta_query,
 371         .rss_hash_update        = nfp_net_rss_hash_update,
 372         .rss_hash_conf_get      = nfp_net_rss_hash_conf_get,
 373         .rx_queue_setup         = nfp_net_rx_queue_setup,
 374         .rx_queue_release       = nfp_net_rx_queue_release,
 375         .tx_queue_setup         = nfp_net_tx_queue_setup,
 376         .tx_queue_release       = nfp_net_tx_queue_release,
 377         .rx_queue_intr_enable   = nfp_rx_queue_intr_enable,
 378         .rx_queue_intr_disable  = nfp_rx_queue_intr_disable,
 379 };
 380
 381 static int
 382 nfp_net_init(struct rte_eth_dev *eth_dev)
 383 {
 384         struct rte_pci_device *pci_dev;
 385         struct nfp_pf_dev *pf_dev;
 386         struct nfp_net_hw *hw;
 387         struct rte_ether_addr *tmp_ether_addr;
 388
 389         uint64_t tx_bar_off = 0, rx_bar_off = 0;
 390         uint32_t start_q;
 391         int stride = 4;
 392         int port = 0;
 393         int err;
 394
 395         PMD_INIT_FUNC_TRACE();
 396
 397         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 398
 399         /* Use backpointer here to the PF of this eth_dev */
 400         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private);
 401
 402         /* NFP can not handle DMA addresses requiring more than 40 bits */
 403         if (rte_mem_check_dma_mask(40)) {
 404                 RTE_LOG(ERR, PMD, "device %s can not be used:",
 405                                    pci_dev->device.name);
 406                 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n");
 407                 return -ENODEV;
 408         };
 409
 410         port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx;
 411         if (port < 0 || port > 7) {
 412                 PMD_DRV_LOG(ERR, "Port value is wrong");
 413                 return -ENODEV;
 414         }
 415
 416         /* Use PF array of physical ports to get pointer to
 417          * this specific port
 418          */
 419         hw = pf_dev->ports[port];
 420
 421         PMD_INIT_LOG(DEBUG, "Working with physical port number: %d, "
 422                             "NFP internal port number: %d",
 423                             port, hw->nfp_idx);
 424
 425         eth_dev->dev_ops = &nfp_net_eth_dev_ops;
 426         eth_dev->rx_queue_count = nfp_net_rx_queue_count;
 427         eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
 428         eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
 429
 430         /* For secondary processes, the primary has done all the work */
 431         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 432                 return 0;
 433
 434         rte_eth_copy_pci_info(eth_dev, pci_dev);
 435
 436         hw->device_id = pci_dev->id.device_id;
 437         hw->vendor_id = pci_dev->id.vendor_id;
 438         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 439         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 440
 441         PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u",
 442                      pci_dev->id.vendor_id, pci_dev->id.device_id,
 443                      pci_dev->addr.domain, pci_dev->addr.bus,
 444                      pci_dev->addr.devid, pci_dev->addr.function);
 445
 446         hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr;
 447         if (hw->ctrl_bar == NULL) {
 448                 PMD_DRV_LOG(ERR,
 449                         "hw->ctrl_bar is NULL. BAR0 not configured");
 450                 return -ENODEV;
 451         }
 452
 453         if (port == 0) {
 454                 hw->ctrl_bar = pf_dev->ctrl_bar;
 455         } else {
 456                 if (!pf_dev->ctrl_bar)
 457                         return -ENODEV;
 458                 /* Use port offset in pf ctrl_bar for this
 459                  * ports control bar
 460                  */
 461                 hw->ctrl_bar = pf_dev->ctrl_bar +
 462                                (port * NFP_PF_CSR_SLICE_SIZE);
 463         }
 464
 465         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar);
 466
 467         hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS);
 468         hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS);
 469
 470         /* Work out where in the BAR the queues start. */
 471         switch (pci_dev->id.device_id) {
 472         case PCI_DEVICE_ID_NFP4000_PF_NIC:
 473         case PCI_DEVICE_ID_NFP6000_PF_NIC:
 474                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
 475                 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
 476                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
 477                 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
 478                 break;
 479         default:
 480                 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
 481                 err = -ENODEV;
 482                 goto dev_err_ctrl_map;
 483         }
 484
 485         PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off);
 486         PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off);
 487
 488         hw->tx_bar = pf_dev->hw_queues + tx_bar_off;
 489         hw->rx_bar = pf_dev->hw_queues + rx_bar_off;
 490         eth_dev->data->dev_private = hw;
 491
 492         PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p",
 493                      hw->ctrl_bar, hw->tx_bar, hw->rx_bar);
 494
 495         nfp_net_cfg_queue_setup(hw);
 496
 497         /* Get some of the read-only fields from the config BAR */
 498         hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION);
 499         hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP);
 500         hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU);
 501         hw->mtu = RTE_ETHER_MTU;
 502
 503         /* VLAN insertion is incompatible with LSOv2 */
 504         if (hw->cap & NFP_NET_CFG_CTRL_LSO2)
 505                 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN;
 506
 507         if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2)
 508                 hw->rx_offset = NFP_NET_RX_OFFSET;
 509         else
 510                 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR);
 511
 512         PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d",
 513                            NFD_CFG_MAJOR_VERSION_of(hw->ver),
 514                            NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu);
 515
 516         PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap,
 517                      hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "",
 518                      hw->cap & NFP_NET_CFG_CTRL_L2BC    ? "L2BCFILT " : "",
 519                      hw->cap & NFP_NET_CFG_CTRL_L2MC    ? "L2MCFILT " : "",
 520                      hw->cap & NFP_NET_CFG_CTRL_RXCSUM  ? "RXCSUM "  : "",
 521                      hw->cap & NFP_NET_CFG_CTRL_TXCSUM  ? "TXCSUM "  : "",
 522                      hw->cap & NFP_NET_CFG_CTRL_RXVLAN  ? "RXVLAN "  : "",
 523                      hw->cap & NFP_NET_CFG_CTRL_TXVLAN  ? "TXVLAN "  : "",
 524                      hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "",
 525                      hw->cap & NFP_NET_CFG_CTRL_GATHER  ? "GATHER "  : "",
 526                      hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR "  : "",
 527                      hw->cap & NFP_NET_CFG_CTRL_LSO     ? "TSO "     : "",
 528                      hw->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSOv2 "     : "",
 529                      hw->cap & NFP_NET_CFG_CTRL_RSS     ? "RSS "     : "",
 530                      hw->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSSv2 "     : "");
 531
 532         hw->ctrl = 0;
 533
 534         hw->stride_rx = stride;
 535         hw->stride_tx = stride;
 536
 537         PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u",
 538                      hw->max_rx_queues, hw->max_tx_queues);
 539
 540         /* Initializing spinlock for reconfigs */
 541         rte_spinlock_init(&hw->reconfig_lock);
 542
 543         /* Allocating memory for mac addr */
 544         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
 545                                                RTE_ETHER_ADDR_LEN, 0);
 546         if (eth_dev->data->mac_addrs == NULL) {
 547                 PMD_INIT_LOG(ERR, "Failed to space for MAC address");
 548                 err = -ENOMEM;
 549                 goto dev_err_queues_map;
 550         }
 551
 552         nfp_net_pf_read_mac(pf_dev, port);
 553         nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
 554
 555         tmp_ether_addr = (struct rte_ether_addr *)&hw->mac_addr;
 556         if (!rte_is_valid_assigned_ether_addr(tmp_ether_addr)) {
 557                 PMD_INIT_LOG(INFO, "Using random mac address for port %d",
 558                                    port);
 559                 /* Using random mac addresses for VFs */
 560                 rte_eth_random_addr(&hw->mac_addr[0]);
 561                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
 562         }
 563
 564         /* Copying mac address to DPDK eth_dev struct */
 565         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
 566                         &eth_dev->data->mac_addrs[0]);
 567
 568         if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 569                 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR;
 570
 571         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 572
 573         PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x "
 574                      "mac=" RTE_ETHER_ADDR_PRT_FMT,
 575                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 576                      pci_dev->id.device_id,
 577                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
 578                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
 579
 580         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 581                 /* Registering LSC interrupt handler */
 582                 rte_intr_callback_register(&pci_dev->intr_handle,
 583                                            nfp_net_dev_interrupt_handler,
 584                                            (void *)eth_dev);
 585                 /* Telling the firmware about the LSC interrupt entry */
 586                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
 587                 /* Recording current stats counters values */
 588                 nfp_net_stats_reset(eth_dev);
 589         }
 590
 591         return 0;
 592
 593 dev_err_queues_map:
 594                 nfp_cpp_area_free(hw->hwqueues_area);
 595 dev_err_ctrl_map:
 596                 nfp_cpp_area_free(hw->ctrl_area);
 597
 598         return err;
 599 }
 600
 601 #define DEFAULT_FW_PATH       "/lib/firmware/netronome"
 602
 603 static int
 604 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 605 {
 606         struct nfp_cpp *cpp = nsp->cpp;
 607         void *fw_buf;
 608         char fw_name[125];
 609         char serial[40];
 610         size_t fsize;
 611
 612         /* Looking for firmware file in order of priority */
 613
 614         /* First try to find a firmware image specific for this device */
 615         snprintf(serial, sizeof(serial),
 616                         "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x",
 617                 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3],
 618                 cpp->serial[4], cpp->serial[5], cpp->interface >> 8,
 619                 cpp->interface & 0xff);
 620
 621         snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 622                         serial);
 623
 624         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
 625         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
 626                 goto load_fw;
 627         /* Then try the PCI name */
 628         snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 629                         dev->device.name);
 630
 631         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
 632         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
 633                 goto load_fw;
 634
 635         /* Finally try the card type and media */
 636         snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 637         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
 638         if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
 639                 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 640                 return -ENOENT;
 641         }
 642
 643 load_fw:
 644         PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
 645                 fw_name, fsize);
 646         PMD_DRV_LOG(INFO, "Uploading the firmware ...");
 647         nfp_nsp_load_fw(nsp, fw_buf, fsize);
 648         PMD_DRV_LOG(INFO, "Done");
 649
 650         free(fw_buf);
 651
 652         return 0;
 653 }
 654
 655 static int
 656 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp,
 657              struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo)
 658 {
 659         struct nfp_nsp *nsp;
 660         const char *nfp_fw_model;
 661         char card_desc[100];
 662         int err = 0;
 663
 664         nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno");
 665
 666         if (nfp_fw_model) {
 667                 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model);
 668         } else {
 669                 PMD_DRV_LOG(ERR, "firmware model NOT found");
 670                 return -EIO;
 671         }
 672
 673         if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) {
 674                 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u",
 675                        nfp_eth_table->count);
 676                 return -EIO;
 677         }
 678
 679         PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports",
 680                            nfp_eth_table->count);
 681
 682         PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed);
 683
 684         snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw",
 685                         nfp_fw_model, nfp_eth_table->count,
 686                         nfp_eth_table->ports[0].speed / 1000);
 687
 688         nsp = nfp_nsp_open(cpp);
 689         if (!nsp) {
 690                 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle");
 691                 return -EIO;
 692         }
 693
 694         nfp_nsp_device_soft_reset(nsp);
 695         err = nfp_fw_upload(dev, nsp, card_desc);
 696
 697         nfp_nsp_close(nsp);
 698         return err;
 699 }
 700
 701 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev)
 702 {
 703         struct nfp_net_hw *hw;
 704         struct rte_eth_dev *eth_dev;
 705         struct nfp_eth_table *nfp_eth_table = NULL;
 706         int ret = 0;
 707         int i;
 708
 709         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
 710         if (!nfp_eth_table) {
 711                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
 712                 ret = -EIO;
 713                 goto error;
 714         }
 715
 716         /* Loop through all physical ports on PF */
 717         for (i = 0; i < pf_dev->total_phyports; i++) {
 718                 const unsigned int numa_node = rte_socket_id();
 719                 char port_name[RTE_ETH_NAME_MAX_LEN];
 720
 721                 snprintf(port_name, sizeof(port_name), "%s_port%d",
 722                          pf_dev->pci_dev->device.name, i);
 723
 724                 /* Allocate a eth_dev for this phyport */
 725                 eth_dev = rte_eth_dev_allocate(port_name);
 726                 if (!eth_dev) {
 727                         ret = -ENODEV;
 728                         goto port_cleanup;
 729                 }
 730
 731                 /* Allocate memory for this phyport */
 732                 eth_dev->data->dev_private =
 733                         rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw),
 734                                            RTE_CACHE_LINE_SIZE, numa_node);
 735                 if (!eth_dev->data->dev_private) {
 736                         ret = -ENOMEM;
 737                         rte_eth_dev_release_port(eth_dev);
 738                         goto port_cleanup;
 739                 }
 740
 741                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 742
 743                 /* Add this device to the PF's array of physical ports */
 744                 pf_dev->ports[i] = hw;
 745
 746                 hw->pf_dev = pf_dev;
 747                 hw->cpp = pf_dev->cpp;
 748                 hw->eth_dev = eth_dev;
 749                 hw->idx = i;
 750                 hw->nfp_idx = nfp_eth_table->ports[i].index;
 751                 hw->is_phyport = true;
 752
 753                 eth_dev->device = &pf_dev->pci_dev->device;
 754
 755                 /* ctrl/tx/rx BAR mappings and remaining init happens in
 756                  * nfp_net_init
 757                  */
 758                 ret = nfp_net_init(eth_dev);
 759
 760                 if (ret) {
 761                         ret = -ENODEV;
 762                         goto port_cleanup;
 763                 }
 764
 765                 rte_eth_dev_probing_finish(eth_dev);
 766
 767         } /* End loop, all ports on this PF */
 768         ret = 0;
 769         goto eth_table_cleanup;
 770
 771 port_cleanup:
 772         for (i = 0; i < pf_dev->total_phyports; i++) {
 773                 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) {
 774                         struct rte_eth_dev *tmp_dev;
 775                         tmp_dev = pf_dev->ports[i]->eth_dev;
 776                         rte_eth_dev_release_port(tmp_dev);
 777                         pf_dev->ports[i] = NULL;
 778                 }
 779         }
 780 eth_table_cleanup:
 781         free(nfp_eth_table);
 782 error:
 783         return ret;
 784 }
 785
 786 static int nfp_pf_init(struct rte_pci_device *pci_dev)
 787 {
 788         struct nfp_pf_dev *pf_dev = NULL;
 789         struct nfp_cpp *cpp;
 790         struct nfp_hwinfo *hwinfo;
 791         struct nfp_rtsym_table *sym_tbl;
 792         struct nfp_eth_table *nfp_eth_table = NULL;
 793         char name[RTE_ETH_NAME_MAX_LEN];
 794         int total_ports;
 795         int ret = -ENODEV;
 796         int err;
 797
 798         if (!pci_dev)
 799                 return ret;
 800
 801         /*
 802          * When device bound to UIO, the device could be used, by mistake,
 803          * by two DPDK apps, and the UIO driver does not avoid it. This
 804          * could lead to a serious problem when configuring the NFP CPP
 805          * interface. Here we avoid this telling to the CPP init code to
 806          * use a lock file if UIO is being used.
 807          */
 808         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
 809                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
 810         else
 811                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
 812
 813         if (!cpp) {
 814                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
 815                 ret = -EIO;
 816                 goto error;
 817         }
 818
 819         hwinfo = nfp_hwinfo_read(cpp);
 820         if (!hwinfo) {
 821                 PMD_INIT_LOG(ERR, "Error reading hwinfo table");
 822                 ret = -EIO;
 823                 goto error;
 824         }
 825
 826         nfp_eth_table = nfp_eth_read_ports(cpp);
 827         if (!nfp_eth_table) {
 828                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
 829                 ret = -EIO;
 830                 goto hwinfo_cleanup;
 831         }
 832
 833         if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) {
 834                 PMD_INIT_LOG(ERR, "Error when uploading firmware");
 835                 ret = -EIO;
 836                 goto eth_table_cleanup;
 837         }
 838
 839         /* Now the symbol table should be there */
 840         sym_tbl = nfp_rtsym_table_read(cpp);
 841         if (!sym_tbl) {
 842                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
 843                                 " symbol table");
 844                 ret = -EIO;
 845                 goto eth_table_cleanup;
 846         }
 847
 848         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
 849         if (total_ports != (int)nfp_eth_table->count) {
 850                 PMD_DRV_LOG(ERR, "Inconsistent number of ports");
 851                 ret = -EIO;
 852                 goto sym_tbl_cleanup;
 853         }
 854
 855         PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports);
 856
 857         if (total_ports <= 0 || total_ports > 8) {
 858                 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
 859                 ret = -ENODEV;
 860                 goto sym_tbl_cleanup;
 861         }
 862         /* Allocate memory for the PF "device" */
 863         snprintf(name, sizeof(name), "nfp_pf%d", 0);
 864         pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
 865         if (!pf_dev) {
 866                 ret = -ENOMEM;
 867                 goto sym_tbl_cleanup;
 868         }
 869
 870         /* Populate the newly created PF device */
 871         pf_dev->cpp = cpp;
 872         pf_dev->hwinfo = hwinfo;
 873         pf_dev->sym_tbl = sym_tbl;
 874         pf_dev->total_phyports = total_ports;
 875
 876         if (total_ports > 1)
 877                 pf_dev->multiport = true;
 878
 879         pf_dev->pci_dev = pci_dev;
 880
 881         /* Map the symbol table */
 882         pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0",
 883                                      pf_dev->total_phyports * 32768,
 884                                      &pf_dev->ctrl_area);
 885         if (!pf_dev->ctrl_bar) {
 886                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar");
 887                 ret = -EIO;
 888                 goto pf_cleanup;
 889         }
 890
 891         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar);
 892
 893         /* configure access to tx/rx vNIC BARs */
 894         pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0,
 895                                               NFP_PCIE_QUEUE(0),
 896                                               NFP_QCP_QUEUE_AREA_SZ,
 897                                               &pf_dev->hwqueues_area);
 898         if (!pf_dev->hw_queues) {
 899                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
 900                 ret = -EIO;
 901                 goto ctrl_area_cleanup;
 902         }
 903
 904         PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues);
 905
 906         /* Initialize and prep physical ports now
 907          * This will loop through all physical ports
 908          */
 909         ret = nfp_init_phyports(pf_dev);
 910         if (ret) {
 911                 PMD_INIT_LOG(ERR, "Could not create physical ports");
 912                 goto hwqueues_cleanup;
 913         }
 914
 915         /* register the CPP bridge service here for primary use */
 916         nfp_register_cpp_service(pf_dev->cpp);
 917
 918         return 0;
 919
 920 hwqueues_cleanup:
 921         nfp_cpp_area_free(pf_dev->hwqueues_area);
 922 ctrl_area_cleanup:
 923         nfp_cpp_area_free(pf_dev->ctrl_area);
 924 pf_cleanup:
 925         rte_free(pf_dev);
 926 sym_tbl_cleanup:
 927         free(sym_tbl);
 928 eth_table_cleanup:
 929         free(nfp_eth_table);
 930 hwinfo_cleanup:
 931         free(hwinfo);
 932 error:
 933         return ret;
 934 }
 935
 936 /*
 937  * When attaching to the NFP4000/6000 PF on a secondary process there
 938  * is no need to initialise the PF again. Only minimal work is required
 939  * here
 940  */
 941 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
 942 {
 943         struct nfp_cpp *cpp;
 944         struct nfp_rtsym_table *sym_tbl;
 945         int total_ports;
 946         int i;
 947         int err;
 948
 949         if (!pci_dev)
 950                 return -ENODEV;
 951
 952         /*
 953          * When device bound to UIO, the device could be used, by mistake,
 954          * by two DPDK apps, and the UIO driver does not avoid it. This
 955          * could lead to a serious problem when configuring the NFP CPP
 956          * interface. Here we avoid this telling to the CPP init code to
 957          * use a lock file if UIO is being used.
 958          */
 959         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
 960                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
 961         else
 962                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
 963
 964         if (!cpp) {
 965                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
 966                 return -EIO;
 967         }
 968
 969         /*
 970          * We don't have access to the PF created in the primary process
 971          * here so we have to read the number of ports from firmware
 972          */
 973         sym_tbl = nfp_rtsym_table_read(cpp);
 974         if (!sym_tbl) {
 975                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
 976                                 " symbol table");
 977                 return -EIO;
 978         }
 979
 980         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
 981
 982         for (i = 0; i < total_ports; i++) {
 983                 struct rte_eth_dev *eth_dev;
 984                 char port_name[RTE_ETH_NAME_MAX_LEN];
 985
 986                 snprintf(port_name, sizeof(port_name), "%s_port%d",
 987                          pci_dev->device.name, i);
 988
 989                 PMD_DRV_LOG(DEBUG, "Secondary attaching to port %s",
 990                     port_name);
 991                 eth_dev = rte_eth_dev_attach_secondary(port_name);
 992                 if (!eth_dev) {
 993                         RTE_LOG(ERR, EAL,
 994                         "secondary process attach failed, "
 995                         "ethdev doesn't exist");
 996                         return -ENODEV;
 997                 }
 998                 eth_dev->process_private = cpp;
 999                 eth_dev->dev_ops = &nfp_net_eth_dev_ops;
1000                 eth_dev->rx_queue_count = nfp_net_rx_queue_count;
1001                 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
1002                 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
1003                 rte_eth_dev_probing_finish(eth_dev);
1004         }
1005
1006         /* Register the CPP bridge service for the secondary too */
1007         nfp_register_cpp_service(cpp);
1008
1009         return 0;
1010 }
1011
1012 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1013                             struct rte_pci_device *dev)
1014 {
1015         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
1016                 return nfp_pf_init(dev);
1017         else
1018                 return nfp_pf_secondary_init(dev);
1019 }
1020
1021 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = {
1022         {
1023                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
1024                                PCI_DEVICE_ID_NFP4000_PF_NIC)
1025         },
1026         {
1027                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
1028                                PCI_DEVICE_ID_NFP6000_PF_NIC)
1029         },
1030         {
1031                 .vendor_id = 0,
1032         },
1033 };
1034
1035 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev)
1036 {
1037         struct rte_pci_device *pci_dev;
1038         uint16_t port_id;
1039
1040         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1041
1042         /* Free up all physical ports under PF */
1043         RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
1044                 rte_eth_dev_close(port_id);
1045         /*
1046          * Ports can be closed and freed but hotplugging is not
1047          * currently supported
1048          */
1049         return -ENOTSUP;
1050 }
1051
1052 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev)
1053 {
1054         return rte_eth_dev_pci_generic_remove(pci_dev, nfp_pci_uninit);
1055 }
1056
1057 static struct rte_pci_driver rte_nfp_net_pf_pmd = {
1058         .id_table = pci_id_nfp_pf_net_map,
1059         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1060         .probe = nfp_pf_pci_probe,
1061         .remove = eth_nfp_pci_remove,
1062 };
1063
1064 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd);
1065 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map);
1066 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio");
1067 /*
1068  * Local variables:
1069  * c-file-style: "Linux"
1070  * indent-tabs-mode: t
1071  * End:
1072  */