drivers/net/nfp/nfp_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2014-2018 Netronome Systems, Inc.
   3  * All rights reserved.
   4  *
   5  * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.
   6  */
   7
   8 /*
   9  * vim:shiftwidth=8:noexpandtab
  10  *
  11  * @file dpdk/pmd/nfp_net.c
  12  *
  13  * Netronome vNIC DPDK Poll-Mode Driver: Main entry point
  14  */
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <ethdev_driver.h>
  21 #include <ethdev_pci.h>
  22 #include <rte_dev.h>
  23 #include <rte_ether.h>
  24 #include <rte_malloc.h>
  25 #include <rte_memzone.h>
  26 #include <rte_mempool.h>
  27 #include <rte_version.h>
  28 #include <rte_string_fns.h>
  29 #include <rte_alarm.h>
  30 #include <rte_spinlock.h>
  31 #include <rte_service_component.h>
  32
  33 #include "eal_firmware.h"
  34
  35 #include "nfpcore/nfp_cpp.h"
  36 #include "nfpcore/nfp_nffw.h"
  37 #include "nfpcore/nfp_hwinfo.h"
  38 #include "nfpcore/nfp_mip.h"
  39 #include "nfpcore/nfp_rtsym.h"
  40 #include "nfpcore/nfp_nsp.h"
  41
  42 #include "nfp_net_pmd.h"
  43 #include "nfp_rxtx.h"
  44 #include "nfp_net_logs.h"
  45 #include "nfp_net_ctrl.h"
  46 #include "nfp_cpp_bridge.h"
  47
  48 #include <sys/types.h>
  49 #include <sys/socket.h>
  50 #include <sys/un.h>
  51 #include <unistd.h>
  52 #include <stdio.h>
  53 #include <sys/ioctl.h>
  54 #include <errno.h>
  55
  56 /* Prototypes */
  57 static int nfp_net_close(struct rte_eth_dev *dev);
  58 static int nfp_net_init(struct rte_eth_dev *eth_dev);
  59 static int nfp_pf_init(struct rte_pci_device *pci_dev);
  60 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev);
  61 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev);
  62 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev);
  63 static int nfp_net_stop(struct rte_eth_dev *dev);
  64 static int nfp_fw_setup(struct rte_pci_device *dev,
  65                         struct nfp_cpp *cpp,
  66                         struct nfp_eth_table *nfp_eth_table,
  67                         struct nfp_hwinfo *hwinfo);
  68
  69 static int
  70 __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update)
  71 {
  72         int cnt;
  73         uint32_t new;
  74         struct timespec wait;
  75
  76         PMD_DRV_LOG(DEBUG, "Writing to the configuration queue (%p)...",
  77                     hw->qcp_cfg);
  78
  79         if (hw->qcp_cfg == NULL)
  80                 rte_panic("Bad configuration queue pointer\n");
  81
  82         nfp_qcp_ptr_add(hw->qcp_cfg, NFP_QCP_WRITE_PTR, 1);
  83
  84         wait.tv_sec = 0;
  85         wait.tv_nsec = 1000000;
  86
  87         PMD_DRV_LOG(DEBUG, "Polling for update ack...");
  88
  89         /* Poll update field, waiting for NFP to ack the config */
  90         for (cnt = 0; ; cnt++) {
  91                 new = nn_cfg_readl(hw, NFP_NET_CFG_UPDATE);
  92                 if (new == 0)
  93                         break;
  94                 if (new & NFP_NET_CFG_UPDATE_ERR) {
  95                         PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new);
  96                         return -1;
  97                 }
  98                 if (cnt >= NFP_NET_POLL_TIMEOUT) {
  99                         PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after"
 100                                           " %dms", update, cnt);
 101                         rte_panic("Exiting\n");
 102                 }
 103                 nanosleep(&wait, 0); /* waiting for a 1ms */
 104         }
 105         PMD_DRV_LOG(DEBUG, "Ack DONE");
 106         return 0;
 107 }
 108
 109 /*
 110  * Reconfigure the NIC
 111  * @nn:    device to reconfigure
 112  * @ctrl:    The value for the ctrl field in the BAR config
 113  * @update:  The value for the update field in the BAR config
 114  *
 115  * Write the update word to the BAR and ping the reconfig queue. Then poll
 116  * until the firmware has acknowledged the update by zeroing the update word.
 117  */
 118 int
 119 nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update)
 120 {
 121         uint32_t err;
 122
 123         PMD_DRV_LOG(DEBUG, "nfp_net_reconfig: ctrl=%08x update=%08x",
 124                     ctrl, update);
 125
 126         rte_spinlock_lock(&hw->reconfig_lock);
 127
 128         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, ctrl);
 129         nn_cfg_writel(hw, NFP_NET_CFG_UPDATE, update);
 130
 131         rte_wmb();
 132
 133         err = __nfp_net_reconfig(hw, update);
 134
 135         rte_spinlock_unlock(&hw->reconfig_lock);
 136
 137         if (!err)
 138                 return 0;
 139
 140         /*
 141          * Reconfig errors imply situations where they can be handled.
 142          * Otherwise, rte_panic is called inside __nfp_net_reconfig
 143          */
 144         PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x",
 145                      ctrl, update);
 146         return -EIO;
 147 }
 148
 149 /*
 150  * Configure an Ethernet device. This function must be invoked first
 151  * before any other function in the Ethernet API. This function can
 152  * also be re-invoked when a device is in the stopped state.
 153  */
 154 int
 155 nfp_net_configure(struct rte_eth_dev *dev)
 156 {
 157         struct rte_eth_conf *dev_conf;
 158         struct rte_eth_rxmode *rxmode;
 159         struct rte_eth_txmode *txmode;
 160         struct nfp_net_hw *hw;
 161
 162         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 163
 164         /*
 165          * A DPDK app sends info about how many queues to use and how
 166          * those queues need to be configured. This is used by the
 167          * DPDK core and it makes sure no more queues than those
 168          * advertised by the driver are requested. This function is
 169          * called after that internal process
 170          */
 171
 172         PMD_INIT_LOG(DEBUG, "Configure");
 173
 174         dev_conf = &dev->data->dev_conf;
 175         rxmode = &dev_conf->rxmode;
 176         txmode = &dev_conf->txmode;
 177
 178         if (rxmode->mq_mode & ETH_MQ_RX_RSS_FLAG)
 179                 rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 180
 181         /* Checking TX mode */
 182         if (txmode->mq_mode) {
 183                 PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported");
 184                 return -EINVAL;
 185         }
 186
 187         /* Checking RX mode */
 188         if (rxmode->mq_mode & ETH_MQ_RX_RSS &&
 189             !(hw->cap & NFP_NET_CFG_CTRL_RSS)) {
 190                 PMD_INIT_LOG(INFO, "RSS not supported");
 191                 return -EINVAL;
 192         }
 193
 194         return 0;
 195 }
 196
 197 void
 198 nfp_net_enable_queues(struct rte_eth_dev *dev)
 199 {
 200         struct nfp_net_hw *hw;
 201         uint64_t enabled_queues = 0;
 202         int i;
 203
 204         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 205
 206         /* Enabling the required TX queues in the device */
 207         for (i = 0; i < dev->data->nb_tx_queues; i++)
 208                 enabled_queues |= (1 << i);
 209
 210         nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, enabled_queues);
 211
 212         enabled_queues = 0;
 213
 214         /* Enabling the required RX queues in the device */
 215         for (i = 0; i < dev->data->nb_rx_queues; i++)
 216                 enabled_queues |= (1 << i);
 217
 218         nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, enabled_queues);
 219 }
 220
 221 void
 222 nfp_net_disable_queues(struct rte_eth_dev *dev)
 223 {
 224         struct nfp_net_hw *hw;
 225         uint32_t new_ctrl, update = 0;
 226
 227         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 228
 229         nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, 0);
 230         nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, 0);
 231
 232         new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_ENABLE;
 233         update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING |
 234                  NFP_NET_CFG_UPDATE_MSIX;
 235
 236         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 237                 new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG;
 238
 239         /* If an error when reconfig we avoid to change hw state */
 240         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 241                 return;
 242
 243         hw->ctrl = new_ctrl;
 244 }
 245
 246 void
 247 nfp_net_params_setup(struct nfp_net_hw *hw)
 248 {
 249         nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu);
 250         nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz);
 251 }
 252
 253 void
 254 nfp_net_cfg_queue_setup(struct nfp_net_hw *hw)
 255 {
 256         hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ;
 257 }
 258
 259 #define ETH_ADDR_LEN    6
 260
 261 void
 262 nfp_eth_copy_mac(uint8_t *dst, const uint8_t *src)
 263 {
 264         int i;
 265
 266         for (i = 0; i < ETH_ADDR_LEN; i++)
 267                 dst[i] = src[i];
 268 }
 269
 270 static int
 271 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port)
 272 {
 273         struct nfp_eth_table *nfp_eth_table;
 274         struct nfp_net_hw *hw = NULL;
 275
 276         /* Grab a pointer to the correct physical port */
 277         hw = pf_dev->ports[port];
 278
 279         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
 280
 281         nfp_eth_copy_mac((uint8_t *)&hw->mac_addr,
 282                          (uint8_t *)&nfp_eth_table->ports[port].mac_addr);
 283
 284         free(nfp_eth_table);
 285         return 0;
 286 }
 287
 288 static void
 289 nfp_net_vf_read_mac(struct nfp_net_hw *hw)
 290 {
 291         uint32_t tmp;
 292
 293         tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR));
 294         memcpy(&hw->mac_addr[0], &tmp, 4);
 295
 296         tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4));
 297         memcpy(&hw->mac_addr[4], &tmp, 2);
 298 }
 299
 300 void
 301 nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac)
 302 {
 303         uint32_t mac0 = *(uint32_t *)mac;
 304         uint16_t mac1;
 305
 306         nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR);
 307
 308         mac += 4;
 309         mac1 = *(uint16_t *)mac;
 310         nn_writew(rte_cpu_to_be_16(mac1),
 311                   hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6);
 312 }
 313
 314 int
 315 nfp_set_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
 316 {
 317         struct nfp_net_hw *hw;
 318         uint32_t update, ctrl;
 319
 320         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 321         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 322             !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) {
 323                 PMD_INIT_LOG(INFO, "MAC address unable to change when"
 324                                   " port enabled");
 325                 return -EBUSY;
 326         }
 327
 328         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 329             !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 330                 return -EBUSY;
 331
 332         /* Writing new MAC to the specific port BAR address */
 333         nfp_net_write_mac(hw, (uint8_t *)mac_addr);
 334
 335         /* Signal the NIC about the change */
 336         update = NFP_NET_CFG_UPDATE_MACADDR;
 337         ctrl = hw->ctrl;
 338         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 339             (hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 340                 ctrl |= NFP_NET_CFG_CTRL_LIVE_ADDR;
 341         if (nfp_net_reconfig(hw, ctrl, update) < 0) {
 342                 PMD_INIT_LOG(INFO, "MAC address update failed");
 343                 return -EIO;
 344         }
 345         return 0;
 346 }
 347
 348 int
 349 nfp_configure_rx_interrupt(struct rte_eth_dev *dev,
 350                            struct rte_intr_handle *intr_handle)
 351 {
 352         struct nfp_net_hw *hw;
 353         int i;
 354
 355         if (!intr_handle->intr_vec) {
 356                 intr_handle->intr_vec =
 357                         rte_zmalloc("intr_vec",
 358                                     dev->data->nb_rx_queues * sizeof(int), 0);
 359                 if (!intr_handle->intr_vec) {
 360                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
 361                                      " intr_vec", dev->data->nb_rx_queues);
 362                         return -ENOMEM;
 363                 }
 364         }
 365
 366         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 367
 368         if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 369                 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO");
 370                 /* UIO just supports one queue and no LSC*/
 371                 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0);
 372                 intr_handle->intr_vec[0] = 0;
 373         } else {
 374                 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO");
 375                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 376                         /*
 377                          * The first msix vector is reserved for non
 378                          * efd interrupts
 379                         */
 380                         nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1);
 381                         intr_handle->intr_vec[i] = i + 1;
 382                         PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d", i,
 383                                             intr_handle->intr_vec[i]);
 384                 }
 385         }
 386
 387         /* Avoiding TX interrupts */
 388         hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF;
 389         return 0;
 390 }
 391
 392 uint32_t
 393 nfp_check_offloads(struct rte_eth_dev *dev)
 394 {
 395         struct nfp_net_hw *hw;
 396         struct rte_eth_conf *dev_conf;
 397         struct rte_eth_rxmode *rxmode;
 398         struct rte_eth_txmode *txmode;
 399         uint32_t ctrl = 0;
 400
 401         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 402
 403         dev_conf = &dev->data->dev_conf;
 404         rxmode = &dev_conf->rxmode;
 405         txmode = &dev_conf->txmode;
 406
 407         if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
 408                 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM)
 409                         ctrl |= NFP_NET_CFG_CTRL_RXCSUM;
 410         }
 411
 412         if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) {
 413                 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN)
 414                         ctrl |= NFP_NET_CFG_CTRL_RXVLAN;
 415         }
 416
 417         if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 418                 hw->mtu = rxmode->max_rx_pkt_len;
 419
 420         if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT)
 421                 ctrl |= NFP_NET_CFG_CTRL_TXVLAN;
 422
 423         /* L2 broadcast */
 424         if (hw->cap & NFP_NET_CFG_CTRL_L2BC)
 425                 ctrl |= NFP_NET_CFG_CTRL_L2BC;
 426
 427         /* L2 multicast */
 428         if (hw->cap & NFP_NET_CFG_CTRL_L2MC)
 429                 ctrl |= NFP_NET_CFG_CTRL_L2MC;
 430
 431         /* TX checksum offload */
 432         if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM ||
 433             txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM ||
 434             txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
 435                 ctrl |= NFP_NET_CFG_CTRL_TXCSUM;
 436
 437         /* LSO offload */
 438         if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) {
 439                 if (hw->cap & NFP_NET_CFG_CTRL_LSO)
 440                         ctrl |= NFP_NET_CFG_CTRL_LSO;
 441                 else
 442                         ctrl |= NFP_NET_CFG_CTRL_LSO2;
 443         }
 444
 445         /* RX gather */
 446         if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 447                 ctrl |= NFP_NET_CFG_CTRL_GATHER;
 448
 449         return ctrl;
 450 }
 451
 452 static int
 453 nfp_net_start(struct rte_eth_dev *dev)
 454 {
 455         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 456         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 457         uint32_t new_ctrl, update = 0;
 458         struct nfp_net_hw *hw;
 459         struct nfp_pf_dev *pf_dev;
 460         struct rte_eth_conf *dev_conf;
 461         struct rte_eth_rxmode *rxmode;
 462         uint32_t intr_vector;
 463         int ret;
 464
 465         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 466         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 467
 468         PMD_INIT_LOG(DEBUG, "Start");
 469
 470         /* Disabling queues just in case... */
 471         nfp_net_disable_queues(dev);
 472
 473         /* Enabling the required queues in the device */
 474         nfp_net_enable_queues(dev);
 475
 476         /* check and configure queue intr-vector mapping */
 477         if (dev->data->dev_conf.intr_conf.rxq != 0) {
 478                 if (pf_dev->multiport) {
 479                         PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported "
 480                                           "with NFP multiport PF");
 481                                 return -EINVAL;
 482                 }
 483                 if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 484                         /*
 485                          * Better not to share LSC with RX interrupts.
 486                          * Unregistering LSC interrupt handler
 487                          */
 488                         rte_intr_callback_unregister(&pci_dev->intr_handle,
 489                                 nfp_net_dev_interrupt_handler, (void *)dev);
 490
 491                         if (dev->data->nb_rx_queues > 1) {
 492                                 PMD_INIT_LOG(ERR, "PMD rx interrupt only "
 493                                              "supports 1 queue with UIO");
 494                                 return -EIO;
 495                         }
 496                 }
 497                 intr_vector = dev->data->nb_rx_queues;
 498                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 499                         return -1;
 500
 501                 nfp_configure_rx_interrupt(dev, intr_handle);
 502                 update = NFP_NET_CFG_UPDATE_MSIX;
 503         }
 504
 505         rte_intr_enable(intr_handle);
 506
 507         new_ctrl = nfp_check_offloads(dev);
 508
 509         /* Writing configuration parameters in the device */
 510         nfp_net_params_setup(hw);
 511
 512         dev_conf = &dev->data->dev_conf;
 513         rxmode = &dev_conf->rxmode;
 514
 515         if (rxmode->mq_mode & ETH_MQ_RX_RSS) {
 516                 nfp_net_rss_config_default(dev);
 517                 update |= NFP_NET_CFG_UPDATE_RSS;
 518                 new_ctrl |= NFP_NET_CFG_CTRL_RSS;
 519         }
 520
 521         /* Enable device */
 522         new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
 523
 524         update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING;
 525
 526         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 527                 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;
 528
 529         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl);
 530         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 531                 return -EIO;
 532
 533         /*
 534          * Allocating rte mbufs for configured rx queues.
 535          * This requires queues being enabled before
 536          */
 537         if (nfp_net_rx_freelist_setup(dev) < 0) {
 538                 ret = -ENOMEM;
 539                 goto error;
 540         }
 541
 542         if (hw->is_phyport) {
 543                 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 544                         /* Configure the physical port up */
 545                         nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 546                 else
 547                         nfp_eth_set_configured(dev->process_private,
 548                                                hw->nfp_idx, 1);
 549         }
 550
 551         hw->ctrl = new_ctrl;
 552
 553         return 0;
 554
 555 error:
 556         /*
 557          * An error returned by this function should mean the app
 558          * exiting and then the system releasing all the memory
 559          * allocated even memory coming from hugepages.
 560          *
 561          * The device could be enabled at this point with some queues
 562          * ready for getting packets. This is true if the call to
 563          * nfp_net_rx_freelist_setup() succeeds for some queues but
 564          * fails for subsequent queues.
 565          *
 566          * This should make the app exiting but better if we tell the
 567          * device first.
 568          */
 569         nfp_net_disable_queues(dev);
 570
 571         return ret;
 572 }
 573
 574 /* Stop device: disable rx and tx functions to allow for reconfiguring. */
 575 static int
 576 nfp_net_stop(struct rte_eth_dev *dev)
 577 {
 578         int i;
 579         struct nfp_net_hw *hw;
 580
 581         PMD_INIT_LOG(DEBUG, "Stop");
 582
 583         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 584
 585         nfp_net_disable_queues(dev);
 586
 587         /* Clear queues */
 588         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 589                 nfp_net_reset_tx_queue(
 590                         (struct nfp_net_txq *)dev->data->tx_queues[i]);
 591         }
 592
 593         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 594                 nfp_net_reset_rx_queue(
 595                         (struct nfp_net_rxq *)dev->data->rx_queues[i]);
 596         }
 597
 598         if (hw->is_phyport) {
 599                 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 600                         /* Configure the physical port down */
 601                         nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 602                 else
 603                         nfp_eth_set_configured(dev->process_private,
 604                                                hw->nfp_idx, 0);
 605         }
 606
 607         return 0;
 608 }
 609
 610 /* Set the link up. */
 611 static int
 612 nfp_net_set_link_up(struct rte_eth_dev *dev)
 613 {
 614         struct nfp_net_hw *hw;
 615
 616         PMD_DRV_LOG(DEBUG, "Set link up");
 617
 618         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 619
 620         if (!hw->is_phyport)
 621                 return -ENOTSUP;
 622
 623         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 624                 /* Configure the physical port down */
 625                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 626         else
 627                 return nfp_eth_set_configured(dev->process_private,
 628                                               hw->nfp_idx, 1);
 629 }
 630
 631 /* Set the link down. */
 632 static int
 633 nfp_net_set_link_down(struct rte_eth_dev *dev)
 634 {
 635         struct nfp_net_hw *hw;
 636
 637         PMD_DRV_LOG(DEBUG, "Set link down");
 638
 639         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 640
 641         if (!hw->is_phyport)
 642                 return -ENOTSUP;
 643
 644         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 645                 /* Configure the physical port down */
 646                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 647         else
 648                 return nfp_eth_set_configured(dev->process_private,
 649                                               hw->nfp_idx, 0);
 650 }
 651
 652 /* Reset and stop device. The device can not be restarted. */
 653 static int
 654 nfp_net_close(struct rte_eth_dev *dev)
 655 {
 656         struct nfp_net_hw *hw;
 657         struct rte_pci_device *pci_dev;
 658         int i;
 659
 660         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 661                 return 0;
 662
 663         PMD_INIT_LOG(DEBUG, "Close");
 664
 665         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 666         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 667
 668         /*
 669          * We assume that the DPDK application is stopping all the
 670          * threads/queues before calling the device close function.
 671          */
 672
 673         nfp_net_disable_queues(dev);
 674
 675         /* Clear queues */
 676         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 677                 nfp_net_reset_tx_queue(
 678                         (struct nfp_net_txq *)dev->data->tx_queues[i]);
 679         }
 680
 681         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 682                 nfp_net_reset_rx_queue(
 683                         (struct nfp_net_rxq *)dev->data->rx_queues[i]);
 684         }
 685
 686         /* Only free PF resources after all physical ports have been closed */
 687         if (pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC ||
 688             pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC) {
 689                 struct nfp_pf_dev *pf_dev;
 690                 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 691
 692                 /* Mark this port as unused and free device priv resources*/
 693                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff);
 694                 pf_dev->ports[hw->idx] = NULL;
 695                 rte_eth_dev_release_port(dev);
 696
 697                 for (i = 0; i < pf_dev->total_phyports; i++) {
 698                         /* Check to see if ports are still in use */
 699                         if (pf_dev->ports[i])
 700                                 return 0;
 701                 }
 702
 703                 /* Now it is safe to free all PF resources */
 704                 PMD_INIT_LOG(INFO, "Freeing PF resources");
 705                 nfp_cpp_area_free(pf_dev->ctrl_area);
 706                 nfp_cpp_area_free(pf_dev->hwqueues_area);
 707                 free(pf_dev->hwinfo);
 708                 free(pf_dev->sym_tbl);
 709                 nfp_cpp_free(pf_dev->cpp);
 710                 rte_free(pf_dev);
 711         }
 712
 713         rte_intr_disable(&pci_dev->intr_handle);
 714
 715         /* unregister callback func from eal lib */
 716         rte_intr_callback_unregister(&pci_dev->intr_handle,
 717                                      nfp_net_dev_interrupt_handler,
 718                                      (void *)dev);
 719
 720         /*
 721          * The ixgbe PMD driver disables the pcie master on the
 722          * device. The i40e does not...
 723          */
 724
 725         return 0;
 726 }
 727
 728 int
 729 nfp_net_promisc_enable(struct rte_eth_dev *dev)
 730 {
 731         uint32_t new_ctrl, update = 0;
 732         struct nfp_net_hw *hw;
 733         int ret;
 734
 735         PMD_DRV_LOG(DEBUG, "Promiscuous mode enable");
 736
 737         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 738
 739         if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) {
 740                 PMD_INIT_LOG(INFO, "Promiscuous mode not supported");
 741                 return -ENOTSUP;
 742         }
 743
 744         if (hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) {
 745                 PMD_DRV_LOG(INFO, "Promiscuous mode already enabled");
 746                 return 0;
 747         }
 748
 749         new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_PROMISC;
 750         update = NFP_NET_CFG_UPDATE_GEN;
 751
 752         /*
 753          * DPDK sets promiscuous mode on just after this call assuming
 754          * it can not fail ...
 755          */
 756         ret = nfp_net_reconfig(hw, new_ctrl, update);
 757         if (ret < 0)
 758                 return ret;
 759
 760         hw->ctrl = new_ctrl;
 761
 762         return 0;
 763 }
 764
 765 int
 766 nfp_net_promisc_disable(struct rte_eth_dev *dev)
 767 {
 768         uint32_t new_ctrl, update = 0;
 769         struct nfp_net_hw *hw;
 770         int ret;
 771
 772         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 773
 774         if ((hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) == 0) {
 775                 PMD_DRV_LOG(INFO, "Promiscuous mode already disabled");
 776                 return 0;
 777         }
 778
 779         new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_PROMISC;
 780         update = NFP_NET_CFG_UPDATE_GEN;
 781
 782         /*
 783          * DPDK sets promiscuous mode off just before this call
 784          * assuming it can not fail ...
 785          */
 786         ret = nfp_net_reconfig(hw, new_ctrl, update);
 787         if (ret < 0)
 788                 return ret;
 789
 790         hw->ctrl = new_ctrl;
 791
 792         return 0;
 793 }
 794
 795 /*
 796  * return 0 means link status changed, -1 means not changed
 797  *
 798  * Wait to complete is needed as it can take up to 9 seconds to get the Link
 799  * status.
 800  */
 801 int
 802 nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
 803 {
 804         struct nfp_net_hw *hw;
 805         struct rte_eth_link link;
 806         uint32_t nn_link_status;
 807         int ret;
 808
 809         static const uint32_t ls_to_ethtool[] = {
 810                 [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE,
 811                 [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN]     = ETH_SPEED_NUM_NONE,
 812                 [NFP_NET_CFG_STS_LINK_RATE_1G]          = ETH_SPEED_NUM_1G,
 813                 [NFP_NET_CFG_STS_LINK_RATE_10G]         = ETH_SPEED_NUM_10G,
 814                 [NFP_NET_CFG_STS_LINK_RATE_25G]         = ETH_SPEED_NUM_25G,
 815                 [NFP_NET_CFG_STS_LINK_RATE_40G]         = ETH_SPEED_NUM_40G,
 816                 [NFP_NET_CFG_STS_LINK_RATE_50G]         = ETH_SPEED_NUM_50G,
 817                 [NFP_NET_CFG_STS_LINK_RATE_100G]        = ETH_SPEED_NUM_100G,
 818         };
 819
 820         PMD_DRV_LOG(DEBUG, "Link update");
 821
 822         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 823
 824         nn_link_status = nn_cfg_readl(hw, NFP_NET_CFG_STS);
 825
 826         memset(&link, 0, sizeof(struct rte_eth_link));
 827
 828         if (nn_link_status & NFP_NET_CFG_STS_LINK)
 829                 link.link_status = ETH_LINK_UP;
 830
 831         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 832
 833         nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) &
 834                          NFP_NET_CFG_STS_LINK_RATE_MASK;
 835
 836         if (nn_link_status >= RTE_DIM(ls_to_ethtool))
 837                 link.link_speed = ETH_SPEED_NUM_NONE;
 838         else
 839                 link.link_speed = ls_to_ethtool[nn_link_status];
 840
 841         ret = rte_eth_linkstatus_set(dev, &link);
 842         if (ret == 0) {
 843                 if (link.link_status)
 844                         PMD_DRV_LOG(INFO, "NIC Link is Up");
 845                 else
 846                         PMD_DRV_LOG(INFO, "NIC Link is Down");
 847         }
 848         return ret;
 849 }
 850
 851 int
 852 nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 853 {
 854         int i;
 855         struct nfp_net_hw *hw;
 856         struct rte_eth_stats nfp_dev_stats;
 857
 858         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859
 860         /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */
 861
 862         memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats));
 863
 864         /* reading per RX ring stats */
 865         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 866                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 867                         break;
 868
 869                 nfp_dev_stats.q_ipackets[i] =
 870                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i));
 871
 872                 nfp_dev_stats.q_ipackets[i] -=
 873                         hw->eth_stats_base.q_ipackets[i];
 874
 875                 nfp_dev_stats.q_ibytes[i] =
 876                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8);
 877
 878                 nfp_dev_stats.q_ibytes[i] -=
 879                         hw->eth_stats_base.q_ibytes[i];
 880         }
 881
 882         /* reading per TX ring stats */
 883         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 884                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 885                         break;
 886
 887                 nfp_dev_stats.q_opackets[i] =
 888                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i));
 889
 890                 nfp_dev_stats.q_opackets[i] -=
 891                         hw->eth_stats_base.q_opackets[i];
 892
 893                 nfp_dev_stats.q_obytes[i] =
 894                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8);
 895
 896                 nfp_dev_stats.q_obytes[i] -=
 897                         hw->eth_stats_base.q_obytes[i];
 898         }
 899
 900         nfp_dev_stats.ipackets =
 901                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES);
 902
 903         nfp_dev_stats.ipackets -= hw->eth_stats_base.ipackets;
 904
 905         nfp_dev_stats.ibytes =
 906                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS);
 907
 908         nfp_dev_stats.ibytes -= hw->eth_stats_base.ibytes;
 909
 910         nfp_dev_stats.opackets =
 911                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES);
 912
 913         nfp_dev_stats.opackets -= hw->eth_stats_base.opackets;
 914
 915         nfp_dev_stats.obytes =
 916                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS);
 917
 918         nfp_dev_stats.obytes -= hw->eth_stats_base.obytes;
 919
 920         /* reading general device stats */
 921         nfp_dev_stats.ierrors =
 922                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS);
 923
 924         nfp_dev_stats.ierrors -= hw->eth_stats_base.ierrors;
 925
 926         nfp_dev_stats.oerrors =
 927                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS);
 928
 929         nfp_dev_stats.oerrors -= hw->eth_stats_base.oerrors;
 930
 931         /* RX ring mbuf allocation failures */
 932         nfp_dev_stats.rx_nombuf = dev->data->rx_mbuf_alloc_failed;
 933
 934         nfp_dev_stats.imissed =
 935                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS);
 936
 937         nfp_dev_stats.imissed -= hw->eth_stats_base.imissed;
 938
 939         if (stats) {
 940                 memcpy(stats, &nfp_dev_stats, sizeof(*stats));
 941                 return 0;
 942         }
 943         return -EINVAL;
 944 }
 945
 946 int
 947 nfp_net_stats_reset(struct rte_eth_dev *dev)
 948 {
 949         int i;
 950         struct nfp_net_hw *hw;
 951
 952         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 953
 954         /*
 955          * hw->eth_stats_base records the per counter starting point.
 956          * Lets update it now
 957          */
 958
 959         /* reading per RX ring stats */
 960         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 961                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 962                         break;
 963
 964                 hw->eth_stats_base.q_ipackets[i] =
 965                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i));
 966
 967                 hw->eth_stats_base.q_ibytes[i] =
 968                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8);
 969         }
 970
 971         /* reading per TX ring stats */
 972         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 973                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 974                         break;
 975
 976                 hw->eth_stats_base.q_opackets[i] =
 977                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i));
 978
 979                 hw->eth_stats_base.q_obytes[i] =
 980                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8);
 981         }
 982
 983         hw->eth_stats_base.ipackets =
 984                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES);
 985
 986         hw->eth_stats_base.ibytes =
 987                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS);
 988
 989         hw->eth_stats_base.opackets =
 990                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES);
 991
 992         hw->eth_stats_base.obytes =
 993                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS);
 994
 995         /* reading general device stats */
 996         hw->eth_stats_base.ierrors =
 997                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS);
 998
 999         hw->eth_stats_base.oerrors =
1000                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS);
1001
1002         /* RX ring mbuf allocation failures */
1003         dev->data->rx_mbuf_alloc_failed = 0;
1004
1005         hw->eth_stats_base.imissed =
1006                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS);
1007
1008         return 0;
1009 }
1010
1011 int
1012 nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1013 {
1014         struct nfp_net_hw *hw;
1015
1016         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1017
1018         dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues;
1019         dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues;
1020         dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
1021         dev_info->max_rx_pktlen = hw->max_mtu;
1022         /* Next should change when PF support is implemented */
1023         dev_info->max_mac_addrs = 1;
1024
1025         if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN)
1026                 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1027
1028         if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM)
1029                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM |
1030                                              DEV_RX_OFFLOAD_UDP_CKSUM |
1031                                              DEV_RX_OFFLOAD_TCP_CKSUM;
1032
1033         if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)
1034                 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
1035
1036         if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM)
1037                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM |
1038                                              DEV_TX_OFFLOAD_UDP_CKSUM |
1039                                              DEV_TX_OFFLOAD_TCP_CKSUM;
1040
1041         if (hw->cap & NFP_NET_CFG_CTRL_LSO_ANY)
1042                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
1043
1044         if (hw->cap & NFP_NET_CFG_CTRL_GATHER)
1045                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MULTI_SEGS;
1046
1047         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1048                 .rx_thresh = {
1049                         .pthresh = DEFAULT_RX_PTHRESH,
1050                         .hthresh = DEFAULT_RX_HTHRESH,
1051                         .wthresh = DEFAULT_RX_WTHRESH,
1052                 },
1053                 .rx_free_thresh = DEFAULT_RX_FREE_THRESH,
1054                 .rx_drop_en = 0,
1055         };
1056
1057         dev_info->default_txconf = (struct rte_eth_txconf) {
1058                 .tx_thresh = {
1059                         .pthresh = DEFAULT_TX_PTHRESH,
1060                         .hthresh = DEFAULT_TX_HTHRESH,
1061                         .wthresh = DEFAULT_TX_WTHRESH,
1062                 },
1063                 .tx_free_thresh = DEFAULT_TX_FREE_THRESH,
1064                 .tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH,
1065         };
1066
1067         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1068                 .nb_max = NFP_NET_MAX_RX_DESC,
1069                 .nb_min = NFP_NET_MIN_RX_DESC,
1070                 .nb_align = NFP_ALIGN_RING_DESC,
1071         };
1072
1073         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1074                 .nb_max = NFP_NET_MAX_TX_DESC,
1075                 .nb_min = NFP_NET_MIN_TX_DESC,
1076                 .nb_align = NFP_ALIGN_RING_DESC,
1077                 .nb_seg_max = NFP_TX_MAX_SEG,
1078                 .nb_mtu_seg_max = NFP_TX_MAX_MTU_SEG,
1079         };
1080
1081         /* All NFP devices support jumbo frames */
1082         dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
1083
1084         if (hw->cap & NFP_NET_CFG_CTRL_RSS) {
1085                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_RSS_HASH;
1086
1087                 dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1088                                                    ETH_RSS_NONFRAG_IPV4_TCP |
1089                                                    ETH_RSS_NONFRAG_IPV4_UDP |
1090                                                    ETH_RSS_IPV6 |
1091                                                    ETH_RSS_NONFRAG_IPV6_TCP |
1092                                                    ETH_RSS_NONFRAG_IPV6_UDP;
1093
1094                 dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ;
1095                 dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ;
1096         }
1097
1098         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G |
1099                                ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G |
1100                                ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G;
1101
1102         return 0;
1103 }
1104
1105 const uint32_t *
1106 nfp_net_supported_ptypes_get(struct rte_eth_dev *dev)
1107 {
1108         static const uint32_t ptypes[] = {
1109                 /* refers to nfp_net_set_hash() */
1110                 RTE_PTYPE_INNER_L3_IPV4,
1111                 RTE_PTYPE_INNER_L3_IPV6,
1112                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1113                 RTE_PTYPE_INNER_L4_MASK,
1114                 RTE_PTYPE_UNKNOWN
1115         };
1116
1117         if (dev->rx_pkt_burst == nfp_net_recv_pkts)
1118                 return ptypes;
1119         return NULL;
1120 }
1121
1122 int
1123 nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1124 {
1125         struct rte_pci_device *pci_dev;
1126         struct nfp_net_hw *hw;
1127         int base = 0;
1128
1129         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1130         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1131
1132         if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO)
1133                 base = 1;
1134
1135         /* Make sure all updates are written before un-masking */
1136         rte_wmb();
1137         nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id),
1138                       NFP_NET_CFG_ICR_UNMASKED);
1139         return 0;
1140 }
1141
1142 int
1143 nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1144 {
1145         struct rte_pci_device *pci_dev;
1146         struct nfp_net_hw *hw;
1147         int base = 0;
1148
1149         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1150         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1151
1152         if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO)
1153                 base = 1;
1154
1155         /* Make sure all updates are written before un-masking */
1156         rte_wmb();
1157         nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1);
1158         return 0;
1159 }
1160
1161 static void
1162 nfp_net_dev_link_status_print(struct rte_eth_dev *dev)
1163 {
1164         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1165         struct rte_eth_link link;
1166
1167         rte_eth_linkstatus_get(dev, &link);
1168         if (link.link_status)
1169                 PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1170                             dev->data->port_id, link.link_speed,
1171                             link.link_duplex == ETH_LINK_FULL_DUPLEX
1172                             ? "full-duplex" : "half-duplex");
1173         else
1174                 PMD_DRV_LOG(INFO, " Port %d: Link Down",
1175                             dev->data->port_id);
1176
1177         PMD_DRV_LOG(INFO, "PCI Address: " PCI_PRI_FMT,
1178                     pci_dev->addr.domain, pci_dev->addr.bus,
1179                     pci_dev->addr.devid, pci_dev->addr.function);
1180 }
1181
1182 /* Interrupt configuration and handling */
1183
1184 /*
1185  * nfp_net_irq_unmask - Unmask an interrupt
1186  *
1187  * If MSI-X auto-masking is enabled clear the mask bit, otherwise
1188  * clear the ICR for the entry.
1189  */
1190 static void
1191 nfp_net_irq_unmask(struct rte_eth_dev *dev)
1192 {
1193         struct nfp_net_hw *hw;
1194         struct rte_pci_device *pci_dev;
1195
1196         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1197         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1198
1199         if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) {
1200                 /* If MSI-X auto-masking is used, clear the entry */
1201                 rte_wmb();
1202                 rte_intr_ack(&pci_dev->intr_handle);
1203         } else {
1204                 /* Make sure all updates are written before un-masking */
1205                 rte_wmb();
1206                 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(NFP_NET_IRQ_LSC_IDX),
1207                               NFP_NET_CFG_ICR_UNMASKED);
1208         }
1209 }
1210
1211 /*
1212  * Interrupt handler which shall be registered for alarm callback for delayed
1213  * handling specific interrupt to wait for the stable nic state. As the NIC
1214  * interrupt state is not stable for nfp after link is just down, it needs
1215  * to wait 4 seconds to get the stable status.
1216  *
1217  * @param handle   Pointer to interrupt handle.
1218  * @param param    The address of parameter (struct rte_eth_dev *)
1219  *
1220  * @return  void
1221  */
1222 static void
1223 nfp_net_dev_interrupt_delayed_handler(void *param)
1224 {
1225         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1226
1227         nfp_net_link_update(dev, 0);
1228         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1229
1230         nfp_net_dev_link_status_print(dev);
1231
1232         /* Unmasking */
1233         nfp_net_irq_unmask(dev);
1234 }
1235
1236 void
1237 nfp_net_dev_interrupt_handler(void *param)
1238 {
1239         int64_t timeout;
1240         struct rte_eth_link link;
1241         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1242
1243         PMD_DRV_LOG(DEBUG, "We got a LSC interrupt!!!");
1244
1245         rte_eth_linkstatus_get(dev, &link);
1246
1247         nfp_net_link_update(dev, 0);
1248
1249         /* likely to up */
1250         if (!link.link_status) {
1251                 /* handle it 1 sec later, wait it being stable */
1252                 timeout = NFP_NET_LINK_UP_CHECK_TIMEOUT;
1253                 /* likely to down */
1254         } else {
1255                 /* handle it 4 sec later, wait it being stable */
1256                 timeout = NFP_NET_LINK_DOWN_CHECK_TIMEOUT;
1257         }
1258
1259         if (rte_eal_alarm_set(timeout * 1000,
1260                               nfp_net_dev_interrupt_delayed_handler,
1261                               (void *)dev) < 0) {
1262                 PMD_INIT_LOG(ERR, "Error setting alarm");
1263                 /* Unmasking */
1264                 nfp_net_irq_unmask(dev);
1265         }
1266 }
1267
1268 int
1269 nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1270 {
1271         struct nfp_net_hw *hw;
1272
1273         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1274
1275         /* check that mtu is within the allowed range */
1276         if (mtu < RTE_ETHER_MIN_MTU || (uint32_t)mtu > hw->max_mtu)
1277                 return -EINVAL;
1278
1279         /* mtu setting is forbidden if port is started */
1280         if (dev->data->dev_started) {
1281                 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration",
1282                             dev->data->port_id);
1283                 return -EBUSY;
1284         }
1285
1286         /* switch to jumbo mode if needed */
1287         if ((uint32_t)mtu > RTE_ETHER_MTU)
1288                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
1289         else
1290                 dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1291
1292         /* update max frame size */
1293         dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu;
1294
1295         /* writing to configuration space */
1296         nn_cfg_writel(hw, NFP_NET_CFG_MTU, (uint32_t)mtu);
1297
1298         hw->mtu = mtu;
1299
1300         return 0;
1301 }
1302
1303 int
1304 nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1305 {
1306         uint32_t new_ctrl, update;
1307         struct nfp_net_hw *hw;
1308         int ret;
1309
1310         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1311         new_ctrl = 0;
1312
1313         /* Enable vlan strip if it is not configured yet */
1314         if ((mask & ETH_VLAN_STRIP_OFFLOAD) &&
1315             !(hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN))
1316                 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_RXVLAN;
1317
1318         /* Disable vlan strip just if it is configured */
1319         if (!(mask & ETH_VLAN_STRIP_OFFLOAD) &&
1320             (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN))
1321                 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN;
1322
1323         if (new_ctrl == 0)
1324                 return 0;
1325
1326         update = NFP_NET_CFG_UPDATE_GEN;
1327
1328         ret = nfp_net_reconfig(hw, new_ctrl, update);
1329         if (!ret)
1330                 hw->ctrl = new_ctrl;
1331
1332         return ret;
1333 }
1334
1335 static int
1336 nfp_net_rss_reta_write(struct rte_eth_dev *dev,
1337                     struct rte_eth_rss_reta_entry64 *reta_conf,
1338                     uint16_t reta_size)
1339 {
1340         uint32_t reta, mask;
1341         int i, j;
1342         int idx, shift;
1343         struct nfp_net_hw *hw =
1344                 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1345
1346         if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) {
1347                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
1348                         "(%d) doesn't match the number hardware can supported "
1349                         "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ);
1350                 return -EINVAL;
1351         }
1352
1353         /*
1354          * Update Redirection Table. There are 128 8bit-entries which can be
1355          * manage as 32 32bit-entries
1356          */
1357         for (i = 0; i < reta_size; i += 4) {
1358                 /* Handling 4 RSS entries per loop */
1359                 idx = i / RTE_RETA_GROUP_SIZE;
1360                 shift = i % RTE_RETA_GROUP_SIZE;
1361                 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF);
1362
1363                 if (!mask)
1364                         continue;
1365
1366                 reta = 0;
1367                 /* If all 4 entries were set, don't need read RETA register */
1368                 if (mask != 0xF)
1369                         reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + i);
1370
1371                 for (j = 0; j < 4; j++) {
1372                         if (!(mask & (0x1 << j)))
1373                                 continue;
1374                         if (mask != 0xF)
1375                                 /* Clearing the entry bits */
1376                                 reta &= ~(0xFF << (8 * j));
1377                         reta |= reta_conf[idx].reta[shift + j] << (8 * j);
1378                 }
1379                 nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift,
1380                               reta);
1381         }
1382         return 0;
1383 }
1384
1385 /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */
1386 int
1387 nfp_net_reta_update(struct rte_eth_dev *dev,
1388                     struct rte_eth_rss_reta_entry64 *reta_conf,
1389                     uint16_t reta_size)
1390 {
1391         struct nfp_net_hw *hw =
1392                 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1393         uint32_t update;
1394         int ret;
1395
1396         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1397                 return -EINVAL;
1398
1399         ret = nfp_net_rss_reta_write(dev, reta_conf, reta_size);
1400         if (ret != 0)
1401                 return ret;
1402
1403         update = NFP_NET_CFG_UPDATE_RSS;
1404
1405         if (nfp_net_reconfig(hw, hw->ctrl, update) < 0)
1406                 return -EIO;
1407
1408         return 0;
1409 }
1410
1411  /* Query Redirection Table(RETA) of Receive Side Scaling of Ethernet device. */
1412 int
1413 nfp_net_reta_query(struct rte_eth_dev *dev,
1414                    struct rte_eth_rss_reta_entry64 *reta_conf,
1415                    uint16_t reta_size)
1416 {
1417         uint8_t i, j, mask;
1418         int idx, shift;
1419         uint32_t reta;
1420         struct nfp_net_hw *hw;
1421
1422         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1423
1424         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1425                 return -EINVAL;
1426
1427         if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) {
1428                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
1429                         "(%d) doesn't match the number hardware can supported "
1430                         "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ);
1431                 return -EINVAL;
1432         }
1433
1434         /*
1435          * Reading Redirection Table. There are 128 8bit-entries which can be
1436          * manage as 32 32bit-entries
1437          */
1438         for (i = 0; i < reta_size; i += 4) {
1439                 /* Handling 4 RSS entries per loop */
1440                 idx = i / RTE_RETA_GROUP_SIZE;
1441                 shift = i % RTE_RETA_GROUP_SIZE;
1442                 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF);
1443
1444                 if (!mask)
1445                         continue;
1446
1447                 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) +
1448                                     shift);
1449                 for (j = 0; j < 4; j++) {
1450                         if (!(mask & (0x1 << j)))
1451                                 continue;
1452                         reta_conf[idx].reta[shift + j] =
1453                                 (uint8_t)((reta >> (8 * j)) & 0xF);
1454                 }
1455         }
1456         return 0;
1457 }
1458
1459 static int
1460 nfp_net_rss_hash_write(struct rte_eth_dev *dev,
1461                         struct rte_eth_rss_conf *rss_conf)
1462 {
1463         struct nfp_net_hw *hw;
1464         uint64_t rss_hf;
1465         uint32_t cfg_rss_ctrl = 0;
1466         uint8_t key;
1467         int i;
1468
1469         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1470
1471         /* Writing the key byte a byte */
1472         for (i = 0; i < rss_conf->rss_key_len; i++) {
1473                 memcpy(&key, &rss_conf->rss_key[i], 1);
1474                 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY + i, key);
1475         }
1476
1477         rss_hf = rss_conf->rss_hf;
1478
1479         if (rss_hf & ETH_RSS_IPV4)
1480                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4;
1481
1482         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1483                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_TCP;
1484
1485         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
1486                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_UDP;
1487
1488         if (rss_hf & ETH_RSS_IPV6)
1489                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6;
1490
1491         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1492                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_TCP;
1493
1494         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
1495                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_UDP;
1496
1497         cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK;
1498         cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ;
1499
1500         /* configuring where to apply the RSS hash */
1501         nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl);
1502
1503         /* Writing the key size */
1504         nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY_SZ, rss_conf->rss_key_len);
1505
1506         return 0;
1507 }
1508
1509 int
1510 nfp_net_rss_hash_update(struct rte_eth_dev *dev,
1511                         struct rte_eth_rss_conf *rss_conf)
1512 {
1513         uint32_t update;
1514         uint64_t rss_hf;
1515         struct nfp_net_hw *hw;
1516
1517         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1518
1519         rss_hf = rss_conf->rss_hf;
1520
1521         /* Checking if RSS is enabled */
1522         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) {
1523                 if (rss_hf != 0) { /* Enable RSS? */
1524                         PMD_DRV_LOG(ERR, "RSS unsupported");
1525                         return -EINVAL;
1526                 }
1527                 return 0; /* Nothing to do */
1528         }
1529
1530         if (rss_conf->rss_key_len > NFP_NET_CFG_RSS_KEY_SZ) {
1531                 PMD_DRV_LOG(ERR, "hash key too long");
1532                 return -EINVAL;
1533         }
1534
1535         nfp_net_rss_hash_write(dev, rss_conf);
1536
1537         update = NFP_NET_CFG_UPDATE_RSS;
1538
1539         if (nfp_net_reconfig(hw, hw->ctrl, update) < 0)
1540                 return -EIO;
1541
1542         return 0;
1543 }
1544
1545 int
1546 nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev,
1547                           struct rte_eth_rss_conf *rss_conf)
1548 {
1549         uint64_t rss_hf;
1550         uint32_t cfg_rss_ctrl;
1551         uint8_t key;
1552         int i;
1553         struct nfp_net_hw *hw;
1554
1555         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1556
1557         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1558                 return -EINVAL;
1559
1560         rss_hf = rss_conf->rss_hf;
1561         cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL);
1562
1563         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4)
1564                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP;
1565
1566         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP)
1567                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
1568
1569         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_TCP)
1570                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
1571
1572         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_UDP)
1573                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
1574
1575         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_UDP)
1576                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
1577
1578         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6)
1579                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP;
1580
1581         /* Propagate current RSS hash functions to caller */
1582         rss_conf->rss_hf = rss_hf;
1583
1584         /* Reading the key size */
1585         rss_conf->rss_key_len = nn_cfg_readl(hw, NFP_NET_CFG_RSS_KEY_SZ);
1586
1587         /* Reading the key byte a byte */
1588         for (i = 0; i < rss_conf->rss_key_len; i++) {
1589                 key = nn_cfg_readb(hw, NFP_NET_CFG_RSS_KEY + i);
1590                 memcpy(&rss_conf->rss_key[i], &key, 1);
1591         }
1592
1593         return 0;
1594 }
1595
1596 int
1597 nfp_net_rss_config_default(struct rte_eth_dev *dev)
1598 {
1599         struct rte_eth_conf *dev_conf;
1600         struct rte_eth_rss_conf rss_conf;
1601         struct rte_eth_rss_reta_entry64 nfp_reta_conf[2];
1602         uint16_t rx_queues = dev->data->nb_rx_queues;
1603         uint16_t queue;
1604         int i, j, ret;
1605
1606         PMD_DRV_LOG(INFO, "setting default RSS conf for %u queues",
1607                 rx_queues);
1608
1609         nfp_reta_conf[0].mask = ~0x0;
1610         nfp_reta_conf[1].mask = ~0x0;
1611
1612         queue = 0;
1613         for (i = 0; i < 0x40; i += 8) {
1614                 for (j = i; j < (i + 8); j++) {
1615                         nfp_reta_conf[0].reta[j] = queue;
1616                         nfp_reta_conf[1].reta[j] = queue++;
1617                         queue %= rx_queues;
1618                 }
1619         }
1620         ret = nfp_net_rss_reta_write(dev, nfp_reta_conf, 0x80);
1621         if (ret != 0)
1622                 return ret;
1623
1624         dev_conf = &dev->data->dev_conf;
1625         if (!dev_conf) {
1626                 PMD_DRV_LOG(INFO, "wrong rss conf");
1627                 return -EINVAL;
1628         }
1629         rss_conf = dev_conf->rx_adv_conf.rss_conf;
1630
1631         ret = nfp_net_rss_hash_write(dev, &rss_conf);
1632
1633         return ret;
1634 }
1635
1636
1637 /* Initialise and register driver with DPDK Application */
1638 static const struct eth_dev_ops nfp_net_eth_dev_ops = {
1639         .dev_configure          = nfp_net_configure,
1640         .dev_start              = nfp_net_start,
1641         .dev_stop               = nfp_net_stop,
1642         .dev_set_link_up        = nfp_net_set_link_up,
1643         .dev_set_link_down      = nfp_net_set_link_down,
1644         .dev_close              = nfp_net_close,
1645         .promiscuous_enable     = nfp_net_promisc_enable,
1646         .promiscuous_disable    = nfp_net_promisc_disable,
1647         .link_update            = nfp_net_link_update,
1648         .stats_get              = nfp_net_stats_get,
1649         .stats_reset            = nfp_net_stats_reset,
1650         .dev_infos_get          = nfp_net_infos_get,
1651         .dev_supported_ptypes_get = nfp_net_supported_ptypes_get,
1652         .mtu_set                = nfp_net_dev_mtu_set,
1653         .mac_addr_set           = nfp_set_mac_addr,
1654         .vlan_offload_set       = nfp_net_vlan_offload_set,
1655         .reta_update            = nfp_net_reta_update,
1656         .reta_query             = nfp_net_reta_query,
1657         .rss_hash_update        = nfp_net_rss_hash_update,
1658         .rss_hash_conf_get      = nfp_net_rss_hash_conf_get,
1659         .rx_queue_setup         = nfp_net_rx_queue_setup,
1660         .rx_queue_release       = nfp_net_rx_queue_release,
1661         .tx_queue_setup         = nfp_net_tx_queue_setup,
1662         .tx_queue_release       = nfp_net_tx_queue_release,
1663         .rx_queue_intr_enable   = nfp_rx_queue_intr_enable,
1664         .rx_queue_intr_disable  = nfp_rx_queue_intr_disable,
1665 };
1666
1667
1668 static int
1669 nfp_net_init(struct rte_eth_dev *eth_dev)
1670 {
1671         struct rte_pci_device *pci_dev;
1672         struct nfp_pf_dev *pf_dev;
1673         struct nfp_net_hw *hw;
1674
1675         uint64_t tx_bar_off = 0, rx_bar_off = 0;
1676         uint32_t start_q;
1677         int stride = 4;
1678         int port = 0;
1679         int err;
1680
1681         PMD_INIT_FUNC_TRACE();
1682
1683         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1684
1685         /* Use backpointer here to the PF of this eth_dev */
1686         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private);
1687
1688         /* NFP can not handle DMA addresses requiring more than 40 bits */
1689         if (rte_mem_check_dma_mask(40)) {
1690                 RTE_LOG(ERR, PMD, "device %s can not be used:",
1691                                    pci_dev->device.name);
1692                 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n");
1693                 return -ENODEV;
1694         };
1695
1696         if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) ||
1697             (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) {
1698                 port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx;
1699                 if (port < 0 || port > 7) {
1700                         PMD_DRV_LOG(ERR, "Port value is wrong");
1701                         return -ENODEV;
1702                 }
1703
1704                 /* Use PF array of physical ports to get pointer to
1705                  * this specific port
1706                  */
1707                 hw = pf_dev->ports[port];
1708
1709                 PMD_INIT_LOG(DEBUG, "Working with physical port number: %d, "
1710                                     "NFP internal port number: %d",
1711                                     port, hw->nfp_idx);
1712
1713         } else {
1714                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1715         }
1716
1717         eth_dev->dev_ops = &nfp_net_eth_dev_ops;
1718         eth_dev->rx_queue_count = nfp_net_rx_queue_count;
1719         eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
1720         eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
1721
1722         /* For secondary processes, the primary has done all the work */
1723         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1724                 return 0;
1725
1726         rte_eth_copy_pci_info(eth_dev, pci_dev);
1727
1728         hw->device_id = pci_dev->id.device_id;
1729         hw->vendor_id = pci_dev->id.vendor_id;
1730         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1731         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1732
1733         PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u",
1734                      pci_dev->id.vendor_id, pci_dev->id.device_id,
1735                      pci_dev->addr.domain, pci_dev->addr.bus,
1736                      pci_dev->addr.devid, pci_dev->addr.function);
1737
1738         hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr;
1739         if (hw->ctrl_bar == NULL) {
1740                 PMD_DRV_LOG(ERR,
1741                         "hw->ctrl_bar is NULL. BAR0 not configured");
1742                 return -ENODEV;
1743         }
1744
1745         if (hw->is_phyport) {
1746                 if (port == 0) {
1747                         hw->ctrl_bar = pf_dev->ctrl_bar;
1748                 } else {
1749                         if (!pf_dev->ctrl_bar)
1750                                 return -ENODEV;
1751                         /* Use port offset in pf ctrl_bar for this
1752                          * ports control bar
1753                          */
1754                         hw->ctrl_bar = pf_dev->ctrl_bar +
1755                                        (port * NFP_PF_CSR_SLICE_SIZE);
1756                 }
1757         }
1758
1759         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar);
1760
1761         hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS);
1762         hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS);
1763
1764         /* Work out where in the BAR the queues start. */
1765         switch (pci_dev->id.device_id) {
1766         case PCI_DEVICE_ID_NFP4000_PF_NIC:
1767         case PCI_DEVICE_ID_NFP6000_PF_NIC:
1768         case PCI_DEVICE_ID_NFP6000_VF_NIC:
1769                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
1770                 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
1771                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
1772                 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
1773                 break;
1774         default:
1775                 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
1776                 err = -ENODEV;
1777                 goto dev_err_ctrl_map;
1778         }
1779
1780         PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off);
1781         PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off);
1782
1783         if (hw->is_phyport) {
1784                 hw->tx_bar = pf_dev->hw_queues + tx_bar_off;
1785                 hw->rx_bar = pf_dev->hw_queues + rx_bar_off;
1786                 eth_dev->data->dev_private = hw;
1787         } else {
1788                 hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr +
1789                              tx_bar_off;
1790                 hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr +
1791                              rx_bar_off;
1792         }
1793
1794         PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p",
1795                      hw->ctrl_bar, hw->tx_bar, hw->rx_bar);
1796
1797         nfp_net_cfg_queue_setup(hw);
1798
1799         /* Get some of the read-only fields from the config BAR */
1800         hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION);
1801         hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP);
1802         hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU);
1803         hw->mtu = RTE_ETHER_MTU;
1804
1805         /* VLAN insertion is incompatible with LSOv2 */
1806         if (hw->cap & NFP_NET_CFG_CTRL_LSO2)
1807                 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN;
1808
1809         if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2)
1810                 hw->rx_offset = NFP_NET_RX_OFFSET;
1811         else
1812                 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR);
1813
1814         PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d",
1815                            NFD_CFG_MAJOR_VERSION_of(hw->ver),
1816                            NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu);
1817
1818         PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap,
1819                      hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "",
1820                      hw->cap & NFP_NET_CFG_CTRL_L2BC    ? "L2BCFILT " : "",
1821                      hw->cap & NFP_NET_CFG_CTRL_L2MC    ? "L2MCFILT " : "",
1822                      hw->cap & NFP_NET_CFG_CTRL_RXCSUM  ? "RXCSUM "  : "",
1823                      hw->cap & NFP_NET_CFG_CTRL_TXCSUM  ? "TXCSUM "  : "",
1824                      hw->cap & NFP_NET_CFG_CTRL_RXVLAN  ? "RXVLAN "  : "",
1825                      hw->cap & NFP_NET_CFG_CTRL_TXVLAN  ? "TXVLAN "  : "",
1826                      hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "",
1827                      hw->cap & NFP_NET_CFG_CTRL_GATHER  ? "GATHER "  : "",
1828                      hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR "  : "",
1829                      hw->cap & NFP_NET_CFG_CTRL_LSO     ? "TSO "     : "",
1830                      hw->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSOv2 "     : "",
1831                      hw->cap & NFP_NET_CFG_CTRL_RSS     ? "RSS "     : "",
1832                      hw->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSSv2 "     : "");
1833
1834         hw->ctrl = 0;
1835
1836         hw->stride_rx = stride;
1837         hw->stride_tx = stride;
1838
1839         PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u",
1840                      hw->max_rx_queues, hw->max_tx_queues);
1841
1842         /* Initializing spinlock for reconfigs */
1843         rte_spinlock_init(&hw->reconfig_lock);
1844
1845         /* Allocating memory for mac addr */
1846         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
1847                                                RTE_ETHER_ADDR_LEN, 0);
1848         if (eth_dev->data->mac_addrs == NULL) {
1849                 PMD_INIT_LOG(ERR, "Failed to space for MAC address");
1850                 err = -ENOMEM;
1851                 goto dev_err_queues_map;
1852         }
1853
1854         if (hw->is_phyport) {
1855                 nfp_net_pf_read_mac(pf_dev, port);
1856                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
1857         } else {
1858                 nfp_net_vf_read_mac(hw);
1859         }
1860
1861         if (!rte_is_valid_assigned_ether_addr(
1862                     (struct rte_ether_addr *)&hw->mac_addr)) {
1863                 PMD_INIT_LOG(INFO, "Using random mac address for port %d",
1864                                    port);
1865                 /* Using random mac addresses for VFs */
1866                 rte_eth_random_addr(&hw->mac_addr[0]);
1867                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
1868         }
1869
1870         /* Copying mac address to DPDK eth_dev struct */
1871         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
1872                         &eth_dev->data->mac_addrs[0]);
1873
1874         if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
1875                 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR;
1876
1877         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1878
1879         PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x "
1880                      "mac=%02x:%02x:%02x:%02x:%02x:%02x",
1881                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1882                      pci_dev->id.device_id,
1883                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1884                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1885
1886         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1887                 /* Registering LSC interrupt handler */
1888                 rte_intr_callback_register(&pci_dev->intr_handle,
1889                                            nfp_net_dev_interrupt_handler,
1890                                            (void *)eth_dev);
1891                 /* Telling the firmware about the LSC interrupt entry */
1892                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
1893                 /* Recording current stats counters values */
1894                 nfp_net_stats_reset(eth_dev);
1895         }
1896
1897         return 0;
1898
1899 dev_err_queues_map:
1900                 nfp_cpp_area_free(hw->hwqueues_area);
1901 dev_err_ctrl_map:
1902                 nfp_cpp_area_free(hw->ctrl_area);
1903
1904         return err;
1905 }
1906
1907 #define DEFAULT_FW_PATH       "/lib/firmware/netronome"
1908
1909 static int
1910 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
1911 {
1912         struct nfp_cpp *cpp = nsp->cpp;
1913         void *fw_buf;
1914         char fw_name[125];
1915         char serial[40];
1916         size_t fsize;
1917
1918         /* Looking for firmware file in order of priority */
1919
1920         /* First try to find a firmware image specific for this device */
1921         snprintf(serial, sizeof(serial),
1922                         "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x",
1923                 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3],
1924                 cpp->serial[4], cpp->serial[5], cpp->interface >> 8,
1925                 cpp->interface & 0xff);
1926
1927         snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
1928                         serial);
1929         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
1930         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
1931                 goto load_fw;
1932
1933         /* Then try the PCI name */
1934         snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
1935                         dev->device.name);
1936         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
1937         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
1938                 goto load_fw;
1939
1940         /* Finally try the card type and media */
1941         snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
1942         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
1943         if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
1944                 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
1945                 return -ENOENT;
1946         }
1947
1948 load_fw:
1949         PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
1950                 fw_name, fsize);
1951
1952         PMD_DRV_LOG(INFO, "Uploading the firmware ...");
1953         nfp_nsp_load_fw(nsp, fw_buf, fsize);
1954         PMD_DRV_LOG(INFO, "Done");
1955
1956         free(fw_buf);
1957         return 0;
1958 }
1959
1960 static int
1961 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp,
1962              struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo)
1963 {
1964         struct nfp_nsp *nsp;
1965         const char *nfp_fw_model;
1966         char card_desc[100];
1967         int err = 0;
1968
1969         nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno");
1970
1971         if (nfp_fw_model) {
1972                 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model);
1973         } else {
1974                 PMD_DRV_LOG(ERR, "firmware model NOT found");
1975                 return -EIO;
1976         }
1977
1978         if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) {
1979                 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u",
1980                        nfp_eth_table->count);
1981                 return -EIO;
1982         }
1983
1984         PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports",
1985                            nfp_eth_table->count);
1986
1987         PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed);
1988
1989         snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw",
1990                         nfp_fw_model, nfp_eth_table->count,
1991                         nfp_eth_table->ports[0].speed / 1000);
1992
1993         nsp = nfp_nsp_open(cpp);
1994         if (!nsp) {
1995                 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle");
1996                 return -EIO;
1997         }
1998
1999         nfp_nsp_device_soft_reset(nsp);
2000         err = nfp_fw_upload(dev, nsp, card_desc);
2001
2002         nfp_nsp_close(nsp);
2003         return err;
2004 }
2005
2006 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev)
2007 {
2008         struct nfp_net_hw *hw;
2009         struct rte_eth_dev *eth_dev;
2010         struct nfp_eth_table *nfp_eth_table = NULL;
2011         int ret = 0;
2012         int i;
2013
2014         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
2015         if (!nfp_eth_table) {
2016                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
2017                 ret = -EIO;
2018                 goto error;
2019         }
2020
2021         /* Loop through all physical ports on PF */
2022         for (i = 0; i < pf_dev->total_phyports; i++) {
2023                 const unsigned int numa_node = rte_socket_id();
2024                 char port_name[RTE_ETH_NAME_MAX_LEN];
2025
2026                 snprintf(port_name, sizeof(port_name), "%s_port%d",
2027                          pf_dev->pci_dev->device.name, i);
2028
2029                 /* Allocate a eth_dev for this phyport */
2030                 eth_dev = rte_eth_dev_allocate(port_name);
2031                 if (!eth_dev) {
2032                         ret = -ENODEV;
2033                         goto port_cleanup;
2034                 }
2035
2036                 /* Allocate memory for this phyport */
2037                 eth_dev->data->dev_private =
2038                         rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw),
2039                                            RTE_CACHE_LINE_SIZE, numa_node);
2040                 if (!eth_dev->data->dev_private) {
2041                         ret = -ENOMEM;
2042                         rte_eth_dev_release_port(eth_dev);
2043                         goto port_cleanup;
2044                 }
2045
2046                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
2047
2048                 /* Add this device to the PF's array of physical ports */
2049                 pf_dev->ports[i] = hw;
2050
2051                 hw->pf_dev = pf_dev;
2052                 hw->cpp = pf_dev->cpp;
2053                 hw->eth_dev = eth_dev;
2054                 hw->idx = i;
2055                 hw->nfp_idx = nfp_eth_table->ports[i].index;
2056                 hw->is_phyport = true;
2057
2058                 eth_dev->device = &pf_dev->pci_dev->device;
2059
2060                 /* ctrl/tx/rx BAR mappings and remaining init happens in
2061                  * nfp_net_init
2062                  */
2063                 ret = nfp_net_init(eth_dev);
2064
2065                 if (ret) {
2066                         ret = -ENODEV;
2067                         goto port_cleanup;
2068                 }
2069
2070                 rte_eth_dev_probing_finish(eth_dev);
2071
2072         } /* End loop, all ports on this PF */
2073         ret = 0;
2074         goto eth_table_cleanup;
2075
2076 port_cleanup:
2077         for (i = 0; i < pf_dev->total_phyports; i++) {
2078                 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) {
2079                         struct rte_eth_dev *tmp_dev;
2080                         tmp_dev = pf_dev->ports[i]->eth_dev;
2081                         rte_eth_dev_release_port(tmp_dev);
2082                         pf_dev->ports[i] = NULL;
2083                 }
2084         }
2085 eth_table_cleanup:
2086         free(nfp_eth_table);
2087 error:
2088         return ret;
2089 }
2090
2091 static int nfp_pf_init(struct rte_pci_device *pci_dev)
2092 {
2093         struct nfp_pf_dev *pf_dev = NULL;
2094         struct nfp_cpp *cpp;
2095         struct nfp_hwinfo *hwinfo;
2096         struct nfp_rtsym_table *sym_tbl;
2097         struct nfp_eth_table *nfp_eth_table = NULL;
2098         char name[RTE_ETH_NAME_MAX_LEN];
2099         int total_ports;
2100         int ret = -ENODEV;
2101         int err;
2102
2103         if (!pci_dev)
2104                 return ret;
2105
2106         /*
2107          * When device bound to UIO, the device could be used, by mistake,
2108          * by two DPDK apps, and the UIO driver does not avoid it. This
2109          * could lead to a serious problem when configuring the NFP CPP
2110          * interface. Here we avoid this telling to the CPP init code to
2111          * use a lock file if UIO is being used.
2112          */
2113         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
2114                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
2115         else
2116                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
2117
2118         if (!cpp) {
2119                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
2120                 ret = -EIO;
2121                 goto error;
2122         }
2123
2124         hwinfo = nfp_hwinfo_read(cpp);
2125         if (!hwinfo) {
2126                 PMD_INIT_LOG(ERR, "Error reading hwinfo table");
2127                 ret = -EIO;
2128                 goto error;
2129         }
2130
2131         nfp_eth_table = nfp_eth_read_ports(cpp);
2132         if (!nfp_eth_table) {
2133                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
2134                 ret = -EIO;
2135                 goto hwinfo_cleanup;
2136         }
2137
2138         if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) {
2139                 PMD_INIT_LOG(ERR, "Error when uploading firmware");
2140                 ret = -EIO;
2141                 goto eth_table_cleanup;
2142         }
2143
2144         /* Now the symbol table should be there */
2145         sym_tbl = nfp_rtsym_table_read(cpp);
2146         if (!sym_tbl) {
2147                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
2148                                 " symbol table");
2149                 ret = -EIO;
2150                 goto eth_table_cleanup;
2151         }
2152
2153         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
2154         if (total_ports != (int)nfp_eth_table->count) {
2155                 PMD_DRV_LOG(ERR, "Inconsistent number of ports");
2156                 ret = -EIO;
2157                 goto sym_tbl_cleanup;
2158         }
2159
2160         PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports);
2161
2162         if (total_ports <= 0 || total_ports > 8) {
2163                 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
2164                 ret = -ENODEV;
2165                 goto sym_tbl_cleanup;
2166         }
2167         /* Allocate memory for the PF "device" */
2168         snprintf(name, sizeof(name), "nfp_pf%d", 0);
2169         pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
2170         if (!pf_dev) {
2171                 ret = -ENOMEM;
2172                 goto sym_tbl_cleanup;
2173         }
2174
2175         /* Populate the newly created PF device */
2176         pf_dev->cpp = cpp;
2177         pf_dev->hwinfo = hwinfo;
2178         pf_dev->sym_tbl = sym_tbl;
2179         pf_dev->total_phyports = total_ports;
2180
2181         if (total_ports > 1)
2182                 pf_dev->multiport = true;
2183
2184         pf_dev->pci_dev = pci_dev;
2185
2186         /* Map the symbol table */
2187         pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0",
2188                                      pf_dev->total_phyports * 32768,
2189                                      &pf_dev->ctrl_area);
2190         if (!pf_dev->ctrl_bar) {
2191                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar");
2192                 ret = -EIO;
2193                 goto pf_cleanup;
2194         }
2195
2196         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar);
2197
2198         /* configure access to tx/rx vNIC BARs */
2199         pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0,
2200                                               NFP_PCIE_QUEUE(0),
2201                                               NFP_QCP_QUEUE_AREA_SZ,
2202                                               &pf_dev->hwqueues_area);
2203         if (!pf_dev->hw_queues) {
2204                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
2205                 ret = -EIO;
2206                 goto ctrl_area_cleanup;
2207         }
2208
2209         PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues);
2210
2211         /* Initialize and prep physical ports now
2212          * This will loop through all physical ports
2213          */
2214         ret = nfp_init_phyports(pf_dev);
2215         if (ret) {
2216                 PMD_INIT_LOG(ERR, "Could not create physical ports");
2217                 goto hwqueues_cleanup;
2218         }
2219
2220         /* register the CPP bridge service here for primary use */
2221         nfp_register_cpp_service(pf_dev->cpp);
2222
2223         return 0;
2224
2225 hwqueues_cleanup:
2226         nfp_cpp_area_free(pf_dev->hwqueues_area);
2227 ctrl_area_cleanup:
2228         nfp_cpp_area_free(pf_dev->ctrl_area);
2229 pf_cleanup:
2230         rte_free(pf_dev);
2231 sym_tbl_cleanup:
2232         free(sym_tbl);
2233 eth_table_cleanup:
2234         free(nfp_eth_table);
2235 hwinfo_cleanup:
2236         free(hwinfo);
2237 error:
2238         return ret;
2239 }
2240
2241 /*
2242  * When attaching to the NFP4000/6000 PF on a secondary process there
2243  * is no need to initialize the PF again. Only minimal work is required
2244  * here
2245  */
2246 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
2247 {
2248         struct nfp_cpp *cpp;
2249         struct nfp_rtsym_table *sym_tbl;
2250         int total_ports;
2251         int i;
2252         int err;
2253
2254         if (!pci_dev)
2255                 return -ENODEV;
2256
2257         /*
2258          * When device bound to UIO, the device could be used, by mistake,
2259          * by two DPDK apps, and the UIO driver does not avoid it. This
2260          * could lead to a serious problem when configuring the NFP CPP
2261          * interface. Here we avoid this telling to the CPP init code to
2262          * use a lock file if UIO is being used.
2263          */
2264         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
2265                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
2266         else
2267                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
2268
2269         if (!cpp) {
2270                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
2271                 return -EIO;
2272         }
2273
2274         /*
2275          * We don't have access to the PF created in the primary process
2276          * here so we have to read the number of ports from firmware
2277          */
2278         sym_tbl = nfp_rtsym_table_read(cpp);
2279         if (!sym_tbl) {
2280                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
2281                                 " symbol table");
2282                 return -EIO;
2283         }
2284
2285         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
2286
2287         for (i = 0; i < total_ports; i++) {
2288                 struct rte_eth_dev *eth_dev;
2289                 char port_name[RTE_ETH_NAME_MAX_LEN];
2290
2291                 snprintf(port_name, sizeof(port_name), "%s_port%d",
2292                          pci_dev->device.name, i);
2293
2294                 PMD_DRV_LOG(DEBUG, "Secondary attaching to port %s",
2295                     port_name);
2296                 eth_dev = rte_eth_dev_attach_secondary(port_name);
2297                 if (!eth_dev) {
2298                         RTE_LOG(ERR, EAL,
2299                         "secondary process attach failed, "
2300                         "ethdev doesn't exist");
2301                         return -ENODEV;
2302                 }
2303                 eth_dev->process_private = cpp;
2304                 eth_dev->dev_ops = &nfp_net_eth_dev_ops;
2305                 eth_dev->rx_queue_count = nfp_net_rx_queue_count;
2306                 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
2307                 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
2308                 rte_eth_dev_probing_finish(eth_dev);
2309         }
2310
2311         /* Register the CPP bridge service for the secondary too */
2312         nfp_register_cpp_service(cpp);
2313
2314         return 0;
2315 }
2316
2317 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2318                             struct rte_pci_device *dev)
2319 {
2320         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2321                 return nfp_pf_init(dev);
2322         else
2323                 return nfp_pf_secondary_init(dev);
2324 }
2325
2326 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = {
2327         {
2328                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
2329                                PCI_DEVICE_ID_NFP4000_PF_NIC)
2330         },
2331         {
2332                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
2333                                PCI_DEVICE_ID_NFP6000_PF_NIC)
2334         },
2335         {
2336                 .vendor_id = 0,
2337         },
2338 };
2339
2340 static const struct rte_pci_id pci_id_nfp_vf_net_map[] = {
2341         {
2342                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
2343                                PCI_DEVICE_ID_NFP6000_VF_NIC)
2344         },
2345         {
2346                 .vendor_id = 0,
2347         },
2348 };
2349
2350 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev)
2351 {
2352         struct rte_pci_device *pci_dev;
2353         uint16_t port_id;
2354
2355         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2356
2357         if (pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC ||
2358             pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC) {
2359                 /* Free up all physical ports under PF */
2360                 RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
2361                         rte_eth_dev_close(port_id);
2362                 /*
2363                  * Ports can be closed and freed but hotplugging is not
2364                  * currently supported
2365                  */
2366                 return -ENOTSUP;
2367         }
2368
2369         /* VF cleanup, just free private port data */
2370         return nfp_net_close(eth_dev);
2371 }
2372
2373 static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2374         struct rte_pci_device *pci_dev)
2375 {
2376         return rte_eth_dev_pci_generic_probe(pci_dev,
2377                 sizeof(struct nfp_net_adapter), nfp_net_init);
2378 }
2379
2380 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev)
2381 {
2382         return rte_eth_dev_pci_generic_remove(pci_dev, nfp_pci_uninit);
2383 }
2384
2385 static struct rte_pci_driver rte_nfp_net_pf_pmd = {
2386         .id_table = pci_id_nfp_pf_net_map,
2387         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2388         .probe = nfp_pf_pci_probe,
2389         .remove = eth_nfp_pci_remove,
2390 };
2391
2392 static struct rte_pci_driver rte_nfp_net_vf_pmd = {
2393         .id_table = pci_id_nfp_vf_net_map,
2394         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2395         .probe = eth_nfp_pci_probe,
2396         .remove = eth_nfp_pci_remove,
2397 };
2398
2399 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd);
2400 RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd);
2401 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map);
2402 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map);
2403 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio");
2404 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio");
2405 RTE_LOG_REGISTER_SUFFIX(nfp_logtype_init, init, NOTICE);
2406 RTE_LOG_REGISTER_SUFFIX(nfp_logtype_driver, driver, NOTICE);
2407 /*
2408  * Local variables:
2409  * c-file-style: "Linux"
2410  * indent-tabs-mode: t
2411  * End:
2412  */