drivers/net/nfp/nfp_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2014-2018 Netronome Systems, Inc.
   3  * All rights reserved.
   4  *
   5  * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.
   6  */
   7
   8 /*
   9  * vim:shiftwidth=8:noexpandtab
  10  *
  11  * @file dpdk/pmd/nfp_net.c
  12  *
  13  * Netronome vNIC DPDK Poll-Mode Driver: Main entry point
  14  */
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <ethdev_driver.h>
  21 #include <ethdev_pci.h>
  22 #include <rte_dev.h>
  23 #include <rte_ether.h>
  24 #include <rte_malloc.h>
  25 #include <rte_memzone.h>
  26 #include <rte_mempool.h>
  27 #include <rte_version.h>
  28 #include <rte_string_fns.h>
  29 #include <rte_alarm.h>
  30 #include <rte_spinlock.h>
  31 #include <rte_service_component.h>
  32
  33 #include "eal_firmware.h"
  34
  35 #include "nfpcore/nfp_cpp.h"
  36 #include "nfpcore/nfp_nffw.h"
  37 #include "nfpcore/nfp_hwinfo.h"
  38 #include "nfpcore/nfp_mip.h"
  39 #include "nfpcore/nfp_rtsym.h"
  40 #include "nfpcore/nfp_nsp.h"
  41
  42 #include "nfp_net_pmd.h"
  43 #include "nfp_rxtx.h"
  44 #include "nfp_net_logs.h"
  45 #include "nfp_net_ctrl.h"
  46 #include "nfp_cpp_bridge.h"
  47
  48 #include <sys/types.h>
  49 #include <sys/socket.h>
  50 #include <sys/un.h>
  51 #include <unistd.h>
  52 #include <stdio.h>
  53 #include <sys/ioctl.h>
  54 #include <errno.h>
  55
  56 /* Prototypes */
  57 static int nfp_net_close(struct rte_eth_dev *dev);
  58 static int nfp_net_configure(struct rte_eth_dev *dev);
  59 static void nfp_net_dev_interrupt_handler(void *param);
  60 static void nfp_net_dev_interrupt_delayed_handler(void *param);
  61 static int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
  62 static int nfp_net_infos_get(struct rte_eth_dev *dev,
  63                              struct rte_eth_dev_info *dev_info);
  64 static int nfp_net_init(struct rte_eth_dev *eth_dev);
  65 static int nfp_pf_init(struct rte_pci_device *pci_dev);
  66 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev);
  67 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev);
  68 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev);
  69 static int nfp_net_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  70 static int nfp_net_promisc_enable(struct rte_eth_dev *dev);
  71 static int nfp_net_promisc_disable(struct rte_eth_dev *dev);
  72 static int nfp_net_start(struct rte_eth_dev *dev);
  73 static int nfp_net_stats_get(struct rte_eth_dev *dev,
  74                               struct rte_eth_stats *stats);
  75 static int nfp_net_stats_reset(struct rte_eth_dev *dev);
  76 static int nfp_net_stop(struct rte_eth_dev *dev);
  77 static int nfp_net_rss_config_default(struct rte_eth_dev *dev);
  78 static int nfp_net_rss_hash_update(struct rte_eth_dev *dev,
  79                                    struct rte_eth_rss_conf *rss_conf);
  80 static int nfp_net_rss_reta_write(struct rte_eth_dev *dev,
  81                     struct rte_eth_rss_reta_entry64 *reta_conf,
  82                     uint16_t reta_size);
  83 static int nfp_net_rss_hash_write(struct rte_eth_dev *dev,
  84                         struct rte_eth_rss_conf *rss_conf);
  85 static int nfp_set_mac_addr(struct rte_eth_dev *dev,
  86                              struct rte_ether_addr *mac_addr);
  87 static int nfp_fw_setup(struct rte_pci_device *dev,
  88                         struct nfp_cpp *cpp,
  89                         struct nfp_eth_table *nfp_eth_table,
  90                         struct nfp_hwinfo *hwinfo);
  91
  92 static int
  93 __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update)
  94 {
  95         int cnt;
  96         uint32_t new;
  97         struct timespec wait;
  98
  99         PMD_DRV_LOG(DEBUG, "Writing to the configuration queue (%p)...",
 100                     hw->qcp_cfg);
 101
 102         if (hw->qcp_cfg == NULL)
 103                 rte_panic("Bad configuration queue pointer\n");
 104
 105         nfp_qcp_ptr_add(hw->qcp_cfg, NFP_QCP_WRITE_PTR, 1);
 106
 107         wait.tv_sec = 0;
 108         wait.tv_nsec = 1000000;
 109
 110         PMD_DRV_LOG(DEBUG, "Polling for update ack...");
 111
 112         /* Poll update field, waiting for NFP to ack the config */
 113         for (cnt = 0; ; cnt++) {
 114                 new = nn_cfg_readl(hw, NFP_NET_CFG_UPDATE);
 115                 if (new == 0)
 116                         break;
 117                 if (new & NFP_NET_CFG_UPDATE_ERR) {
 118                         PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new);
 119                         return -1;
 120                 }
 121                 if (cnt >= NFP_NET_POLL_TIMEOUT) {
 122                         PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after"
 123                                           " %dms", update, cnt);
 124                         rte_panic("Exiting\n");
 125                 }
 126                 nanosleep(&wait, 0); /* waiting for a 1ms */
 127         }
 128         PMD_DRV_LOG(DEBUG, "Ack DONE");
 129         return 0;
 130 }
 131
 132 /*
 133  * Reconfigure the NIC
 134  * @nn:    device to reconfigure
 135  * @ctrl:    The value for the ctrl field in the BAR config
 136  * @update:  The value for the update field in the BAR config
 137  *
 138  * Write the update word to the BAR and ping the reconfig queue. Then poll
 139  * until the firmware has acknowledged the update by zeroing the update word.
 140  */
 141 static int
 142 nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update)
 143 {
 144         uint32_t err;
 145
 146         PMD_DRV_LOG(DEBUG, "nfp_net_reconfig: ctrl=%08x update=%08x",
 147                     ctrl, update);
 148
 149         rte_spinlock_lock(&hw->reconfig_lock);
 150
 151         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, ctrl);
 152         nn_cfg_writel(hw, NFP_NET_CFG_UPDATE, update);
 153
 154         rte_wmb();
 155
 156         err = __nfp_net_reconfig(hw, update);
 157
 158         rte_spinlock_unlock(&hw->reconfig_lock);
 159
 160         if (!err)
 161                 return 0;
 162
 163         /*
 164          * Reconfig errors imply situations where they can be handled.
 165          * Otherwise, rte_panic is called inside __nfp_net_reconfig
 166          */
 167         PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x",
 168                      ctrl, update);
 169         return -EIO;
 170 }
 171
 172 /*
 173  * Configure an Ethernet device. This function must be invoked first
 174  * before any other function in the Ethernet API. This function can
 175  * also be re-invoked when a device is in the stopped state.
 176  */
 177 static int
 178 nfp_net_configure(struct rte_eth_dev *dev)
 179 {
 180         struct rte_eth_conf *dev_conf;
 181         struct rte_eth_rxmode *rxmode;
 182         struct rte_eth_txmode *txmode;
 183         struct nfp_net_hw *hw;
 184
 185         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 186
 187         /*
 188          * A DPDK app sends info about how many queues to use and how
 189          * those queues need to be configured. This is used by the
 190          * DPDK core and it makes sure no more queues than those
 191          * advertised by the driver are requested. This function is
 192          * called after that internal process
 193          */
 194
 195         PMD_INIT_LOG(DEBUG, "Configure");
 196
 197         dev_conf = &dev->data->dev_conf;
 198         rxmode = &dev_conf->rxmode;
 199         txmode = &dev_conf->txmode;
 200
 201         if (rxmode->mq_mode & ETH_MQ_RX_RSS_FLAG)
 202                 rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 203
 204         /* Checking TX mode */
 205         if (txmode->mq_mode) {
 206                 PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported");
 207                 return -EINVAL;
 208         }
 209
 210         /* Checking RX mode */
 211         if (rxmode->mq_mode & ETH_MQ_RX_RSS &&
 212             !(hw->cap & NFP_NET_CFG_CTRL_RSS)) {
 213                 PMD_INIT_LOG(INFO, "RSS not supported");
 214                 return -EINVAL;
 215         }
 216
 217         return 0;
 218 }
 219
 220 static void
 221 nfp_net_enable_queues(struct rte_eth_dev *dev)
 222 {
 223         struct nfp_net_hw *hw;
 224         uint64_t enabled_queues = 0;
 225         int i;
 226
 227         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 228
 229         /* Enabling the required TX queues in the device */
 230         for (i = 0; i < dev->data->nb_tx_queues; i++)
 231                 enabled_queues |= (1 << i);
 232
 233         nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, enabled_queues);
 234
 235         enabled_queues = 0;
 236
 237         /* Enabling the required RX queues in the device */
 238         for (i = 0; i < dev->data->nb_rx_queues; i++)
 239                 enabled_queues |= (1 << i);
 240
 241         nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, enabled_queues);
 242 }
 243
 244 static void
 245 nfp_net_disable_queues(struct rte_eth_dev *dev)
 246 {
 247         struct nfp_net_hw *hw;
 248         uint32_t new_ctrl, update = 0;
 249
 250         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 251
 252         nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, 0);
 253         nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, 0);
 254
 255         new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_ENABLE;
 256         update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING |
 257                  NFP_NET_CFG_UPDATE_MSIX;
 258
 259         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 260                 new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG;
 261
 262         /* If an error when reconfig we avoid to change hw state */
 263         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 264                 return;
 265
 266         hw->ctrl = new_ctrl;
 267 }
 268
 269 static void
 270 nfp_net_params_setup(struct nfp_net_hw *hw)
 271 {
 272         nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu);
 273         nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz);
 274 }
 275
 276 static void
 277 nfp_net_cfg_queue_setup(struct nfp_net_hw *hw)
 278 {
 279         hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ;
 280 }
 281
 282 #define ETH_ADDR_LEN    6
 283
 284 static void
 285 nfp_eth_copy_mac(uint8_t *dst, const uint8_t *src)
 286 {
 287         int i;
 288
 289         for (i = 0; i < ETH_ADDR_LEN; i++)
 290                 dst[i] = src[i];
 291 }
 292
 293 static int
 294 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port)
 295 {
 296         struct nfp_eth_table *nfp_eth_table;
 297         struct nfp_net_hw *hw = NULL;
 298
 299         /* Grab a pointer to the correct physical port */
 300         hw = pf_dev->ports[port];
 301
 302         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
 303
 304         nfp_eth_copy_mac((uint8_t *)&hw->mac_addr,
 305                          (uint8_t *)&nfp_eth_table->ports[port].mac_addr);
 306
 307         free(nfp_eth_table);
 308         return 0;
 309 }
 310
 311 static void
 312 nfp_net_vf_read_mac(struct nfp_net_hw *hw)
 313 {
 314         uint32_t tmp;
 315
 316         tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR));
 317         memcpy(&hw->mac_addr[0], &tmp, 4);
 318
 319         tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4));
 320         memcpy(&hw->mac_addr[4], &tmp, 2);
 321 }
 322
 323 static void
 324 nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac)
 325 {
 326         uint32_t mac0 = *(uint32_t *)mac;
 327         uint16_t mac1;
 328
 329         nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR);
 330
 331         mac += 4;
 332         mac1 = *(uint16_t *)mac;
 333         nn_writew(rte_cpu_to_be_16(mac1),
 334                   hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6);
 335 }
 336
 337 int
 338 nfp_set_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
 339 {
 340         struct nfp_net_hw *hw;
 341         uint32_t update, ctrl;
 342
 343         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 344         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 345             !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) {
 346                 PMD_INIT_LOG(INFO, "MAC address unable to change when"
 347                                   " port enabled");
 348                 return -EBUSY;
 349         }
 350
 351         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 352             !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 353                 return -EBUSY;
 354
 355         /* Writing new MAC to the specific port BAR address */
 356         nfp_net_write_mac(hw, (uint8_t *)mac_addr);
 357
 358         /* Signal the NIC about the change */
 359         update = NFP_NET_CFG_UPDATE_MACADDR;
 360         ctrl = hw->ctrl;
 361         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 362             (hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 363                 ctrl |= NFP_NET_CFG_CTRL_LIVE_ADDR;
 364         if (nfp_net_reconfig(hw, ctrl, update) < 0) {
 365                 PMD_INIT_LOG(INFO, "MAC address update failed");
 366                 return -EIO;
 367         }
 368         return 0;
 369 }
 370
 371 static int
 372 nfp_configure_rx_interrupt(struct rte_eth_dev *dev,
 373                            struct rte_intr_handle *intr_handle)
 374 {
 375         struct nfp_net_hw *hw;
 376         int i;
 377
 378         if (!intr_handle->intr_vec) {
 379                 intr_handle->intr_vec =
 380                         rte_zmalloc("intr_vec",
 381                                     dev->data->nb_rx_queues * sizeof(int), 0);
 382                 if (!intr_handle->intr_vec) {
 383                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
 384                                      " intr_vec", dev->data->nb_rx_queues);
 385                         return -ENOMEM;
 386                 }
 387         }
 388
 389         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 390
 391         if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 392                 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO");
 393                 /* UIO just supports one queue and no LSC*/
 394                 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0);
 395                 intr_handle->intr_vec[0] = 0;
 396         } else {
 397                 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO");
 398                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 399                         /*
 400                          * The first msix vector is reserved for non
 401                          * efd interrupts
 402                         */
 403                         nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1);
 404                         intr_handle->intr_vec[i] = i + 1;
 405                         PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d", i,
 406                                             intr_handle->intr_vec[i]);
 407                 }
 408         }
 409
 410         /* Avoiding TX interrupts */
 411         hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF;
 412         return 0;
 413 }
 414
 415 static uint32_t
 416 nfp_check_offloads(struct rte_eth_dev *dev)
 417 {
 418         struct nfp_net_hw *hw;
 419         struct rte_eth_conf *dev_conf;
 420         struct rte_eth_rxmode *rxmode;
 421         struct rte_eth_txmode *txmode;
 422         uint32_t ctrl = 0;
 423
 424         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 425
 426         dev_conf = &dev->data->dev_conf;
 427         rxmode = &dev_conf->rxmode;
 428         txmode = &dev_conf->txmode;
 429
 430         if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
 431                 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM)
 432                         ctrl |= NFP_NET_CFG_CTRL_RXCSUM;
 433         }
 434
 435         if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) {
 436                 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN)
 437                         ctrl |= NFP_NET_CFG_CTRL_RXVLAN;
 438         }
 439
 440         if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 441                 hw->mtu = rxmode->max_rx_pkt_len;
 442
 443         if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT)
 444                 ctrl |= NFP_NET_CFG_CTRL_TXVLAN;
 445
 446         /* L2 broadcast */
 447         if (hw->cap & NFP_NET_CFG_CTRL_L2BC)
 448                 ctrl |= NFP_NET_CFG_CTRL_L2BC;
 449
 450         /* L2 multicast */
 451         if (hw->cap & NFP_NET_CFG_CTRL_L2MC)
 452                 ctrl |= NFP_NET_CFG_CTRL_L2MC;
 453
 454         /* TX checksum offload */
 455         if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM ||
 456             txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM ||
 457             txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
 458                 ctrl |= NFP_NET_CFG_CTRL_TXCSUM;
 459
 460         /* LSO offload */
 461         if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) {
 462                 if (hw->cap & NFP_NET_CFG_CTRL_LSO)
 463                         ctrl |= NFP_NET_CFG_CTRL_LSO;
 464                 else
 465                         ctrl |= NFP_NET_CFG_CTRL_LSO2;
 466         }
 467
 468         /* RX gather */
 469         if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 470                 ctrl |= NFP_NET_CFG_CTRL_GATHER;
 471
 472         return ctrl;
 473 }
 474
 475 static int
 476 nfp_net_start(struct rte_eth_dev *dev)
 477 {
 478         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 479         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 480         uint32_t new_ctrl, update = 0;
 481         struct nfp_net_hw *hw;
 482         struct nfp_pf_dev *pf_dev;
 483         struct rte_eth_conf *dev_conf;
 484         struct rte_eth_rxmode *rxmode;
 485         uint32_t intr_vector;
 486         int ret;
 487
 488         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 490
 491         PMD_INIT_LOG(DEBUG, "Start");
 492
 493         /* Disabling queues just in case... */
 494         nfp_net_disable_queues(dev);
 495
 496         /* Enabling the required queues in the device */
 497         nfp_net_enable_queues(dev);
 498
 499         /* check and configure queue intr-vector mapping */
 500         if (dev->data->dev_conf.intr_conf.rxq != 0) {
 501                 if (pf_dev->multiport) {
 502                         PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported "
 503                                           "with NFP multiport PF");
 504                                 return -EINVAL;
 505                 }
 506                 if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 507                         /*
 508                          * Better not to share LSC with RX interrupts.
 509                          * Unregistering LSC interrupt handler
 510                          */
 511                         rte_intr_callback_unregister(&pci_dev->intr_handle,
 512                                 nfp_net_dev_interrupt_handler, (void *)dev);
 513
 514                         if (dev->data->nb_rx_queues > 1) {
 515                                 PMD_INIT_LOG(ERR, "PMD rx interrupt only "
 516                                              "supports 1 queue with UIO");
 517                                 return -EIO;
 518                         }
 519                 }
 520                 intr_vector = dev->data->nb_rx_queues;
 521                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 522                         return -1;
 523
 524                 nfp_configure_rx_interrupt(dev, intr_handle);
 525                 update = NFP_NET_CFG_UPDATE_MSIX;
 526         }
 527
 528         rte_intr_enable(intr_handle);
 529
 530         new_ctrl = nfp_check_offloads(dev);
 531
 532         /* Writing configuration parameters in the device */
 533         nfp_net_params_setup(hw);
 534
 535         dev_conf = &dev->data->dev_conf;
 536         rxmode = &dev_conf->rxmode;
 537
 538         if (rxmode->mq_mode & ETH_MQ_RX_RSS) {
 539                 nfp_net_rss_config_default(dev);
 540                 update |= NFP_NET_CFG_UPDATE_RSS;
 541                 new_ctrl |= NFP_NET_CFG_CTRL_RSS;
 542         }
 543
 544         /* Enable device */
 545         new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
 546
 547         update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING;
 548
 549         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 550                 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;
 551
 552         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl);
 553         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 554                 return -EIO;
 555
 556         /*
 557          * Allocating rte mbufs for configured rx queues.
 558          * This requires queues being enabled before
 559          */
 560         if (nfp_net_rx_freelist_setup(dev) < 0) {
 561                 ret = -ENOMEM;
 562                 goto error;
 563         }
 564
 565         if (hw->is_phyport) {
 566                 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 567                         /* Configure the physical port up */
 568                         nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 569                 else
 570                         nfp_eth_set_configured(dev->process_private,
 571                                                hw->nfp_idx, 1);
 572         }
 573
 574         hw->ctrl = new_ctrl;
 575
 576         return 0;
 577
 578 error:
 579         /*
 580          * An error returned by this function should mean the app
 581          * exiting and then the system releasing all the memory
 582          * allocated even memory coming from hugepages.
 583          *
 584          * The device could be enabled at this point with some queues
 585          * ready for getting packets. This is true if the call to
 586          * nfp_net_rx_freelist_setup() succeeds for some queues but
 587          * fails for subsequent queues.
 588          *
 589          * This should make the app exiting but better if we tell the
 590          * device first.
 591          */
 592         nfp_net_disable_queues(dev);
 593
 594         return ret;
 595 }
 596
 597 /* Stop device: disable rx and tx functions to allow for reconfiguring. */
 598 static int
 599 nfp_net_stop(struct rte_eth_dev *dev)
 600 {
 601         int i;
 602         struct nfp_net_hw *hw;
 603
 604         PMD_INIT_LOG(DEBUG, "Stop");
 605
 606         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 607
 608         nfp_net_disable_queues(dev);
 609
 610         /* Clear queues */
 611         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 612                 nfp_net_reset_tx_queue(
 613                         (struct nfp_net_txq *)dev->data->tx_queues[i]);
 614         }
 615
 616         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 617                 nfp_net_reset_rx_queue(
 618                         (struct nfp_net_rxq *)dev->data->rx_queues[i]);
 619         }
 620
 621         if (hw->is_phyport) {
 622                 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 623                         /* Configure the physical port down */
 624                         nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 625                 else
 626                         nfp_eth_set_configured(dev->process_private,
 627                                                hw->nfp_idx, 0);
 628         }
 629
 630         return 0;
 631 }
 632
 633 /* Set the link up. */
 634 static int
 635 nfp_net_set_link_up(struct rte_eth_dev *dev)
 636 {
 637         struct nfp_net_hw *hw;
 638
 639         PMD_DRV_LOG(DEBUG, "Set link up");
 640
 641         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 642
 643         if (!hw->is_phyport)
 644                 return -ENOTSUP;
 645
 646         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 647                 /* Configure the physical port down */
 648                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 1);
 649         else
 650                 return nfp_eth_set_configured(dev->process_private,
 651                                               hw->nfp_idx, 1);
 652 }
 653
 654 /* Set the link down. */
 655 static int
 656 nfp_net_set_link_down(struct rte_eth_dev *dev)
 657 {
 658         struct nfp_net_hw *hw;
 659
 660         PMD_DRV_LOG(DEBUG, "Set link down");
 661
 662         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 663
 664         if (!hw->is_phyport)
 665                 return -ENOTSUP;
 666
 667         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 668                 /* Configure the physical port down */
 669                 return nfp_eth_set_configured(hw->cpp, hw->nfp_idx, 0);
 670         else
 671                 return nfp_eth_set_configured(dev->process_private,
 672                                               hw->nfp_idx, 0);
 673 }
 674
 675 /* Reset and stop device. The device can not be restarted. */
 676 static int
 677 nfp_net_close(struct rte_eth_dev *dev)
 678 {
 679         struct nfp_net_hw *hw;
 680         struct rte_pci_device *pci_dev;
 681         int i;
 682
 683         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 684                 return 0;
 685
 686         PMD_INIT_LOG(DEBUG, "Close");
 687
 688         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 689         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 690
 691         /*
 692          * We assume that the DPDK application is stopping all the
 693          * threads/queues before calling the device close function.
 694          */
 695
 696         nfp_net_disable_queues(dev);
 697
 698         /* Clear queues */
 699         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 700                 nfp_net_reset_tx_queue(
 701                         (struct nfp_net_txq *)dev->data->tx_queues[i]);
 702         }
 703
 704         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 705                 nfp_net_reset_rx_queue(
 706                         (struct nfp_net_rxq *)dev->data->rx_queues[i]);
 707         }
 708
 709         /* Only free PF resources after all physical ports have been closed */
 710         if (pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC ||
 711             pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC) {
 712                 struct nfp_pf_dev *pf_dev;
 713                 pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 714
 715                 /* Mark this port as unused and free device priv resources*/
 716                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff);
 717                 pf_dev->ports[hw->idx] = NULL;
 718                 rte_eth_dev_release_port(dev);
 719
 720                 for (i = 0; i < pf_dev->total_phyports; i++) {
 721                         /* Check to see if ports are still in use */
 722                         if (pf_dev->ports[i])
 723                                 return 0;
 724                 }
 725
 726                 /* Now it is safe to free all PF resources */
 727                 PMD_INIT_LOG(INFO, "Freeing PF resources");
 728                 nfp_cpp_area_free(pf_dev->ctrl_area);
 729                 nfp_cpp_area_free(pf_dev->hwqueues_area);
 730                 free(pf_dev->hwinfo);
 731                 free(pf_dev->sym_tbl);
 732                 nfp_cpp_free(pf_dev->cpp);
 733                 rte_free(pf_dev);
 734         }
 735
 736         rte_intr_disable(&pci_dev->intr_handle);
 737
 738         /* unregister callback func from eal lib */
 739         rte_intr_callback_unregister(&pci_dev->intr_handle,
 740                                      nfp_net_dev_interrupt_handler,
 741                                      (void *)dev);
 742
 743         /*
 744          * The ixgbe PMD driver disables the pcie master on the
 745          * device. The i40e does not...
 746          */
 747
 748         return 0;
 749 }
 750
 751 static int
 752 nfp_net_promisc_enable(struct rte_eth_dev *dev)
 753 {
 754         uint32_t new_ctrl, update = 0;
 755         struct nfp_net_hw *hw;
 756         int ret;
 757
 758         PMD_DRV_LOG(DEBUG, "Promiscuous mode enable");
 759
 760         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 761
 762         if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) {
 763                 PMD_INIT_LOG(INFO, "Promiscuous mode not supported");
 764                 return -ENOTSUP;
 765         }
 766
 767         if (hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) {
 768                 PMD_DRV_LOG(INFO, "Promiscuous mode already enabled");
 769                 return 0;
 770         }
 771
 772         new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_PROMISC;
 773         update = NFP_NET_CFG_UPDATE_GEN;
 774
 775         /*
 776          * DPDK sets promiscuous mode on just after this call assuming
 777          * it can not fail ...
 778          */
 779         ret = nfp_net_reconfig(hw, new_ctrl, update);
 780         if (ret < 0)
 781                 return ret;
 782
 783         hw->ctrl = new_ctrl;
 784
 785         return 0;
 786 }
 787
 788 static int
 789 nfp_net_promisc_disable(struct rte_eth_dev *dev)
 790 {
 791         uint32_t new_ctrl, update = 0;
 792         struct nfp_net_hw *hw;
 793         int ret;
 794
 795         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 796
 797         if ((hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) == 0) {
 798                 PMD_DRV_LOG(INFO, "Promiscuous mode already disabled");
 799                 return 0;
 800         }
 801
 802         new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_PROMISC;
 803         update = NFP_NET_CFG_UPDATE_GEN;
 804
 805         /*
 806          * DPDK sets promiscuous mode off just before this call
 807          * assuming it can not fail ...
 808          */
 809         ret = nfp_net_reconfig(hw, new_ctrl, update);
 810         if (ret < 0)
 811                 return ret;
 812
 813         hw->ctrl = new_ctrl;
 814
 815         return 0;
 816 }
 817
 818 /*
 819  * return 0 means link status changed, -1 means not changed
 820  *
 821  * Wait to complete is needed as it can take up to 9 seconds to get the Link
 822  * status.
 823  */
 824 static int
 825 nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
 826 {
 827         struct nfp_net_hw *hw;
 828         struct rte_eth_link link;
 829         uint32_t nn_link_status;
 830         int ret;
 831
 832         static const uint32_t ls_to_ethtool[] = {
 833                 [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE,
 834                 [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN]     = ETH_SPEED_NUM_NONE,
 835                 [NFP_NET_CFG_STS_LINK_RATE_1G]          = ETH_SPEED_NUM_1G,
 836                 [NFP_NET_CFG_STS_LINK_RATE_10G]         = ETH_SPEED_NUM_10G,
 837                 [NFP_NET_CFG_STS_LINK_RATE_25G]         = ETH_SPEED_NUM_25G,
 838                 [NFP_NET_CFG_STS_LINK_RATE_40G]         = ETH_SPEED_NUM_40G,
 839                 [NFP_NET_CFG_STS_LINK_RATE_50G]         = ETH_SPEED_NUM_50G,
 840                 [NFP_NET_CFG_STS_LINK_RATE_100G]        = ETH_SPEED_NUM_100G,
 841         };
 842
 843         PMD_DRV_LOG(DEBUG, "Link update");
 844
 845         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 846
 847         nn_link_status = nn_cfg_readl(hw, NFP_NET_CFG_STS);
 848
 849         memset(&link, 0, sizeof(struct rte_eth_link));
 850
 851         if (nn_link_status & NFP_NET_CFG_STS_LINK)
 852                 link.link_status = ETH_LINK_UP;
 853
 854         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 855
 856         nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) &
 857                          NFP_NET_CFG_STS_LINK_RATE_MASK;
 858
 859         if (nn_link_status >= RTE_DIM(ls_to_ethtool))
 860                 link.link_speed = ETH_SPEED_NUM_NONE;
 861         else
 862                 link.link_speed = ls_to_ethtool[nn_link_status];
 863
 864         ret = rte_eth_linkstatus_set(dev, &link);
 865         if (ret == 0) {
 866                 if (link.link_status)
 867                         PMD_DRV_LOG(INFO, "NIC Link is Up");
 868                 else
 869                         PMD_DRV_LOG(INFO, "NIC Link is Down");
 870         }
 871         return ret;
 872 }
 873
 874 static int
 875 nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 876 {
 877         int i;
 878         struct nfp_net_hw *hw;
 879         struct rte_eth_stats nfp_dev_stats;
 880
 881         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 882
 883         /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */
 884
 885         memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats));
 886
 887         /* reading per RX ring stats */
 888         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 889                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 890                         break;
 891
 892                 nfp_dev_stats.q_ipackets[i] =
 893                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i));
 894
 895                 nfp_dev_stats.q_ipackets[i] -=
 896                         hw->eth_stats_base.q_ipackets[i];
 897
 898                 nfp_dev_stats.q_ibytes[i] =
 899                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8);
 900
 901                 nfp_dev_stats.q_ibytes[i] -=
 902                         hw->eth_stats_base.q_ibytes[i];
 903         }
 904
 905         /* reading per TX ring stats */
 906         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 907                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 908                         break;
 909
 910                 nfp_dev_stats.q_opackets[i] =
 911                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i));
 912
 913                 nfp_dev_stats.q_opackets[i] -=
 914                         hw->eth_stats_base.q_opackets[i];
 915
 916                 nfp_dev_stats.q_obytes[i] =
 917                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8);
 918
 919                 nfp_dev_stats.q_obytes[i] -=
 920                         hw->eth_stats_base.q_obytes[i];
 921         }
 922
 923         nfp_dev_stats.ipackets =
 924                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES);
 925
 926         nfp_dev_stats.ipackets -= hw->eth_stats_base.ipackets;
 927
 928         nfp_dev_stats.ibytes =
 929                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS);
 930
 931         nfp_dev_stats.ibytes -= hw->eth_stats_base.ibytes;
 932
 933         nfp_dev_stats.opackets =
 934                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES);
 935
 936         nfp_dev_stats.opackets -= hw->eth_stats_base.opackets;
 937
 938         nfp_dev_stats.obytes =
 939                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS);
 940
 941         nfp_dev_stats.obytes -= hw->eth_stats_base.obytes;
 942
 943         /* reading general device stats */
 944         nfp_dev_stats.ierrors =
 945                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS);
 946
 947         nfp_dev_stats.ierrors -= hw->eth_stats_base.ierrors;
 948
 949         nfp_dev_stats.oerrors =
 950                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS);
 951
 952         nfp_dev_stats.oerrors -= hw->eth_stats_base.oerrors;
 953
 954         /* RX ring mbuf allocation failures */
 955         nfp_dev_stats.rx_nombuf = dev->data->rx_mbuf_alloc_failed;
 956
 957         nfp_dev_stats.imissed =
 958                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS);
 959
 960         nfp_dev_stats.imissed -= hw->eth_stats_base.imissed;
 961
 962         if (stats) {
 963                 memcpy(stats, &nfp_dev_stats, sizeof(*stats));
 964                 return 0;
 965         }
 966         return -EINVAL;
 967 }
 968
 969 static int
 970 nfp_net_stats_reset(struct rte_eth_dev *dev)
 971 {
 972         int i;
 973         struct nfp_net_hw *hw;
 974
 975         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 976
 977         /*
 978          * hw->eth_stats_base records the per counter starting point.
 979          * Lets update it now
 980          */
 981
 982         /* reading per RX ring stats */
 983         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 984                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 985                         break;
 986
 987                 hw->eth_stats_base.q_ipackets[i] =
 988                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i));
 989
 990                 hw->eth_stats_base.q_ibytes[i] =
 991                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8);
 992         }
 993
 994         /* reading per TX ring stats */
 995         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 996                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
 997                         break;
 998
 999                 hw->eth_stats_base.q_opackets[i] =
1000                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i));
1001
1002                 hw->eth_stats_base.q_obytes[i] =
1003                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8);
1004         }
1005
1006         hw->eth_stats_base.ipackets =
1007                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES);
1008
1009         hw->eth_stats_base.ibytes =
1010                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS);
1011
1012         hw->eth_stats_base.opackets =
1013                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES);
1014
1015         hw->eth_stats_base.obytes =
1016                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS);
1017
1018         /* reading general device stats */
1019         hw->eth_stats_base.ierrors =
1020                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS);
1021
1022         hw->eth_stats_base.oerrors =
1023                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS);
1024
1025         /* RX ring mbuf allocation failures */
1026         dev->data->rx_mbuf_alloc_failed = 0;
1027
1028         hw->eth_stats_base.imissed =
1029                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS);
1030
1031         return 0;
1032 }
1033
1034 static int
1035 nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1036 {
1037         struct nfp_net_hw *hw;
1038
1039         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1040
1041         dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues;
1042         dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues;
1043         dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
1044         dev_info->max_rx_pktlen = hw->max_mtu;
1045         /* Next should change when PF support is implemented */
1046         dev_info->max_mac_addrs = 1;
1047
1048         if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN)
1049                 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1050
1051         if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM)
1052                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM |
1053                                              DEV_RX_OFFLOAD_UDP_CKSUM |
1054                                              DEV_RX_OFFLOAD_TCP_CKSUM;
1055
1056         if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)
1057                 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
1058
1059         if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM)
1060                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM |
1061                                              DEV_TX_OFFLOAD_UDP_CKSUM |
1062                                              DEV_TX_OFFLOAD_TCP_CKSUM;
1063
1064         if (hw->cap & NFP_NET_CFG_CTRL_LSO_ANY)
1065                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
1066
1067         if (hw->cap & NFP_NET_CFG_CTRL_GATHER)
1068                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MULTI_SEGS;
1069
1070         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1071                 .rx_thresh = {
1072                         .pthresh = DEFAULT_RX_PTHRESH,
1073                         .hthresh = DEFAULT_RX_HTHRESH,
1074                         .wthresh = DEFAULT_RX_WTHRESH,
1075                 },
1076                 .rx_free_thresh = DEFAULT_RX_FREE_THRESH,
1077                 .rx_drop_en = 0,
1078         };
1079
1080         dev_info->default_txconf = (struct rte_eth_txconf) {
1081                 .tx_thresh = {
1082                         .pthresh = DEFAULT_TX_PTHRESH,
1083                         .hthresh = DEFAULT_TX_HTHRESH,
1084                         .wthresh = DEFAULT_TX_WTHRESH,
1085                 },
1086                 .tx_free_thresh = DEFAULT_TX_FREE_THRESH,
1087                 .tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH,
1088         };
1089
1090         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1091                 .nb_max = NFP_NET_MAX_RX_DESC,
1092                 .nb_min = NFP_NET_MIN_RX_DESC,
1093                 .nb_align = NFP_ALIGN_RING_DESC,
1094         };
1095
1096         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1097                 .nb_max = NFP_NET_MAX_TX_DESC,
1098                 .nb_min = NFP_NET_MIN_TX_DESC,
1099                 .nb_align = NFP_ALIGN_RING_DESC,
1100                 .nb_seg_max = NFP_TX_MAX_SEG,
1101                 .nb_mtu_seg_max = NFP_TX_MAX_MTU_SEG,
1102         };
1103
1104         /* All NFP devices support jumbo frames */
1105         dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME;
1106
1107         if (hw->cap & NFP_NET_CFG_CTRL_RSS) {
1108                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_RSS_HASH;
1109
1110                 dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1111                                                    ETH_RSS_NONFRAG_IPV4_TCP |
1112                                                    ETH_RSS_NONFRAG_IPV4_UDP |
1113                                                    ETH_RSS_IPV6 |
1114                                                    ETH_RSS_NONFRAG_IPV6_TCP |
1115                                                    ETH_RSS_NONFRAG_IPV6_UDP;
1116
1117                 dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ;
1118                 dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ;
1119         }
1120
1121         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G |
1122                                ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G |
1123                                ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G;
1124
1125         return 0;
1126 }
1127
1128 static const uint32_t *
1129 nfp_net_supported_ptypes_get(struct rte_eth_dev *dev)
1130 {
1131         static const uint32_t ptypes[] = {
1132                 /* refers to nfp_net_set_hash() */
1133                 RTE_PTYPE_INNER_L3_IPV4,
1134                 RTE_PTYPE_INNER_L3_IPV6,
1135                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1136                 RTE_PTYPE_INNER_L4_MASK,
1137                 RTE_PTYPE_UNKNOWN
1138         };
1139
1140         if (dev->rx_pkt_burst == nfp_net_recv_pkts)
1141                 return ptypes;
1142         return NULL;
1143 }
1144
1145 static int
1146 nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1147 {
1148         struct rte_pci_device *pci_dev;
1149         struct nfp_net_hw *hw;
1150         int base = 0;
1151
1152         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1153         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1154
1155         if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO)
1156                 base = 1;
1157
1158         /* Make sure all updates are written before un-masking */
1159         rte_wmb();
1160         nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id),
1161                       NFP_NET_CFG_ICR_UNMASKED);
1162         return 0;
1163 }
1164
1165 static int
1166 nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1167 {
1168         struct rte_pci_device *pci_dev;
1169         struct nfp_net_hw *hw;
1170         int base = 0;
1171
1172         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1173         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1174
1175         if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO)
1176                 base = 1;
1177
1178         /* Make sure all updates are written before un-masking */
1179         rte_wmb();
1180         nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1);
1181         return 0;
1182 }
1183
1184 static void
1185 nfp_net_dev_link_status_print(struct rte_eth_dev *dev)
1186 {
1187         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1188         struct rte_eth_link link;
1189
1190         rte_eth_linkstatus_get(dev, &link);
1191         if (link.link_status)
1192                 PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1193                             dev->data->port_id, link.link_speed,
1194                             link.link_duplex == ETH_LINK_FULL_DUPLEX
1195                             ? "full-duplex" : "half-duplex");
1196         else
1197                 PMD_DRV_LOG(INFO, " Port %d: Link Down",
1198                             dev->data->port_id);
1199
1200         PMD_DRV_LOG(INFO, "PCI Address: " PCI_PRI_FMT,
1201                     pci_dev->addr.domain, pci_dev->addr.bus,
1202                     pci_dev->addr.devid, pci_dev->addr.function);
1203 }
1204
1205 /* Interrupt configuration and handling */
1206
1207 /*
1208  * nfp_net_irq_unmask - Unmask an interrupt
1209  *
1210  * If MSI-X auto-masking is enabled clear the mask bit, otherwise
1211  * clear the ICR for the entry.
1212  */
1213 static void
1214 nfp_net_irq_unmask(struct rte_eth_dev *dev)
1215 {
1216         struct nfp_net_hw *hw;
1217         struct rte_pci_device *pci_dev;
1218
1219         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1220         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1221
1222         if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) {
1223                 /* If MSI-X auto-masking is used, clear the entry */
1224                 rte_wmb();
1225                 rte_intr_ack(&pci_dev->intr_handle);
1226         } else {
1227                 /* Make sure all updates are written before un-masking */
1228                 rte_wmb();
1229                 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(NFP_NET_IRQ_LSC_IDX),
1230                               NFP_NET_CFG_ICR_UNMASKED);
1231         }
1232 }
1233
1234 static void
1235 nfp_net_dev_interrupt_handler(void *param)
1236 {
1237         int64_t timeout;
1238         struct rte_eth_link link;
1239         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1240
1241         PMD_DRV_LOG(DEBUG, "We got a LSC interrupt!!!");
1242
1243         rte_eth_linkstatus_get(dev, &link);
1244
1245         nfp_net_link_update(dev, 0);
1246
1247         /* likely to up */
1248         if (!link.link_status) {
1249                 /* handle it 1 sec later, wait it being stable */
1250                 timeout = NFP_NET_LINK_UP_CHECK_TIMEOUT;
1251                 /* likely to down */
1252         } else {
1253                 /* handle it 4 sec later, wait it being stable */
1254                 timeout = NFP_NET_LINK_DOWN_CHECK_TIMEOUT;
1255         }
1256
1257         if (rte_eal_alarm_set(timeout * 1000,
1258                               nfp_net_dev_interrupt_delayed_handler,
1259                               (void *)dev) < 0) {
1260                 PMD_INIT_LOG(ERR, "Error setting alarm");
1261                 /* Unmasking */
1262                 nfp_net_irq_unmask(dev);
1263         }
1264 }
1265
1266 /*
1267  * Interrupt handler which shall be registered for alarm callback for delayed
1268  * handling specific interrupt to wait for the stable nic state. As the NIC
1269  * interrupt state is not stable for nfp after link is just down, it needs
1270  * to wait 4 seconds to get the stable status.
1271  *
1272  * @param handle   Pointer to interrupt handle.
1273  * @param param    The address of parameter (struct rte_eth_dev *)
1274  *
1275  * @return  void
1276  */
1277 static void
1278 nfp_net_dev_interrupt_delayed_handler(void *param)
1279 {
1280         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1281
1282         nfp_net_link_update(dev, 0);
1283         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1284
1285         nfp_net_dev_link_status_print(dev);
1286
1287         /* Unmasking */
1288         nfp_net_irq_unmask(dev);
1289 }
1290
1291 static int
1292 nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1293 {
1294         struct nfp_net_hw *hw;
1295
1296         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1297
1298         /* check that mtu is within the allowed range */
1299         if (mtu < RTE_ETHER_MIN_MTU || (uint32_t)mtu > hw->max_mtu)
1300                 return -EINVAL;
1301
1302         /* mtu setting is forbidden if port is started */
1303         if (dev->data->dev_started) {
1304                 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration",
1305                             dev->data->port_id);
1306                 return -EBUSY;
1307         }
1308
1309         /* switch to jumbo mode if needed */
1310         if ((uint32_t)mtu > RTE_ETHER_MTU)
1311                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
1312         else
1313                 dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1314
1315         /* update max frame size */
1316         dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu;
1317
1318         /* writing to configuration space */
1319         nn_cfg_writel(hw, NFP_NET_CFG_MTU, (uint32_t)mtu);
1320
1321         hw->mtu = mtu;
1322
1323         return 0;
1324 }
1325
1326 static int
1327 nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1328 {
1329         uint32_t new_ctrl, update;
1330         struct nfp_net_hw *hw;
1331         int ret;
1332
1333         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1334         new_ctrl = 0;
1335
1336         /* Enable vlan strip if it is not configured yet */
1337         if ((mask & ETH_VLAN_STRIP_OFFLOAD) &&
1338             !(hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN))
1339                 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_RXVLAN;
1340
1341         /* Disable vlan strip just if it is configured */
1342         if (!(mask & ETH_VLAN_STRIP_OFFLOAD) &&
1343             (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN))
1344                 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN;
1345
1346         if (new_ctrl == 0)
1347                 return 0;
1348
1349         update = NFP_NET_CFG_UPDATE_GEN;
1350
1351         ret = nfp_net_reconfig(hw, new_ctrl, update);
1352         if (!ret)
1353                 hw->ctrl = new_ctrl;
1354
1355         return ret;
1356 }
1357
1358 static int
1359 nfp_net_rss_reta_write(struct rte_eth_dev *dev,
1360                     struct rte_eth_rss_reta_entry64 *reta_conf,
1361                     uint16_t reta_size)
1362 {
1363         uint32_t reta, mask;
1364         int i, j;
1365         int idx, shift;
1366         struct nfp_net_hw *hw =
1367                 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368
1369         if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) {
1370                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
1371                         "(%d) doesn't match the number hardware can supported "
1372                         "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ);
1373                 return -EINVAL;
1374         }
1375
1376         /*
1377          * Update Redirection Table. There are 128 8bit-entries which can be
1378          * manage as 32 32bit-entries
1379          */
1380         for (i = 0; i < reta_size; i += 4) {
1381                 /* Handling 4 RSS entries per loop */
1382                 idx = i / RTE_RETA_GROUP_SIZE;
1383                 shift = i % RTE_RETA_GROUP_SIZE;
1384                 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF);
1385
1386                 if (!mask)
1387                         continue;
1388
1389                 reta = 0;
1390                 /* If all 4 entries were set, don't need read RETA register */
1391                 if (mask != 0xF)
1392                         reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + i);
1393
1394                 for (j = 0; j < 4; j++) {
1395                         if (!(mask & (0x1 << j)))
1396                                 continue;
1397                         if (mask != 0xF)
1398                                 /* Clearing the entry bits */
1399                                 reta &= ~(0xFF << (8 * j));
1400                         reta |= reta_conf[idx].reta[shift + j] << (8 * j);
1401                 }
1402                 nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift,
1403                               reta);
1404         }
1405         return 0;
1406 }
1407
1408 /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */
1409 static int
1410 nfp_net_reta_update(struct rte_eth_dev *dev,
1411                     struct rte_eth_rss_reta_entry64 *reta_conf,
1412                     uint16_t reta_size)
1413 {
1414         struct nfp_net_hw *hw =
1415                 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1416         uint32_t update;
1417         int ret;
1418
1419         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1420                 return -EINVAL;
1421
1422         ret = nfp_net_rss_reta_write(dev, reta_conf, reta_size);
1423         if (ret != 0)
1424                 return ret;
1425
1426         update = NFP_NET_CFG_UPDATE_RSS;
1427
1428         if (nfp_net_reconfig(hw, hw->ctrl, update) < 0)
1429                 return -EIO;
1430
1431         return 0;
1432 }
1433
1434  /* Query Redirection Table(RETA) of Receive Side Scaling of Ethernet device. */
1435 static int
1436 nfp_net_reta_query(struct rte_eth_dev *dev,
1437                    struct rte_eth_rss_reta_entry64 *reta_conf,
1438                    uint16_t reta_size)
1439 {
1440         uint8_t i, j, mask;
1441         int idx, shift;
1442         uint32_t reta;
1443         struct nfp_net_hw *hw;
1444
1445         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1446
1447         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1448                 return -EINVAL;
1449
1450         if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) {
1451                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
1452                         "(%d) doesn't match the number hardware can supported "
1453                         "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ);
1454                 return -EINVAL;
1455         }
1456
1457         /*
1458          * Reading Redirection Table. There are 128 8bit-entries which can be
1459          * manage as 32 32bit-entries
1460          */
1461         for (i = 0; i < reta_size; i += 4) {
1462                 /* Handling 4 RSS entries per loop */
1463                 idx = i / RTE_RETA_GROUP_SIZE;
1464                 shift = i % RTE_RETA_GROUP_SIZE;
1465                 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF);
1466
1467                 if (!mask)
1468                         continue;
1469
1470                 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) +
1471                                     shift);
1472                 for (j = 0; j < 4; j++) {
1473                         if (!(mask & (0x1 << j)))
1474                                 continue;
1475                         reta_conf[idx].reta[shift + j] =
1476                                 (uint8_t)((reta >> (8 * j)) & 0xF);
1477                 }
1478         }
1479         return 0;
1480 }
1481
1482 static int
1483 nfp_net_rss_hash_write(struct rte_eth_dev *dev,
1484                         struct rte_eth_rss_conf *rss_conf)
1485 {
1486         struct nfp_net_hw *hw;
1487         uint64_t rss_hf;
1488         uint32_t cfg_rss_ctrl = 0;
1489         uint8_t key;
1490         int i;
1491
1492         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1493
1494         /* Writing the key byte a byte */
1495         for (i = 0; i < rss_conf->rss_key_len; i++) {
1496                 memcpy(&key, &rss_conf->rss_key[i], 1);
1497                 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY + i, key);
1498         }
1499
1500         rss_hf = rss_conf->rss_hf;
1501
1502         if (rss_hf & ETH_RSS_IPV4)
1503                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4;
1504
1505         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1506                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_TCP;
1507
1508         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
1509                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_UDP;
1510
1511         if (rss_hf & ETH_RSS_IPV6)
1512                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6;
1513
1514         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1515                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_TCP;
1516
1517         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
1518                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_UDP;
1519
1520         cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK;
1521         cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ;
1522
1523         /* configuring where to apply the RSS hash */
1524         nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl);
1525
1526         /* Writing the key size */
1527         nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY_SZ, rss_conf->rss_key_len);
1528
1529         return 0;
1530 }
1531
1532 static int
1533 nfp_net_rss_hash_update(struct rte_eth_dev *dev,
1534                         struct rte_eth_rss_conf *rss_conf)
1535 {
1536         uint32_t update;
1537         uint64_t rss_hf;
1538         struct nfp_net_hw *hw;
1539
1540         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1541
1542         rss_hf = rss_conf->rss_hf;
1543
1544         /* Checking if RSS is enabled */
1545         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) {
1546                 if (rss_hf != 0) { /* Enable RSS? */
1547                         PMD_DRV_LOG(ERR, "RSS unsupported");
1548                         return -EINVAL;
1549                 }
1550                 return 0; /* Nothing to do */
1551         }
1552
1553         if (rss_conf->rss_key_len > NFP_NET_CFG_RSS_KEY_SZ) {
1554                 PMD_DRV_LOG(ERR, "hash key too long");
1555                 return -EINVAL;
1556         }
1557
1558         nfp_net_rss_hash_write(dev, rss_conf);
1559
1560         update = NFP_NET_CFG_UPDATE_RSS;
1561
1562         if (nfp_net_reconfig(hw, hw->ctrl, update) < 0)
1563                 return -EIO;
1564
1565         return 0;
1566 }
1567
1568 static int
1569 nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev,
1570                           struct rte_eth_rss_conf *rss_conf)
1571 {
1572         uint64_t rss_hf;
1573         uint32_t cfg_rss_ctrl;
1574         uint8_t key;
1575         int i;
1576         struct nfp_net_hw *hw;
1577
1578         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1579
1580         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1581                 return -EINVAL;
1582
1583         rss_hf = rss_conf->rss_hf;
1584         cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL);
1585
1586         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4)
1587                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP;
1588
1589         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP)
1590                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
1591
1592         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_TCP)
1593                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
1594
1595         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_UDP)
1596                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
1597
1598         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_UDP)
1599                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
1600
1601         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6)
1602                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP;
1603
1604         /* Propagate current RSS hash functions to caller */
1605         rss_conf->rss_hf = rss_hf;
1606
1607         /* Reading the key size */
1608         rss_conf->rss_key_len = nn_cfg_readl(hw, NFP_NET_CFG_RSS_KEY_SZ);
1609
1610         /* Reading the key byte a byte */
1611         for (i = 0; i < rss_conf->rss_key_len; i++) {
1612                 key = nn_cfg_readb(hw, NFP_NET_CFG_RSS_KEY + i);
1613                 memcpy(&rss_conf->rss_key[i], &key, 1);
1614         }
1615
1616         return 0;
1617 }
1618
1619 static int
1620 nfp_net_rss_config_default(struct rte_eth_dev *dev)
1621 {
1622         struct rte_eth_conf *dev_conf;
1623         struct rte_eth_rss_conf rss_conf;
1624         struct rte_eth_rss_reta_entry64 nfp_reta_conf[2];
1625         uint16_t rx_queues = dev->data->nb_rx_queues;
1626         uint16_t queue;
1627         int i, j, ret;
1628
1629         PMD_DRV_LOG(INFO, "setting default RSS conf for %u queues",
1630                 rx_queues);
1631
1632         nfp_reta_conf[0].mask = ~0x0;
1633         nfp_reta_conf[1].mask = ~0x0;
1634
1635         queue = 0;
1636         for (i = 0; i < 0x40; i += 8) {
1637                 for (j = i; j < (i + 8); j++) {
1638                         nfp_reta_conf[0].reta[j] = queue;
1639                         nfp_reta_conf[1].reta[j] = queue++;
1640                         queue %= rx_queues;
1641                 }
1642         }
1643         ret = nfp_net_rss_reta_write(dev, nfp_reta_conf, 0x80);
1644         if (ret != 0)
1645                 return ret;
1646
1647         dev_conf = &dev->data->dev_conf;
1648         if (!dev_conf) {
1649                 PMD_DRV_LOG(INFO, "wrong rss conf");
1650                 return -EINVAL;
1651         }
1652         rss_conf = dev_conf->rx_adv_conf.rss_conf;
1653
1654         ret = nfp_net_rss_hash_write(dev, &rss_conf);
1655
1656         return ret;
1657 }
1658
1659
1660 /* Initialise and register driver with DPDK Application */
1661 static const struct eth_dev_ops nfp_net_eth_dev_ops = {
1662         .dev_configure          = nfp_net_configure,
1663         .dev_start              = nfp_net_start,
1664         .dev_stop               = nfp_net_stop,
1665         .dev_set_link_up        = nfp_net_set_link_up,
1666         .dev_set_link_down      = nfp_net_set_link_down,
1667         .dev_close              = nfp_net_close,
1668         .promiscuous_enable     = nfp_net_promisc_enable,
1669         .promiscuous_disable    = nfp_net_promisc_disable,
1670         .link_update            = nfp_net_link_update,
1671         .stats_get              = nfp_net_stats_get,
1672         .stats_reset            = nfp_net_stats_reset,
1673         .dev_infos_get          = nfp_net_infos_get,
1674         .dev_supported_ptypes_get = nfp_net_supported_ptypes_get,
1675         .mtu_set                = nfp_net_dev_mtu_set,
1676         .mac_addr_set           = nfp_set_mac_addr,
1677         .vlan_offload_set       = nfp_net_vlan_offload_set,
1678         .reta_update            = nfp_net_reta_update,
1679         .reta_query             = nfp_net_reta_query,
1680         .rss_hash_update        = nfp_net_rss_hash_update,
1681         .rss_hash_conf_get      = nfp_net_rss_hash_conf_get,
1682         .rx_queue_setup         = nfp_net_rx_queue_setup,
1683         .rx_queue_release       = nfp_net_rx_queue_release,
1684         .tx_queue_setup         = nfp_net_tx_queue_setup,
1685         .tx_queue_release       = nfp_net_tx_queue_release,
1686         .rx_queue_intr_enable   = nfp_rx_queue_intr_enable,
1687         .rx_queue_intr_disable  = nfp_rx_queue_intr_disable,
1688 };
1689
1690
1691 static int
1692 nfp_net_init(struct rte_eth_dev *eth_dev)
1693 {
1694         struct rte_pci_device *pci_dev;
1695         struct nfp_pf_dev *pf_dev;
1696         struct nfp_net_hw *hw;
1697
1698         uint64_t tx_bar_off = 0, rx_bar_off = 0;
1699         uint32_t start_q;
1700         int stride = 4;
1701         int port = 0;
1702         int err;
1703
1704         PMD_INIT_FUNC_TRACE();
1705
1706         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1707
1708         /* Use backpointer here to the PF of this eth_dev */
1709         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private);
1710
1711         /* NFP can not handle DMA addresses requiring more than 40 bits */
1712         if (rte_mem_check_dma_mask(40)) {
1713                 RTE_LOG(ERR, PMD, "device %s can not be used:",
1714                                    pci_dev->device.name);
1715                 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n");
1716                 return -ENODEV;
1717         };
1718
1719         if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) ||
1720             (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) {
1721                 port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx;
1722                 if (port < 0 || port > 7) {
1723                         PMD_DRV_LOG(ERR, "Port value is wrong");
1724                         return -ENODEV;
1725                 }
1726
1727                 /* Use PF array of physical ports to get pointer to
1728                  * this specific port
1729                  */
1730                 hw = pf_dev->ports[port];
1731
1732                 PMD_INIT_LOG(DEBUG, "Working with physical port number: %d, "
1733                                     "NFP internal port number: %d",
1734                                     port, hw->nfp_idx);
1735
1736         } else {
1737                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1738         }
1739
1740         eth_dev->dev_ops = &nfp_net_eth_dev_ops;
1741         eth_dev->rx_queue_count = nfp_net_rx_queue_count;
1742         eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
1743         eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
1744
1745         /* For secondary processes, the primary has done all the work */
1746         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1747                 return 0;
1748
1749         rte_eth_copy_pci_info(eth_dev, pci_dev);
1750
1751         hw->device_id = pci_dev->id.device_id;
1752         hw->vendor_id = pci_dev->id.vendor_id;
1753         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1754         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1755
1756         PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u",
1757                      pci_dev->id.vendor_id, pci_dev->id.device_id,
1758                      pci_dev->addr.domain, pci_dev->addr.bus,
1759                      pci_dev->addr.devid, pci_dev->addr.function);
1760
1761         hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr;
1762         if (hw->ctrl_bar == NULL) {
1763                 PMD_DRV_LOG(ERR,
1764                         "hw->ctrl_bar is NULL. BAR0 not configured");
1765                 return -ENODEV;
1766         }
1767
1768         if (hw->is_phyport) {
1769                 if (port == 0) {
1770                         hw->ctrl_bar = pf_dev->ctrl_bar;
1771                 } else {
1772                         if (!pf_dev->ctrl_bar)
1773                                 return -ENODEV;
1774                         /* Use port offset in pf ctrl_bar for this
1775                          * ports control bar
1776                          */
1777                         hw->ctrl_bar = pf_dev->ctrl_bar +
1778                                        (port * NFP_PF_CSR_SLICE_SIZE);
1779                 }
1780         }
1781
1782         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar);
1783
1784         hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS);
1785         hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS);
1786
1787         /* Work out where in the BAR the queues start. */
1788         switch (pci_dev->id.device_id) {
1789         case PCI_DEVICE_ID_NFP4000_PF_NIC:
1790         case PCI_DEVICE_ID_NFP6000_PF_NIC:
1791         case PCI_DEVICE_ID_NFP6000_VF_NIC:
1792                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
1793                 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
1794                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
1795                 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
1796                 break;
1797         default:
1798                 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
1799                 err = -ENODEV;
1800                 goto dev_err_ctrl_map;
1801         }
1802
1803         PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off);
1804         PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off);
1805
1806         if (hw->is_phyport) {
1807                 hw->tx_bar = pf_dev->hw_queues + tx_bar_off;
1808                 hw->rx_bar = pf_dev->hw_queues + rx_bar_off;
1809                 eth_dev->data->dev_private = hw;
1810         } else {
1811                 hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr +
1812                              tx_bar_off;
1813                 hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr +
1814                              rx_bar_off;
1815         }
1816
1817         PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p",
1818                      hw->ctrl_bar, hw->tx_bar, hw->rx_bar);
1819
1820         nfp_net_cfg_queue_setup(hw);
1821
1822         /* Get some of the read-only fields from the config BAR */
1823         hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION);
1824         hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP);
1825         hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU);
1826         hw->mtu = RTE_ETHER_MTU;
1827
1828         /* VLAN insertion is incompatible with LSOv2 */
1829         if (hw->cap & NFP_NET_CFG_CTRL_LSO2)
1830                 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN;
1831
1832         if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2)
1833                 hw->rx_offset = NFP_NET_RX_OFFSET;
1834         else
1835                 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR);
1836
1837         PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d",
1838                            NFD_CFG_MAJOR_VERSION_of(hw->ver),
1839                            NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu);
1840
1841         PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap,
1842                      hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "",
1843                      hw->cap & NFP_NET_CFG_CTRL_L2BC    ? "L2BCFILT " : "",
1844                      hw->cap & NFP_NET_CFG_CTRL_L2MC    ? "L2MCFILT " : "",
1845                      hw->cap & NFP_NET_CFG_CTRL_RXCSUM  ? "RXCSUM "  : "",
1846                      hw->cap & NFP_NET_CFG_CTRL_TXCSUM  ? "TXCSUM "  : "",
1847                      hw->cap & NFP_NET_CFG_CTRL_RXVLAN  ? "RXVLAN "  : "",
1848                      hw->cap & NFP_NET_CFG_CTRL_TXVLAN  ? "TXVLAN "  : "",
1849                      hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "",
1850                      hw->cap & NFP_NET_CFG_CTRL_GATHER  ? "GATHER "  : "",
1851                      hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR "  : "",
1852                      hw->cap & NFP_NET_CFG_CTRL_LSO     ? "TSO "     : "",
1853                      hw->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSOv2 "     : "",
1854                      hw->cap & NFP_NET_CFG_CTRL_RSS     ? "RSS "     : "",
1855                      hw->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSSv2 "     : "");
1856
1857         hw->ctrl = 0;
1858
1859         hw->stride_rx = stride;
1860         hw->stride_tx = stride;
1861
1862         PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u",
1863                      hw->max_rx_queues, hw->max_tx_queues);
1864
1865         /* Initializing spinlock for reconfigs */
1866         rte_spinlock_init(&hw->reconfig_lock);
1867
1868         /* Allocating memory for mac addr */
1869         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
1870                                                RTE_ETHER_ADDR_LEN, 0);
1871         if (eth_dev->data->mac_addrs == NULL) {
1872                 PMD_INIT_LOG(ERR, "Failed to space for MAC address");
1873                 err = -ENOMEM;
1874                 goto dev_err_queues_map;
1875         }
1876
1877         if (hw->is_phyport) {
1878                 nfp_net_pf_read_mac(pf_dev, port);
1879                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
1880         } else {
1881                 nfp_net_vf_read_mac(hw);
1882         }
1883
1884         if (!rte_is_valid_assigned_ether_addr(
1885                     (struct rte_ether_addr *)&hw->mac_addr)) {
1886                 PMD_INIT_LOG(INFO, "Using random mac address for port %d",
1887                                    port);
1888                 /* Using random mac addresses for VFs */
1889                 rte_eth_random_addr(&hw->mac_addr[0]);
1890                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
1891         }
1892
1893         /* Copying mac address to DPDK eth_dev struct */
1894         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
1895                         &eth_dev->data->mac_addrs[0]);
1896
1897         if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
1898                 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR;
1899
1900         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1901
1902         PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x "
1903                      "mac=%02x:%02x:%02x:%02x:%02x:%02x",
1904                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1905                      pci_dev->id.device_id,
1906                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1907                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1908
1909         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1910                 /* Registering LSC interrupt handler */
1911                 rte_intr_callback_register(&pci_dev->intr_handle,
1912                                            nfp_net_dev_interrupt_handler,
1913                                            (void *)eth_dev);
1914                 /* Telling the firmware about the LSC interrupt entry */
1915                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
1916                 /* Recording current stats counters values */
1917                 nfp_net_stats_reset(eth_dev);
1918         }
1919
1920         return 0;
1921
1922 dev_err_queues_map:
1923                 nfp_cpp_area_free(hw->hwqueues_area);
1924 dev_err_ctrl_map:
1925                 nfp_cpp_area_free(hw->ctrl_area);
1926
1927         return err;
1928 }
1929
1930 #define DEFAULT_FW_PATH       "/lib/firmware/netronome"
1931
1932 static int
1933 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
1934 {
1935         struct nfp_cpp *cpp = nsp->cpp;
1936         void *fw_buf;
1937         char fw_name[125];
1938         char serial[40];
1939         size_t fsize;
1940
1941         /* Looking for firmware file in order of priority */
1942
1943         /* First try to find a firmware image specific for this device */
1944         snprintf(serial, sizeof(serial),
1945                         "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x",
1946                 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3],
1947                 cpp->serial[4], cpp->serial[5], cpp->interface >> 8,
1948                 cpp->interface & 0xff);
1949
1950         snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
1951                         serial);
1952         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
1953         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
1954                 goto load_fw;
1955
1956         /* Then try the PCI name */
1957         snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
1958                         dev->device.name);
1959         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
1960         if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
1961                 goto load_fw;
1962
1963         /* Finally try the card type and media */
1964         snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
1965         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
1966         if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
1967                 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
1968                 return -ENOENT;
1969         }
1970
1971 load_fw:
1972         PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
1973                 fw_name, fsize);
1974
1975         PMD_DRV_LOG(INFO, "Uploading the firmware ...");
1976         nfp_nsp_load_fw(nsp, fw_buf, fsize);
1977         PMD_DRV_LOG(INFO, "Done");
1978
1979         free(fw_buf);
1980         return 0;
1981 }
1982
1983 static int
1984 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp,
1985              struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo)
1986 {
1987         struct nfp_nsp *nsp;
1988         const char *nfp_fw_model;
1989         char card_desc[100];
1990         int err = 0;
1991
1992         nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno");
1993
1994         if (nfp_fw_model) {
1995                 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model);
1996         } else {
1997                 PMD_DRV_LOG(ERR, "firmware model NOT found");
1998                 return -EIO;
1999         }
2000
2001         if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) {
2002                 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u",
2003                        nfp_eth_table->count);
2004                 return -EIO;
2005         }
2006
2007         PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports",
2008                            nfp_eth_table->count);
2009
2010         PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed);
2011
2012         snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw",
2013                         nfp_fw_model, nfp_eth_table->count,
2014                         nfp_eth_table->ports[0].speed / 1000);
2015
2016         nsp = nfp_nsp_open(cpp);
2017         if (!nsp) {
2018                 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle");
2019                 return -EIO;
2020         }
2021
2022         nfp_nsp_device_soft_reset(nsp);
2023         err = nfp_fw_upload(dev, nsp, card_desc);
2024
2025         nfp_nsp_close(nsp);
2026         return err;
2027 }
2028
2029 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev)
2030 {
2031         struct nfp_net_hw *hw;
2032         struct rte_eth_dev *eth_dev;
2033         struct nfp_eth_table *nfp_eth_table = NULL;
2034         int ret = 0;
2035         int i;
2036
2037         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
2038         if (!nfp_eth_table) {
2039                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
2040                 ret = -EIO;
2041                 goto error;
2042         }
2043
2044         /* Loop through all physical ports on PF */
2045         for (i = 0; i < pf_dev->total_phyports; i++) {
2046                 const unsigned int numa_node = rte_socket_id();
2047                 char port_name[RTE_ETH_NAME_MAX_LEN];
2048
2049                 snprintf(port_name, sizeof(port_name), "%s_port%d",
2050                          pf_dev->pci_dev->device.name, i);
2051
2052                 /* Allocate a eth_dev for this phyport */
2053                 eth_dev = rte_eth_dev_allocate(port_name);
2054                 if (!eth_dev) {
2055                         ret = -ENODEV;
2056                         goto port_cleanup;
2057                 }
2058
2059                 /* Allocate memory for this phyport */
2060                 eth_dev->data->dev_private =
2061                         rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw),
2062                                            RTE_CACHE_LINE_SIZE, numa_node);
2063                 if (!eth_dev->data->dev_private) {
2064                         ret = -ENOMEM;
2065                         rte_eth_dev_release_port(eth_dev);
2066                         goto port_cleanup;
2067                 }
2068
2069                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
2070
2071                 /* Add this device to the PF's array of physical ports */
2072                 pf_dev->ports[i] = hw;
2073
2074                 hw->pf_dev = pf_dev;
2075                 hw->cpp = pf_dev->cpp;
2076                 hw->eth_dev = eth_dev;
2077                 hw->idx = i;
2078                 hw->nfp_idx = nfp_eth_table->ports[i].index;
2079                 hw->is_phyport = true;
2080
2081                 eth_dev->device = &pf_dev->pci_dev->device;
2082
2083                 /* ctrl/tx/rx BAR mappings and remaining init happens in
2084                  * nfp_net_init
2085                  */
2086                 ret = nfp_net_init(eth_dev);
2087
2088                 if (ret) {
2089                         ret = -ENODEV;
2090                         goto port_cleanup;
2091                 }
2092
2093                 rte_eth_dev_probing_finish(eth_dev);
2094
2095         } /* End loop, all ports on this PF */
2096         ret = 0;
2097         goto eth_table_cleanup;
2098
2099 port_cleanup:
2100         for (i = 0; i < pf_dev->total_phyports; i++) {
2101                 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) {
2102                         struct rte_eth_dev *tmp_dev;
2103                         tmp_dev = pf_dev->ports[i]->eth_dev;
2104                         rte_eth_dev_release_port(tmp_dev);
2105                         pf_dev->ports[i] = NULL;
2106                 }
2107         }
2108 eth_table_cleanup:
2109         free(nfp_eth_table);
2110 error:
2111         return ret;
2112 }
2113
2114 static int nfp_pf_init(struct rte_pci_device *pci_dev)
2115 {
2116         struct nfp_pf_dev *pf_dev = NULL;
2117         struct nfp_cpp *cpp;
2118         struct nfp_hwinfo *hwinfo;
2119         struct nfp_rtsym_table *sym_tbl;
2120         struct nfp_eth_table *nfp_eth_table = NULL;
2121         char name[RTE_ETH_NAME_MAX_LEN];
2122         int total_ports;
2123         int ret = -ENODEV;
2124         int err;
2125
2126         if (!pci_dev)
2127                 return ret;
2128
2129         /*
2130          * When device bound to UIO, the device could be used, by mistake,
2131          * by two DPDK apps, and the UIO driver does not avoid it. This
2132          * could lead to a serious problem when configuring the NFP CPP
2133          * interface. Here we avoid this telling to the CPP init code to
2134          * use a lock file if UIO is being used.
2135          */
2136         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
2137                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
2138         else
2139                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
2140
2141         if (!cpp) {
2142                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
2143                 ret = -EIO;
2144                 goto error;
2145         }
2146
2147         hwinfo = nfp_hwinfo_read(cpp);
2148         if (!hwinfo) {
2149                 PMD_INIT_LOG(ERR, "Error reading hwinfo table");
2150                 ret = -EIO;
2151                 goto error;
2152         }
2153
2154         nfp_eth_table = nfp_eth_read_ports(cpp);
2155         if (!nfp_eth_table) {
2156                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
2157                 ret = -EIO;
2158                 goto hwinfo_cleanup;
2159         }
2160
2161         if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) {
2162                 PMD_INIT_LOG(ERR, "Error when uploading firmware");
2163                 ret = -EIO;
2164                 goto eth_table_cleanup;
2165         }
2166
2167         /* Now the symbol table should be there */
2168         sym_tbl = nfp_rtsym_table_read(cpp);
2169         if (!sym_tbl) {
2170                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
2171                                 " symbol table");
2172                 ret = -EIO;
2173                 goto eth_table_cleanup;
2174         }
2175
2176         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
2177         if (total_ports != (int)nfp_eth_table->count) {
2178                 PMD_DRV_LOG(ERR, "Inconsistent number of ports");
2179                 ret = -EIO;
2180                 goto sym_tbl_cleanup;
2181         }
2182
2183         PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports);
2184
2185         if (total_ports <= 0 || total_ports > 8) {
2186                 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
2187                 ret = -ENODEV;
2188                 goto sym_tbl_cleanup;
2189         }
2190         /* Allocate memory for the PF "device" */
2191         snprintf(name, sizeof(name), "nfp_pf%d", 0);
2192         pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
2193         if (!pf_dev) {
2194                 ret = -ENOMEM;
2195                 goto sym_tbl_cleanup;
2196         }
2197
2198         /* Populate the newly created PF device */
2199         pf_dev->cpp = cpp;
2200         pf_dev->hwinfo = hwinfo;
2201         pf_dev->sym_tbl = sym_tbl;
2202         pf_dev->total_phyports = total_ports;
2203
2204         if (total_ports > 1)
2205                 pf_dev->multiport = true;
2206
2207         pf_dev->pci_dev = pci_dev;
2208
2209         /* Map the symbol table */
2210         pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0",
2211                                      pf_dev->total_phyports * 32768,
2212                                      &pf_dev->ctrl_area);
2213         if (!pf_dev->ctrl_bar) {
2214                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar");
2215                 ret = -EIO;
2216                 goto pf_cleanup;
2217         }
2218
2219         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar);
2220
2221         /* configure access to tx/rx vNIC BARs */
2222         pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0,
2223                                               NFP_PCIE_QUEUE(0),
2224                                               NFP_QCP_QUEUE_AREA_SZ,
2225                                               &pf_dev->hwqueues_area);
2226         if (!pf_dev->hw_queues) {
2227                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
2228                 ret = -EIO;
2229                 goto ctrl_area_cleanup;
2230         }
2231
2232         PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues);
2233
2234         /* Initialize and prep physical ports now
2235          * This will loop through all physical ports
2236          */
2237         ret = nfp_init_phyports(pf_dev);
2238         if (ret) {
2239                 PMD_INIT_LOG(ERR, "Could not create physical ports");
2240                 goto hwqueues_cleanup;
2241         }
2242
2243         /* register the CPP bridge service here for primary use */
2244         nfp_register_cpp_service(pf_dev->cpp);
2245
2246         return 0;
2247
2248 hwqueues_cleanup:
2249         nfp_cpp_area_free(pf_dev->hwqueues_area);
2250 ctrl_area_cleanup:
2251         nfp_cpp_area_free(pf_dev->ctrl_area);
2252 pf_cleanup:
2253         rte_free(pf_dev);
2254 sym_tbl_cleanup:
2255         free(sym_tbl);
2256 eth_table_cleanup:
2257         free(nfp_eth_table);
2258 hwinfo_cleanup:
2259         free(hwinfo);
2260 error:
2261         return ret;
2262 }
2263
2264 /*
2265  * When attaching to the NFP4000/6000 PF on a secondary process there
2266  * is no need to initialize the PF again. Only minimal work is required
2267  * here
2268  */
2269 static int nfp_pf_secondary_init(struct rte_pci_device *pci_dev)
2270 {
2271         struct nfp_cpp *cpp;
2272         struct nfp_rtsym_table *sym_tbl;
2273         int total_ports;
2274         int i;
2275         int err;
2276
2277         if (!pci_dev)
2278                 return -ENODEV;
2279
2280         /*
2281          * When device bound to UIO, the device could be used, by mistake,
2282          * by two DPDK apps, and the UIO driver does not avoid it. This
2283          * could lead to a serious problem when configuring the NFP CPP
2284          * interface. Here we avoid this telling to the CPP init code to
2285          * use a lock file if UIO is being used.
2286          */
2287         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
2288                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
2289         else
2290                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
2291
2292         if (!cpp) {
2293                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
2294                 return -EIO;
2295         }
2296
2297         /*
2298          * We don't have access to the PF created in the primary process
2299          * here so we have to read the number of ports from firmware
2300          */
2301         sym_tbl = nfp_rtsym_table_read(cpp);
2302         if (!sym_tbl) {
2303                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
2304                                 " symbol table");
2305                 return -EIO;
2306         }
2307
2308         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
2309
2310         for (i = 0; i < total_ports; i++) {
2311                 struct rte_eth_dev *eth_dev;
2312                 char port_name[RTE_ETH_NAME_MAX_LEN];
2313
2314                 snprintf(port_name, sizeof(port_name), "%s_port%d",
2315                          pci_dev->device.name, i);
2316
2317                 PMD_DRV_LOG(DEBUG, "Secondary attaching to port %s",
2318                     port_name);
2319                 eth_dev = rte_eth_dev_attach_secondary(port_name);
2320                 if (!eth_dev) {
2321                         RTE_LOG(ERR, EAL,
2322                         "secondary process attach failed, "
2323                         "ethdev doesn't exist");
2324                         return -ENODEV;
2325                 }
2326                 eth_dev->process_private = cpp;
2327                 eth_dev->dev_ops = &nfp_net_eth_dev_ops;
2328                 eth_dev->rx_queue_count = nfp_net_rx_queue_count;
2329                 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
2330                 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
2331                 rte_eth_dev_probing_finish(eth_dev);
2332         }
2333
2334         /* Register the CPP bridge service for the secondary too */
2335         nfp_register_cpp_service(cpp);
2336
2337         return 0;
2338 }
2339
2340 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2341                             struct rte_pci_device *dev)
2342 {
2343         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2344                 return nfp_pf_init(dev);
2345         else
2346                 return nfp_pf_secondary_init(dev);
2347 }
2348
2349 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = {
2350         {
2351                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
2352                                PCI_DEVICE_ID_NFP4000_PF_NIC)
2353         },
2354         {
2355                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
2356                                PCI_DEVICE_ID_NFP6000_PF_NIC)
2357         },
2358         {
2359                 .vendor_id = 0,
2360         },
2361 };
2362
2363 static const struct rte_pci_id pci_id_nfp_vf_net_map[] = {
2364         {
2365                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
2366                                PCI_DEVICE_ID_NFP6000_VF_NIC)
2367         },
2368         {
2369                 .vendor_id = 0,
2370         },
2371 };
2372
2373 static int nfp_pci_uninit(struct rte_eth_dev *eth_dev)
2374 {
2375         struct rte_pci_device *pci_dev;
2376         uint16_t port_id;
2377
2378         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2379
2380         if (pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC ||
2381             pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC) {
2382                 /* Free up all physical ports under PF */
2383                 RTE_ETH_FOREACH_DEV_OF(port_id, &pci_dev->device)
2384                         rte_eth_dev_close(port_id);
2385                 /*
2386                  * Ports can be closed and freed but hotplugging is not
2387                  * currently supported
2388                  */
2389                 return -ENOTSUP;
2390         }
2391
2392         /* VF cleanup, just free private port data */
2393         return nfp_net_close(eth_dev);
2394 }
2395
2396 static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2397         struct rte_pci_device *pci_dev)
2398 {
2399         return rte_eth_dev_pci_generic_probe(pci_dev,
2400                 sizeof(struct nfp_net_adapter), nfp_net_init);
2401 }
2402
2403 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev)
2404 {
2405         return rte_eth_dev_pci_generic_remove(pci_dev, nfp_pci_uninit);
2406 }
2407
2408 static struct rte_pci_driver rte_nfp_net_pf_pmd = {
2409         .id_table = pci_id_nfp_pf_net_map,
2410         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2411         .probe = nfp_pf_pci_probe,
2412         .remove = eth_nfp_pci_remove,
2413 };
2414
2415 static struct rte_pci_driver rte_nfp_net_vf_pmd = {
2416         .id_table = pci_id_nfp_vf_net_map,
2417         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2418         .probe = eth_nfp_pci_probe,
2419         .remove = eth_nfp_pci_remove,
2420 };
2421
2422 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd);
2423 RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd);
2424 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map);
2425 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map);
2426 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio");
2427 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio");
2428 RTE_LOG_REGISTER_SUFFIX(nfp_logtype_init, init, NOTICE);
2429 RTE_LOG_REGISTER_SUFFIX(nfp_logtype_driver, driver, NOTICE);
2430 /*
2431  * Local variables:
2432  * c-file-style: "Linux"
2433  * indent-tabs-mode: t
2434  * End:
2435  */