drivers/net/nfp/nfp_net.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2014-2018 Netronome Systems, Inc.
   3  * All rights reserved.
   4  *
   5  * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.
   6  */
   7
   8 /*
   9  * vim:shiftwidth=8:noexpandtab
  10  *
  11  * @file dpdk/pmd/nfp_net.c
  12  *
  13  * Netronome vNIC DPDK Poll-Mode Driver: Main entry point
  14  */
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <ethdev_driver.h>
  21 #include <ethdev_pci.h>
  22 #include <rte_dev.h>
  23 #include <rte_ether.h>
  24 #include <rte_malloc.h>
  25 #include <rte_memzone.h>
  26 #include <rte_mempool.h>
  27 #include <rte_version.h>
  28 #include <rte_string_fns.h>
  29 #include <rte_alarm.h>
  30 #include <rte_spinlock.h>
  31 #include <rte_service_component.h>
  32
  33 #include "nfpcore/nfp_cpp.h"
  34 #include "nfpcore/nfp_nffw.h"
  35 #include "nfpcore/nfp_hwinfo.h"
  36 #include "nfpcore/nfp_mip.h"
  37 #include "nfpcore/nfp_rtsym.h"
  38 #include "nfpcore/nfp_nsp.h"
  39
  40 #include "nfp_net_pmd.h"
  41 #include "nfp_net_logs.h"
  42 #include "nfp_net_ctrl.h"
  43
  44 #include <sys/types.h>
  45 #include <sys/socket.h>
  46 #include <sys/un.h>
  47 #include <unistd.h>
  48 #include <stdio.h>
  49 #include <sys/ioctl.h>
  50 #include <errno.h>
  51
  52 /* Prototypes */
  53 static int nfp_net_close(struct rte_eth_dev *dev);
  54 static int nfp_net_configure(struct rte_eth_dev *dev);
  55 static void nfp_net_dev_interrupt_handler(void *param);
  56 static void nfp_net_dev_interrupt_delayed_handler(void *param);
  57 static int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
  58 static int nfp_net_infos_get(struct rte_eth_dev *dev,
  59                              struct rte_eth_dev_info *dev_info);
  60 static int nfp_net_init(struct rte_eth_dev *eth_dev);
  61 static int nfp_pf_init(struct rte_eth_dev *eth_dev);
  62 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev);
  63 static int nfp_net_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  64 static int nfp_net_promisc_enable(struct rte_eth_dev *dev);
  65 static int nfp_net_promisc_disable(struct rte_eth_dev *dev);
  66 static int nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq);
  67 static uint32_t nfp_net_rx_queue_count(struct rte_eth_dev *dev,
  68                                        uint16_t queue_idx);
  69 static uint16_t nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
  70                                   uint16_t nb_pkts);
  71 static void nfp_net_rx_queue_release(void *rxq);
  72 static int nfp_net_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
  73                                   uint16_t nb_desc, unsigned int socket_id,
  74                                   const struct rte_eth_rxconf *rx_conf,
  75                                   struct rte_mempool *mp);
  76 static int nfp_net_tx_free_bufs(struct nfp_net_txq *txq);
  77 static void nfp_net_tx_queue_release(void *txq);
  78 static int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
  79                                   uint16_t nb_desc, unsigned int socket_id,
  80                                   const struct rte_eth_txconf *tx_conf);
  81 static int nfp_net_start(struct rte_eth_dev *dev);
  82 static int nfp_net_stats_get(struct rte_eth_dev *dev,
  83                               struct rte_eth_stats *stats);
  84 static int nfp_net_stats_reset(struct rte_eth_dev *dev);
  85 static int nfp_net_stop(struct rte_eth_dev *dev);
  86 static uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
  87                                   uint16_t nb_pkts);
  88
  89 static int nfp_net_rss_config_default(struct rte_eth_dev *dev);
  90 static int nfp_net_rss_hash_update(struct rte_eth_dev *dev,
  91                                    struct rte_eth_rss_conf *rss_conf);
  92 static int nfp_net_rss_reta_write(struct rte_eth_dev *dev,
  93                     struct rte_eth_rss_reta_entry64 *reta_conf,
  94                     uint16_t reta_size);
  95 static int nfp_net_rss_hash_write(struct rte_eth_dev *dev,
  96                         struct rte_eth_rss_conf *rss_conf);
  97 static int nfp_set_mac_addr(struct rte_eth_dev *dev,
  98                              struct rte_ether_addr *mac_addr);
  99 static int32_t nfp_cpp_bridge_service_func(void *args);
 100 static int nfp_fw_setup(struct rte_pci_device *dev,
 101                         struct nfp_cpp *cpp,
 102                         struct nfp_eth_table *nfp_eth_table,
 103                         struct nfp_hwinfo *hwinfo);
 104
 105
 106 /* The offset of the queue controller queues in the PCIe Target */
 107 #define NFP_PCIE_QUEUE(_q) (0x80000 + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & 0xff)))
 108
 109 /* Maximum value which can be added to a queue with one transaction */
 110 #define NFP_QCP_MAX_ADD 0x7f
 111
 112 #define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \
 113         (uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM)
 114
 115 /* nfp_qcp_ptr - Read or Write Pointer of a queue */
 116 enum nfp_qcp_ptr {
 117         NFP_QCP_READ_PTR = 0,
 118         NFP_QCP_WRITE_PTR
 119 };
 120
 121 /*
 122  * nfp_qcp_ptr_add - Add the value to the selected pointer of a queue
 123  * @q: Base address for queue structure
 124  * @ptr: Add to the Read or Write pointer
 125  * @val: Value to add to the queue pointer
 126  *
 127  * If @val is greater than @NFP_QCP_MAX_ADD multiple writes are performed.
 128  */
 129 static inline void
 130 nfp_qcp_ptr_add(uint8_t *q, enum nfp_qcp_ptr ptr, uint32_t val)
 131 {
 132         uint32_t off;
 133
 134         if (ptr == NFP_QCP_READ_PTR)
 135                 off = NFP_QCP_QUEUE_ADD_RPTR;
 136         else
 137                 off = NFP_QCP_QUEUE_ADD_WPTR;
 138
 139         while (val > NFP_QCP_MAX_ADD) {
 140                 nn_writel(rte_cpu_to_le_32(NFP_QCP_MAX_ADD), q + off);
 141                 val -= NFP_QCP_MAX_ADD;
 142         }
 143
 144         nn_writel(rte_cpu_to_le_32(val), q + off);
 145 }
 146
 147 /*
 148  * nfp_qcp_read - Read the current Read/Write pointer value for a queue
 149  * @q:  Base address for queue structure
 150  * @ptr: Read or Write pointer
 151  */
 152 static inline uint32_t
 153 nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr)
 154 {
 155         uint32_t off;
 156         uint32_t val;
 157
 158         if (ptr == NFP_QCP_READ_PTR)
 159                 off = NFP_QCP_QUEUE_STS_LO;
 160         else
 161                 off = NFP_QCP_QUEUE_STS_HI;
 162
 163         val = rte_cpu_to_le_32(nn_readl(q + off));
 164
 165         if (ptr == NFP_QCP_READ_PTR)
 166                 return val & NFP_QCP_QUEUE_STS_LO_READPTR_mask;
 167         else
 168                 return val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask;
 169 }
 170
 171 /*
 172  * Functions to read/write from/to Config BAR
 173  * Performs any endian conversion necessary.
 174  */
 175 static inline uint8_t
 176 nn_cfg_readb(struct nfp_net_hw *hw, int off)
 177 {
 178         return nn_readb(hw->ctrl_bar + off);
 179 }
 180
 181 static inline void
 182 nn_cfg_writeb(struct nfp_net_hw *hw, int off, uint8_t val)
 183 {
 184         nn_writeb(val, hw->ctrl_bar + off);
 185 }
 186
 187 static inline uint32_t
 188 nn_cfg_readl(struct nfp_net_hw *hw, int off)
 189 {
 190         return rte_le_to_cpu_32(nn_readl(hw->ctrl_bar + off));
 191 }
 192
 193 static inline void
 194 nn_cfg_writel(struct nfp_net_hw *hw, int off, uint32_t val)
 195 {
 196         nn_writel(rte_cpu_to_le_32(val), hw->ctrl_bar + off);
 197 }
 198
 199 static inline uint64_t
 200 nn_cfg_readq(struct nfp_net_hw *hw, int off)
 201 {
 202         return rte_le_to_cpu_64(nn_readq(hw->ctrl_bar + off));
 203 }
 204
 205 static inline void
 206 nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val)
 207 {
 208         nn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off);
 209 }
 210
 211 static void
 212 nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq)
 213 {
 214         unsigned i;
 215
 216         if (rxq->rxbufs == NULL)
 217                 return;
 218
 219         for (i = 0; i < rxq->rx_count; i++) {
 220                 if (rxq->rxbufs[i].mbuf) {
 221                         rte_pktmbuf_free_seg(rxq->rxbufs[i].mbuf);
 222                         rxq->rxbufs[i].mbuf = NULL;
 223                 }
 224         }
 225 }
 226
 227 static void
 228 nfp_net_rx_queue_release(void *rx_queue)
 229 {
 230         struct nfp_net_rxq *rxq = rx_queue;
 231
 232         if (rxq) {
 233                 nfp_net_rx_queue_release_mbufs(rxq);
 234                 rte_free(rxq->rxbufs);
 235                 rte_free(rxq);
 236         }
 237 }
 238
 239 static void
 240 nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq)
 241 {
 242         nfp_net_rx_queue_release_mbufs(rxq);
 243         rxq->rd_p = 0;
 244         rxq->nb_rx_hold = 0;
 245 }
 246
 247 static void
 248 nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq)
 249 {
 250         unsigned i;
 251
 252         if (txq->txbufs == NULL)
 253                 return;
 254
 255         for (i = 0; i < txq->tx_count; i++) {
 256                 if (txq->txbufs[i].mbuf) {
 257                         rte_pktmbuf_free_seg(txq->txbufs[i].mbuf);
 258                         txq->txbufs[i].mbuf = NULL;
 259                 }
 260         }
 261 }
 262
 263 static void
 264 nfp_net_tx_queue_release(void *tx_queue)
 265 {
 266         struct nfp_net_txq *txq = tx_queue;
 267
 268         if (txq) {
 269                 nfp_net_tx_queue_release_mbufs(txq);
 270                 rte_free(txq->txbufs);
 271                 rte_free(txq);
 272         }
 273 }
 274
 275 static void
 276 nfp_net_reset_tx_queue(struct nfp_net_txq *txq)
 277 {
 278         nfp_net_tx_queue_release_mbufs(txq);
 279         txq->wr_p = 0;
 280         txq->rd_p = 0;
 281 }
 282
 283 static int
 284 __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update)
 285 {
 286         int cnt;
 287         uint32_t new;
 288         struct timespec wait;
 289
 290         PMD_DRV_LOG(DEBUG, "Writing to the configuration queue (%p)...",
 291                     hw->qcp_cfg);
 292
 293         if (hw->qcp_cfg == NULL)
 294                 rte_panic("Bad configuration queue pointer\n");
 295
 296         nfp_qcp_ptr_add(hw->qcp_cfg, NFP_QCP_WRITE_PTR, 1);
 297
 298         wait.tv_sec = 0;
 299         wait.tv_nsec = 1000000;
 300
 301         PMD_DRV_LOG(DEBUG, "Polling for update ack...");
 302
 303         /* Poll update field, waiting for NFP to ack the config */
 304         for (cnt = 0; ; cnt++) {
 305                 new = nn_cfg_readl(hw, NFP_NET_CFG_UPDATE);
 306                 if (new == 0)
 307                         break;
 308                 if (new & NFP_NET_CFG_UPDATE_ERR) {
 309                         PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new);
 310                         return -1;
 311                 }
 312                 if (cnt >= NFP_NET_POLL_TIMEOUT) {
 313                         PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after"
 314                                           " %dms", update, cnt);
 315                         rte_panic("Exiting\n");
 316                 }
 317                 nanosleep(&wait, 0); /* waiting for a 1ms */
 318         }
 319         PMD_DRV_LOG(DEBUG, "Ack DONE");
 320         return 0;
 321 }
 322
 323 /*
 324  * Reconfigure the NIC
 325  * @nn:    device to reconfigure
 326  * @ctrl:    The value for the ctrl field in the BAR config
 327  * @update:  The value for the update field in the BAR config
 328  *
 329  * Write the update word to the BAR and ping the reconfig queue. Then poll
 330  * until the firmware has acknowledged the update by zeroing the update word.
 331  */
 332 static int
 333 nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update)
 334 {
 335         uint32_t err;
 336
 337         PMD_DRV_LOG(DEBUG, "nfp_net_reconfig: ctrl=%08x update=%08x",
 338                     ctrl, update);
 339
 340         rte_spinlock_lock(&hw->reconfig_lock);
 341
 342         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, ctrl);
 343         nn_cfg_writel(hw, NFP_NET_CFG_UPDATE, update);
 344
 345         rte_wmb();
 346
 347         err = __nfp_net_reconfig(hw, update);
 348
 349         rte_spinlock_unlock(&hw->reconfig_lock);
 350
 351         if (!err)
 352                 return 0;
 353
 354         /*
 355          * Reconfig errors imply situations where they can be handled.
 356          * Otherwise, rte_panic is called inside __nfp_net_reconfig
 357          */
 358         PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x",
 359                      ctrl, update);
 360         return -EIO;
 361 }
 362
 363 /*
 364  * Configure an Ethernet device. This function must be invoked first
 365  * before any other function in the Ethernet API. This function can
 366  * also be re-invoked when a device is in the stopped state.
 367  */
 368 static int
 369 nfp_net_configure(struct rte_eth_dev *dev)
 370 {
 371         struct rte_eth_conf *dev_conf;
 372         struct rte_eth_rxmode *rxmode;
 373         struct rte_eth_txmode *txmode;
 374         struct nfp_net_hw *hw;
 375
 376         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 377
 378         /*
 379          * A DPDK app sends info about how many queues to use and how
 380          * those queues need to be configured. This is used by the
 381          * DPDK core and it makes sure no more queues than those
 382          * advertised by the driver are requested. This function is
 383          * called after that internal process
 384          */
 385
 386         PMD_INIT_LOG(DEBUG, "Configure");
 387
 388         dev_conf = &dev->data->dev_conf;
 389         rxmode = &dev_conf->rxmode;
 390         txmode = &dev_conf->txmode;
 391
 392         if (rxmode->mq_mode & ETH_MQ_RX_RSS_FLAG)
 393                 rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 394
 395         /* Checking TX mode */
 396         if (txmode->mq_mode) {
 397                 PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported");
 398                 return -EINVAL;
 399         }
 400
 401         /* Checking RX mode */
 402         if (rxmode->mq_mode & ETH_MQ_RX_RSS &&
 403             !(hw->cap & NFP_NET_CFG_CTRL_RSS)) {
 404                 PMD_INIT_LOG(INFO, "RSS not supported");
 405                 return -EINVAL;
 406         }
 407
 408         return 0;
 409 }
 410
 411 static void
 412 nfp_net_enable_queues(struct rte_eth_dev *dev)
 413 {
 414         struct nfp_net_hw *hw;
 415         uint64_t enabled_queues = 0;
 416         int i;
 417
 418         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 419
 420         /* Enabling the required TX queues in the device */
 421         for (i = 0; i < dev->data->nb_tx_queues; i++)
 422                 enabled_queues |= (1 << i);
 423
 424         nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, enabled_queues);
 425
 426         enabled_queues = 0;
 427
 428         /* Enabling the required RX queues in the device */
 429         for (i = 0; i < dev->data->nb_rx_queues; i++)
 430                 enabled_queues |= (1 << i);
 431
 432         nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, enabled_queues);
 433 }
 434
 435 static void
 436 nfp_net_disable_queues(struct rte_eth_dev *dev)
 437 {
 438         struct nfp_net_hw *hw;
 439         uint32_t new_ctrl, update = 0;
 440
 441         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 442
 443         nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, 0);
 444         nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, 0);
 445
 446         new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_ENABLE;
 447         update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING |
 448                  NFP_NET_CFG_UPDATE_MSIX;
 449
 450         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 451                 new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG;
 452
 453         /* If an error when reconfig we avoid to change hw state */
 454         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 455                 return;
 456
 457         hw->ctrl = new_ctrl;
 458 }
 459
 460 static int
 461 nfp_net_rx_freelist_setup(struct rte_eth_dev *dev)
 462 {
 463         int i;
 464
 465         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 466                 if (nfp_net_rx_fill_freelist(dev->data->rx_queues[i]) < 0)
 467                         return -1;
 468         }
 469         return 0;
 470 }
 471
 472 static void
 473 nfp_net_params_setup(struct nfp_net_hw *hw)
 474 {
 475         nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu);
 476         nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz);
 477 }
 478
 479 static void
 480 nfp_net_cfg_queue_setup(struct nfp_net_hw *hw)
 481 {
 482         hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ;
 483 }
 484
 485 #define ETH_ADDR_LEN    6
 486
 487 static void
 488 nfp_eth_copy_mac(uint8_t *dst, const uint8_t *src)
 489 {
 490         int i;
 491
 492         for (i = 0; i < ETH_ADDR_LEN; i++)
 493                 dst[i] = src[i];
 494 }
 495
 496 static int
 497 nfp_net_pf_read_mac(struct nfp_pf_dev *pf_dev, int port)
 498 {
 499         struct nfp_eth_table *nfp_eth_table;
 500         struct nfp_net_hw *hw = NULL;
 501
 502         /* Grab a pointer to the correct physical port */
 503         hw = pf_dev->ports[port];
 504
 505         nfp_eth_table = nfp_eth_read_ports(pf_dev->cpp);
 506
 507         nfp_eth_copy_mac((uint8_t *)&hw->mac_addr,
 508                          (uint8_t *)&nfp_eth_table->ports[port].mac_addr);
 509
 510         free(nfp_eth_table);
 511         return 0;
 512 }
 513
 514 static void
 515 nfp_net_vf_read_mac(struct nfp_net_hw *hw)
 516 {
 517         uint32_t tmp;
 518
 519         tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR));
 520         memcpy(&hw->mac_addr[0], &tmp, 4);
 521
 522         tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4));
 523         memcpy(&hw->mac_addr[4], &tmp, 2);
 524 }
 525
 526 static void
 527 nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac)
 528 {
 529         uint32_t mac0 = *(uint32_t *)mac;
 530         uint16_t mac1;
 531
 532         nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR);
 533
 534         mac += 4;
 535         mac1 = *(uint16_t *)mac;
 536         nn_writew(rte_cpu_to_be_16(mac1),
 537                   hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6);
 538 }
 539
 540 int
 541 nfp_set_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
 542 {
 543         struct nfp_net_hw *hw;
 544         uint32_t update, ctrl;
 545
 546         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 547         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 548             !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) {
 549                 PMD_INIT_LOG(INFO, "MAC address unable to change when"
 550                                   " port enabled");
 551                 return -EBUSY;
 552         }
 553
 554         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 555             !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 556                 return -EBUSY;
 557
 558         /* Writing new MAC to the specific port BAR address */
 559         nfp_net_write_mac(hw, (uint8_t *)mac_addr);
 560
 561         /* Signal the NIC about the change */
 562         update = NFP_NET_CFG_UPDATE_MACADDR;
 563         ctrl = hw->ctrl;
 564         if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) &&
 565             (hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
 566                 ctrl |= NFP_NET_CFG_CTRL_LIVE_ADDR;
 567         if (nfp_net_reconfig(hw, ctrl, update) < 0) {
 568                 PMD_INIT_LOG(INFO, "MAC address update failed");
 569                 return -EIO;
 570         }
 571         return 0;
 572 }
 573
 574 static int
 575 nfp_configure_rx_interrupt(struct rte_eth_dev *dev,
 576                            struct rte_intr_handle *intr_handle)
 577 {
 578         struct nfp_net_hw *hw;
 579         int i;
 580
 581         if (!intr_handle->intr_vec) {
 582                 intr_handle->intr_vec =
 583                         rte_zmalloc("intr_vec",
 584                                     dev->data->nb_rx_queues * sizeof(int), 0);
 585                 if (!intr_handle->intr_vec) {
 586                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
 587                                      " intr_vec", dev->data->nb_rx_queues);
 588                         return -ENOMEM;
 589                 }
 590         }
 591
 592         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 593
 594         if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 595                 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO");
 596                 /* UIO just supports one queue and no LSC*/
 597                 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0);
 598                 intr_handle->intr_vec[0] = 0;
 599         } else {
 600                 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO");
 601                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 602                         /*
 603                          * The first msix vector is reserved for non
 604                          * efd interrupts
 605                         */
 606                         nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1);
 607                         intr_handle->intr_vec[i] = i + 1;
 608                         PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d", i,
 609                                             intr_handle->intr_vec[i]);
 610                 }
 611         }
 612
 613         /* Avoiding TX interrupts */
 614         hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF;
 615         return 0;
 616 }
 617
 618 static uint32_t
 619 nfp_check_offloads(struct rte_eth_dev *dev)
 620 {
 621         struct nfp_net_hw *hw;
 622         struct rte_eth_conf *dev_conf;
 623         struct rte_eth_rxmode *rxmode;
 624         struct rte_eth_txmode *txmode;
 625         uint32_t ctrl = 0;
 626
 627         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 628
 629         dev_conf = &dev->data->dev_conf;
 630         rxmode = &dev_conf->rxmode;
 631         txmode = &dev_conf->txmode;
 632
 633         if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
 634                 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM)
 635                         ctrl |= NFP_NET_CFG_CTRL_RXCSUM;
 636         }
 637
 638         if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) {
 639                 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN)
 640                         ctrl |= NFP_NET_CFG_CTRL_RXVLAN;
 641         }
 642
 643         if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 644                 hw->mtu = rxmode->max_rx_pkt_len;
 645
 646         if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT)
 647                 ctrl |= NFP_NET_CFG_CTRL_TXVLAN;
 648
 649         /* L2 broadcast */
 650         if (hw->cap & NFP_NET_CFG_CTRL_L2BC)
 651                 ctrl |= NFP_NET_CFG_CTRL_L2BC;
 652
 653         /* L2 multicast */
 654         if (hw->cap & NFP_NET_CFG_CTRL_L2MC)
 655                 ctrl |= NFP_NET_CFG_CTRL_L2MC;
 656
 657         /* TX checksum offload */
 658         if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM ||
 659             txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM ||
 660             txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
 661                 ctrl |= NFP_NET_CFG_CTRL_TXCSUM;
 662
 663         /* LSO offload */
 664         if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) {
 665                 if (hw->cap & NFP_NET_CFG_CTRL_LSO)
 666                         ctrl |= NFP_NET_CFG_CTRL_LSO;
 667                 else
 668                         ctrl |= NFP_NET_CFG_CTRL_LSO2;
 669         }
 670
 671         /* RX gather */
 672         if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 673                 ctrl |= NFP_NET_CFG_CTRL_GATHER;
 674
 675         return ctrl;
 676 }
 677
 678 static int
 679 nfp_net_start(struct rte_eth_dev *dev)
 680 {
 681         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 682         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 683         uint32_t new_ctrl, update = 0;
 684         struct nfp_net_hw *hw;
 685         struct nfp_pf_dev *pf_dev;
 686         struct rte_eth_conf *dev_conf;
 687         struct rte_eth_rxmode *rxmode;
 688         uint32_t intr_vector;
 689         int ret;
 690
 691         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 692         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(dev->data->dev_private);
 693
 694         PMD_INIT_LOG(DEBUG, "Start");
 695
 696         /* Disabling queues just in case... */
 697         nfp_net_disable_queues(dev);
 698
 699         /* Enabling the required queues in the device */
 700         nfp_net_enable_queues(dev);
 701
 702         /* check and configure queue intr-vector mapping */
 703         if (dev->data->dev_conf.intr_conf.rxq != 0) {
 704                 if (pf_dev->multiport) {
 705                         PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported "
 706                                           "with NFP multiport PF");
 707                                 return -EINVAL;
 708                 }
 709                 if (intr_handle->type == RTE_INTR_HANDLE_UIO) {
 710                         /*
 711                          * Better not to share LSC with RX interrupts.
 712                          * Unregistering LSC interrupt handler
 713                          */
 714                         rte_intr_callback_unregister(&pci_dev->intr_handle,
 715                                 nfp_net_dev_interrupt_handler, (void *)dev);
 716
 717                         if (dev->data->nb_rx_queues > 1) {
 718                                 PMD_INIT_LOG(ERR, "PMD rx interrupt only "
 719                                              "supports 1 queue with UIO");
 720                                 return -EIO;
 721                         }
 722                 }
 723                 intr_vector = dev->data->nb_rx_queues;
 724                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 725                         return -1;
 726
 727                 nfp_configure_rx_interrupt(dev, intr_handle);
 728                 update = NFP_NET_CFG_UPDATE_MSIX;
 729         }
 730
 731         rte_intr_enable(intr_handle);
 732
 733         new_ctrl = nfp_check_offloads(dev);
 734
 735         /* Writing configuration parameters in the device */
 736         nfp_net_params_setup(hw);
 737
 738         dev_conf = &dev->data->dev_conf;
 739         rxmode = &dev_conf->rxmode;
 740
 741         if (rxmode->mq_mode & ETH_MQ_RX_RSS) {
 742                 nfp_net_rss_config_default(dev);
 743                 update |= NFP_NET_CFG_UPDATE_RSS;
 744                 new_ctrl |= NFP_NET_CFG_CTRL_RSS;
 745         }
 746
 747         /* Enable device */
 748         new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
 749
 750         update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING;
 751
 752         if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG)
 753                 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;
 754
 755         nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl);
 756         if (nfp_net_reconfig(hw, new_ctrl, update) < 0)
 757                 return -EIO;
 758
 759         /*
 760          * Allocating rte mbufs for configured rx queues.
 761          * This requires queues being enabled before
 762          */
 763         if (nfp_net_rx_freelist_setup(dev) < 0) {
 764                 ret = -ENOMEM;
 765                 goto error;
 766         }
 767
 768         if (hw->is_phyport) {
 769                 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 770                         /* Configure the physical port up */
 771                         nfp_eth_set_configured(hw->cpp, hw->idx, 1);
 772                 else
 773                         nfp_eth_set_configured(dev->process_private,
 774                                                hw->idx, 1);
 775         }
 776
 777         hw->ctrl = new_ctrl;
 778
 779         return 0;
 780
 781 error:
 782         /*
 783          * An error returned by this function should mean the app
 784          * exiting and then the system releasing all the memory
 785          * allocated even memory coming from hugepages.
 786          *
 787          * The device could be enabled at this point with some queues
 788          * ready for getting packets. This is true if the call to
 789          * nfp_net_rx_freelist_setup() succeeds for some queues but
 790          * fails for subsequent queues.
 791          *
 792          * This should make the app exiting but better if we tell the
 793          * device first.
 794          */
 795         nfp_net_disable_queues(dev);
 796
 797         return ret;
 798 }
 799
 800 /* Stop device: disable rx and tx functions to allow for reconfiguring. */
 801 static int
 802 nfp_net_stop(struct rte_eth_dev *dev)
 803 {
 804         int i;
 805         struct nfp_net_hw *hw;
 806
 807         PMD_INIT_LOG(DEBUG, "Stop");
 808
 809         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 810
 811         nfp_net_disable_queues(dev);
 812
 813         /* Clear queues */
 814         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 815                 nfp_net_reset_tx_queue(
 816                         (struct nfp_net_txq *)dev->data->tx_queues[i]);
 817         }
 818
 819         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 820                 nfp_net_reset_rx_queue(
 821                         (struct nfp_net_rxq *)dev->data->rx_queues[i]);
 822         }
 823
 824         if (hw->is_phyport) {
 825                 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 826                         /* Configure the physical port down */
 827                         nfp_eth_set_configured(hw->cpp, hw->idx, 0);
 828                 else
 829                         nfp_eth_set_configured(dev->process_private,
 830                                                hw->idx, 0);
 831         }
 832
 833         return 0;
 834 }
 835
 836 /* Set the link up. */
 837 static int
 838 nfp_net_set_link_up(struct rte_eth_dev *dev)
 839 {
 840         struct nfp_net_hw *hw;
 841
 842         PMD_DRV_LOG(DEBUG, "Set link up");
 843
 844         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 845
 846         if (!hw->is_phyport)
 847                 return -ENOTSUP;
 848
 849         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 850                 /* Configure the physical port down */
 851                 return nfp_eth_set_configured(hw->cpp, hw->idx, 1);
 852         else
 853                 return nfp_eth_set_configured(dev->process_private,
 854                                               hw->idx, 1);
 855 }
 856
 857 /* Set the link down. */
 858 static int
 859 nfp_net_set_link_down(struct rte_eth_dev *dev)
 860 {
 861         struct nfp_net_hw *hw;
 862
 863         PMD_DRV_LOG(DEBUG, "Set link down");
 864
 865         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 866
 867         if (!hw->is_phyport)
 868                 return -ENOTSUP;
 869
 870         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 871                 /* Configure the physical port down */
 872                 return nfp_eth_set_configured(hw->cpp, hw->idx, 0);
 873         else
 874                 return nfp_eth_set_configured(dev->process_private,
 875                                               hw->idx, 0);
 876 }
 877
 878 /* Reset and stop device. The device can not be restarted. */
 879 static int
 880 nfp_net_close(struct rte_eth_dev *dev)
 881 {
 882         struct nfp_net_hw *hw;
 883         struct rte_pci_device *pci_dev;
 884         int i;
 885
 886         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 887                 return 0;
 888
 889         PMD_INIT_LOG(DEBUG, "Close");
 890
 891         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 892         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 893
 894         /*
 895          * We assume that the DPDK application is stopping all the
 896          * threads/queues before calling the device close function.
 897          */
 898
 899         nfp_net_disable_queues(dev);
 900
 901         /* Clear queues */
 902         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 903                 nfp_net_reset_tx_queue(
 904                         (struct nfp_net_txq *)dev->data->tx_queues[i]);
 905         }
 906
 907         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 908                 nfp_net_reset_rx_queue(
 909                         (struct nfp_net_rxq *)dev->data->rx_queues[i]);
 910         }
 911
 912         rte_intr_disable(&pci_dev->intr_handle);
 913         nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff);
 914
 915         /* unregister callback func from eal lib */
 916         rte_intr_callback_unregister(&pci_dev->intr_handle,
 917                                      nfp_net_dev_interrupt_handler,
 918                                      (void *)dev);
 919
 920         /*
 921          * The ixgbe PMD driver disables the pcie master on the
 922          * device. The i40e does not...
 923          */
 924
 925         return 0;
 926 }
 927
 928 static int
 929 nfp_net_promisc_enable(struct rte_eth_dev *dev)
 930 {
 931         uint32_t new_ctrl, update = 0;
 932         struct nfp_net_hw *hw;
 933         int ret;
 934
 935         PMD_DRV_LOG(DEBUG, "Promiscuous mode enable");
 936
 937         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 938
 939         if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) {
 940                 PMD_INIT_LOG(INFO, "Promiscuous mode not supported");
 941                 return -ENOTSUP;
 942         }
 943
 944         if (hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) {
 945                 PMD_DRV_LOG(INFO, "Promiscuous mode already enabled");
 946                 return 0;
 947         }
 948
 949         new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_PROMISC;
 950         update = NFP_NET_CFG_UPDATE_GEN;
 951
 952         /*
 953          * DPDK sets promiscuous mode on just after this call assuming
 954          * it can not fail ...
 955          */
 956         ret = nfp_net_reconfig(hw, new_ctrl, update);
 957         if (ret < 0)
 958                 return ret;
 959
 960         hw->ctrl = new_ctrl;
 961
 962         return 0;
 963 }
 964
 965 static int
 966 nfp_net_promisc_disable(struct rte_eth_dev *dev)
 967 {
 968         uint32_t new_ctrl, update = 0;
 969         struct nfp_net_hw *hw;
 970         int ret;
 971
 972         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 973
 974         if ((hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) == 0) {
 975                 PMD_DRV_LOG(INFO, "Promiscuous mode already disabled");
 976                 return 0;
 977         }
 978
 979         new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_PROMISC;
 980         update = NFP_NET_CFG_UPDATE_GEN;
 981
 982         /*
 983          * DPDK sets promiscuous mode off just before this call
 984          * assuming it can not fail ...
 985          */
 986         ret = nfp_net_reconfig(hw, new_ctrl, update);
 987         if (ret < 0)
 988                 return ret;
 989
 990         hw->ctrl = new_ctrl;
 991
 992         return 0;
 993 }
 994
 995 /*
 996  * return 0 means link status changed, -1 means not changed
 997  *
 998  * Wait to complete is needed as it can take up to 9 seconds to get the Link
 999  * status.
1000  */
1001 static int
1002 nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
1003 {
1004         struct nfp_net_hw *hw;
1005         struct rte_eth_link link;
1006         uint32_t nn_link_status;
1007         int ret;
1008
1009         static const uint32_t ls_to_ethtool[] = {
1010                 [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE,
1011                 [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN]     = ETH_SPEED_NUM_NONE,
1012                 [NFP_NET_CFG_STS_LINK_RATE_1G]          = ETH_SPEED_NUM_1G,
1013                 [NFP_NET_CFG_STS_LINK_RATE_10G]         = ETH_SPEED_NUM_10G,
1014                 [NFP_NET_CFG_STS_LINK_RATE_25G]         = ETH_SPEED_NUM_25G,
1015                 [NFP_NET_CFG_STS_LINK_RATE_40G]         = ETH_SPEED_NUM_40G,
1016                 [NFP_NET_CFG_STS_LINK_RATE_50G]         = ETH_SPEED_NUM_50G,
1017                 [NFP_NET_CFG_STS_LINK_RATE_100G]        = ETH_SPEED_NUM_100G,
1018         };
1019
1020         PMD_DRV_LOG(DEBUG, "Link update");
1021
1022         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1023
1024         nn_link_status = nn_cfg_readl(hw, NFP_NET_CFG_STS);
1025
1026         memset(&link, 0, sizeof(struct rte_eth_link));
1027
1028         if (nn_link_status & NFP_NET_CFG_STS_LINK)
1029                 link.link_status = ETH_LINK_UP;
1030
1031         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1032
1033         nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) &
1034                          NFP_NET_CFG_STS_LINK_RATE_MASK;
1035
1036         if (nn_link_status >= RTE_DIM(ls_to_ethtool))
1037                 link.link_speed = ETH_SPEED_NUM_NONE;
1038         else
1039                 link.link_speed = ls_to_ethtool[nn_link_status];
1040
1041         ret = rte_eth_linkstatus_set(dev, &link);
1042         if (ret == 0) {
1043                 if (link.link_status)
1044                         PMD_DRV_LOG(INFO, "NIC Link is Up");
1045                 else
1046                         PMD_DRV_LOG(INFO, "NIC Link is Down");
1047         }
1048         return ret;
1049 }
1050
1051 static int
1052 nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1053 {
1054         int i;
1055         struct nfp_net_hw *hw;
1056         struct rte_eth_stats nfp_dev_stats;
1057
1058         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1059
1060         /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */
1061
1062         memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats));
1063
1064         /* reading per RX ring stats */
1065         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1066                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
1067                         break;
1068
1069                 nfp_dev_stats.q_ipackets[i] =
1070                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i));
1071
1072                 nfp_dev_stats.q_ipackets[i] -=
1073                         hw->eth_stats_base.q_ipackets[i];
1074
1075                 nfp_dev_stats.q_ibytes[i] =
1076                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8);
1077
1078                 nfp_dev_stats.q_ibytes[i] -=
1079                         hw->eth_stats_base.q_ibytes[i];
1080         }
1081
1082         /* reading per TX ring stats */
1083         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1084                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
1085                         break;
1086
1087                 nfp_dev_stats.q_opackets[i] =
1088                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i));
1089
1090                 nfp_dev_stats.q_opackets[i] -=
1091                         hw->eth_stats_base.q_opackets[i];
1092
1093                 nfp_dev_stats.q_obytes[i] =
1094                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8);
1095
1096                 nfp_dev_stats.q_obytes[i] -=
1097                         hw->eth_stats_base.q_obytes[i];
1098         }
1099
1100         nfp_dev_stats.ipackets =
1101                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES);
1102
1103         nfp_dev_stats.ipackets -= hw->eth_stats_base.ipackets;
1104
1105         nfp_dev_stats.ibytes =
1106                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS);
1107
1108         nfp_dev_stats.ibytes -= hw->eth_stats_base.ibytes;
1109
1110         nfp_dev_stats.opackets =
1111                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES);
1112
1113         nfp_dev_stats.opackets -= hw->eth_stats_base.opackets;
1114
1115         nfp_dev_stats.obytes =
1116                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS);
1117
1118         nfp_dev_stats.obytes -= hw->eth_stats_base.obytes;
1119
1120         /* reading general device stats */
1121         nfp_dev_stats.ierrors =
1122                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS);
1123
1124         nfp_dev_stats.ierrors -= hw->eth_stats_base.ierrors;
1125
1126         nfp_dev_stats.oerrors =
1127                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS);
1128
1129         nfp_dev_stats.oerrors -= hw->eth_stats_base.oerrors;
1130
1131         /* RX ring mbuf allocation failures */
1132         nfp_dev_stats.rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1133
1134         nfp_dev_stats.imissed =
1135                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS);
1136
1137         nfp_dev_stats.imissed -= hw->eth_stats_base.imissed;
1138
1139         if (stats) {
1140                 memcpy(stats, &nfp_dev_stats, sizeof(*stats));
1141                 return 0;
1142         }
1143         return -EINVAL;
1144 }
1145
1146 static int
1147 nfp_net_stats_reset(struct rte_eth_dev *dev)
1148 {
1149         int i;
1150         struct nfp_net_hw *hw;
1151
1152         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1153
1154         /*
1155          * hw->eth_stats_base records the per counter starting point.
1156          * Lets update it now
1157          */
1158
1159         /* reading per RX ring stats */
1160         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1161                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
1162                         break;
1163
1164                 hw->eth_stats_base.q_ipackets[i] =
1165                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i));
1166
1167                 hw->eth_stats_base.q_ibytes[i] =
1168                         nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8);
1169         }
1170
1171         /* reading per TX ring stats */
1172         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1173                 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS)
1174                         break;
1175
1176                 hw->eth_stats_base.q_opackets[i] =
1177                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i));
1178
1179                 hw->eth_stats_base.q_obytes[i] =
1180                         nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8);
1181         }
1182
1183         hw->eth_stats_base.ipackets =
1184                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES);
1185
1186         hw->eth_stats_base.ibytes =
1187                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS);
1188
1189         hw->eth_stats_base.opackets =
1190                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES);
1191
1192         hw->eth_stats_base.obytes =
1193                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS);
1194
1195         /* reading general device stats */
1196         hw->eth_stats_base.ierrors =
1197                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS);
1198
1199         hw->eth_stats_base.oerrors =
1200                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS);
1201
1202         /* RX ring mbuf allocation failures */
1203         dev->data->rx_mbuf_alloc_failed = 0;
1204
1205         hw->eth_stats_base.imissed =
1206                 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS);
1207
1208         return 0;
1209 }
1210
1211 static int
1212 nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1213 {
1214         struct nfp_net_hw *hw;
1215
1216         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1217
1218         dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues;
1219         dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues;
1220         dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
1221         dev_info->max_rx_pktlen = hw->max_mtu;
1222         /* Next should change when PF support is implemented */
1223         dev_info->max_mac_addrs = 1;
1224
1225         if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN)
1226                 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1227
1228         if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM)
1229                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM |
1230                                              DEV_RX_OFFLOAD_UDP_CKSUM |
1231                                              DEV_RX_OFFLOAD_TCP_CKSUM;
1232
1233         dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME |
1234                                      DEV_RX_OFFLOAD_RSS_HASH;
1235
1236         if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)
1237                 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
1238
1239         if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM)
1240                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM |
1241                                              DEV_TX_OFFLOAD_UDP_CKSUM |
1242                                              DEV_TX_OFFLOAD_TCP_CKSUM;
1243
1244         if (hw->cap & NFP_NET_CFG_CTRL_LSO_ANY)
1245                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
1246
1247         if (hw->cap & NFP_NET_CFG_CTRL_GATHER)
1248                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MULTI_SEGS;
1249
1250         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1251                 .rx_thresh = {
1252                         .pthresh = DEFAULT_RX_PTHRESH,
1253                         .hthresh = DEFAULT_RX_HTHRESH,
1254                         .wthresh = DEFAULT_RX_WTHRESH,
1255                 },
1256                 .rx_free_thresh = DEFAULT_RX_FREE_THRESH,
1257                 .rx_drop_en = 0,
1258         };
1259
1260         dev_info->default_txconf = (struct rte_eth_txconf) {
1261                 .tx_thresh = {
1262                         .pthresh = DEFAULT_TX_PTHRESH,
1263                         .hthresh = DEFAULT_TX_HTHRESH,
1264                         .wthresh = DEFAULT_TX_WTHRESH,
1265                 },
1266                 .tx_free_thresh = DEFAULT_TX_FREE_THRESH,
1267                 .tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH,
1268         };
1269
1270         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1271                 .nb_max = NFP_NET_MAX_RX_DESC,
1272                 .nb_min = NFP_NET_MIN_RX_DESC,
1273                 .nb_align = NFP_ALIGN_RING_DESC,
1274         };
1275
1276         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1277                 .nb_max = NFP_NET_MAX_TX_DESC,
1278                 .nb_min = NFP_NET_MIN_TX_DESC,
1279                 .nb_align = NFP_ALIGN_RING_DESC,
1280                 .nb_seg_max = NFP_TX_MAX_SEG,
1281                 .nb_mtu_seg_max = NFP_TX_MAX_MTU_SEG,
1282         };
1283
1284         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1285                                            ETH_RSS_NONFRAG_IPV4_TCP |
1286                                            ETH_RSS_NONFRAG_IPV4_UDP |
1287                                            ETH_RSS_IPV6 |
1288                                            ETH_RSS_NONFRAG_IPV6_TCP |
1289                                            ETH_RSS_NONFRAG_IPV6_UDP;
1290
1291         dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ;
1292         dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ;
1293
1294         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G |
1295                                ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G |
1296                                ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G;
1297
1298         return 0;
1299 }
1300
1301 static const uint32_t *
1302 nfp_net_supported_ptypes_get(struct rte_eth_dev *dev)
1303 {
1304         static const uint32_t ptypes[] = {
1305                 /* refers to nfp_net_set_hash() */
1306                 RTE_PTYPE_INNER_L3_IPV4,
1307                 RTE_PTYPE_INNER_L3_IPV6,
1308                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1309                 RTE_PTYPE_INNER_L4_MASK,
1310                 RTE_PTYPE_UNKNOWN
1311         };
1312
1313         if (dev->rx_pkt_burst == nfp_net_recv_pkts)
1314                 return ptypes;
1315         return NULL;
1316 }
1317
1318 static uint32_t
1319 nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx)
1320 {
1321         struct nfp_net_rxq *rxq;
1322         struct nfp_net_rx_desc *rxds;
1323         uint32_t idx;
1324         uint32_t count;
1325
1326         rxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx];
1327
1328         idx = rxq->rd_p;
1329
1330         count = 0;
1331
1332         /*
1333          * Other PMDs are just checking the DD bit in intervals of 4
1334          * descriptors and counting all four if the first has the DD
1335          * bit on. Of course, this is not accurate but can be good for
1336          * performance. But ideally that should be done in descriptors
1337          * chunks belonging to the same cache line
1338          */
1339
1340         while (count < rxq->rx_count) {
1341                 rxds = &rxq->rxds[idx];
1342                 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0)
1343                         break;
1344
1345                 count++;
1346                 idx++;
1347
1348                 /* Wrapping? */
1349                 if ((idx) == rxq->rx_count)
1350                         idx = 0;
1351         }
1352
1353         return count;
1354 }
1355
1356 static int
1357 nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1358 {
1359         struct rte_pci_device *pci_dev;
1360         struct nfp_net_hw *hw;
1361         int base = 0;
1362
1363         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1364         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1365
1366         if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO)
1367                 base = 1;
1368
1369         /* Make sure all updates are written before un-masking */
1370         rte_wmb();
1371         nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id),
1372                       NFP_NET_CFG_ICR_UNMASKED);
1373         return 0;
1374 }
1375
1376 static int
1377 nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1378 {
1379         struct rte_pci_device *pci_dev;
1380         struct nfp_net_hw *hw;
1381         int base = 0;
1382
1383         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1384         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1385
1386         if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO)
1387                 base = 1;
1388
1389         /* Make sure all updates are written before un-masking */
1390         rte_wmb();
1391         nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1);
1392         return 0;
1393 }
1394
1395 static void
1396 nfp_net_dev_link_status_print(struct rte_eth_dev *dev)
1397 {
1398         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1399         struct rte_eth_link link;
1400
1401         rte_eth_linkstatus_get(dev, &link);
1402         if (link.link_status)
1403                 PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
1404                             dev->data->port_id, link.link_speed,
1405                             link.link_duplex == ETH_LINK_FULL_DUPLEX
1406                             ? "full-duplex" : "half-duplex");
1407         else
1408                 PMD_DRV_LOG(INFO, " Port %d: Link Down",
1409                             dev->data->port_id);
1410
1411         PMD_DRV_LOG(INFO, "PCI Address: " PCI_PRI_FMT,
1412                     pci_dev->addr.domain, pci_dev->addr.bus,
1413                     pci_dev->addr.devid, pci_dev->addr.function);
1414 }
1415
1416 /* Interrupt configuration and handling */
1417
1418 /*
1419  * nfp_net_irq_unmask - Unmask an interrupt
1420  *
1421  * If MSI-X auto-masking is enabled clear the mask bit, otherwise
1422  * clear the ICR for the entry.
1423  */
1424 static void
1425 nfp_net_irq_unmask(struct rte_eth_dev *dev)
1426 {
1427         struct nfp_net_hw *hw;
1428         struct rte_pci_device *pci_dev;
1429
1430         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1431         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1432
1433         if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) {
1434                 /* If MSI-X auto-masking is used, clear the entry */
1435                 rte_wmb();
1436                 rte_intr_ack(&pci_dev->intr_handle);
1437         } else {
1438                 /* Make sure all updates are written before un-masking */
1439                 rte_wmb();
1440                 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(NFP_NET_IRQ_LSC_IDX),
1441                               NFP_NET_CFG_ICR_UNMASKED);
1442         }
1443 }
1444
1445 static void
1446 nfp_net_dev_interrupt_handler(void *param)
1447 {
1448         int64_t timeout;
1449         struct rte_eth_link link;
1450         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1451
1452         PMD_DRV_LOG(DEBUG, "We got a LSC interrupt!!!");
1453
1454         rte_eth_linkstatus_get(dev, &link);
1455
1456         nfp_net_link_update(dev, 0);
1457
1458         /* likely to up */
1459         if (!link.link_status) {
1460                 /* handle it 1 sec later, wait it being stable */
1461                 timeout = NFP_NET_LINK_UP_CHECK_TIMEOUT;
1462                 /* likely to down */
1463         } else {
1464                 /* handle it 4 sec later, wait it being stable */
1465                 timeout = NFP_NET_LINK_DOWN_CHECK_TIMEOUT;
1466         }
1467
1468         if (rte_eal_alarm_set(timeout * 1000,
1469                               nfp_net_dev_interrupt_delayed_handler,
1470                               (void *)dev) < 0) {
1471                 PMD_INIT_LOG(ERR, "Error setting alarm");
1472                 /* Unmasking */
1473                 nfp_net_irq_unmask(dev);
1474         }
1475 }
1476
1477 /*
1478  * Interrupt handler which shall be registered for alarm callback for delayed
1479  * handling specific interrupt to wait for the stable nic state. As the NIC
1480  * interrupt state is not stable for nfp after link is just down, it needs
1481  * to wait 4 seconds to get the stable status.
1482  *
1483  * @param handle   Pointer to interrupt handle.
1484  * @param param    The address of parameter (struct rte_eth_dev *)
1485  *
1486  * @return  void
1487  */
1488 static void
1489 nfp_net_dev_interrupt_delayed_handler(void *param)
1490 {
1491         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1492
1493         nfp_net_link_update(dev, 0);
1494         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1495
1496         nfp_net_dev_link_status_print(dev);
1497
1498         /* Unmasking */
1499         nfp_net_irq_unmask(dev);
1500 }
1501
1502 static int
1503 nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1504 {
1505         struct nfp_net_hw *hw;
1506
1507         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1508
1509         /* check that mtu is within the allowed range */
1510         if (mtu < RTE_ETHER_MIN_MTU || (uint32_t)mtu > hw->max_mtu)
1511                 return -EINVAL;
1512
1513         /* mtu setting is forbidden if port is started */
1514         if (dev->data->dev_started) {
1515                 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration",
1516                             dev->data->port_id);
1517                 return -EBUSY;
1518         }
1519
1520         /* switch to jumbo mode if needed */
1521         if ((uint32_t)mtu > RTE_ETHER_MTU)
1522                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
1523         else
1524                 dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1525
1526         /* update max frame size */
1527         dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu;
1528
1529         /* writing to configuration space */
1530         nn_cfg_writel(hw, NFP_NET_CFG_MTU, (uint32_t)mtu);
1531
1532         hw->mtu = mtu;
1533
1534         return 0;
1535 }
1536
1537 static int
1538 nfp_net_rx_queue_setup(struct rte_eth_dev *dev,
1539                        uint16_t queue_idx, uint16_t nb_desc,
1540                        unsigned int socket_id,
1541                        const struct rte_eth_rxconf *rx_conf,
1542                        struct rte_mempool *mp)
1543 {
1544         const struct rte_memzone *tz;
1545         struct nfp_net_rxq *rxq;
1546         struct nfp_net_hw *hw;
1547         uint32_t rx_desc_sz;
1548
1549         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1550
1551         PMD_INIT_FUNC_TRACE();
1552
1553         /* Validating number of descriptors */
1554         rx_desc_sz = nb_desc * sizeof(struct nfp_net_rx_desc);
1555         if (rx_desc_sz % NFP_ALIGN_RING_DESC != 0 ||
1556             nb_desc > NFP_NET_MAX_RX_DESC ||
1557             nb_desc < NFP_NET_MIN_RX_DESC) {
1558                 PMD_DRV_LOG(ERR, "Wrong nb_desc value");
1559                 return -EINVAL;
1560         }
1561
1562         /*
1563          * Free memory prior to re-allocation if needed. This is the case after
1564          * calling nfp_net_stop
1565          */
1566         if (dev->data->rx_queues[queue_idx]) {
1567                 nfp_net_rx_queue_release(dev->data->rx_queues[queue_idx]);
1568                 dev->data->rx_queues[queue_idx] = NULL;
1569         }
1570
1571         /* Allocating rx queue data structure */
1572         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct nfp_net_rxq),
1573                                  RTE_CACHE_LINE_SIZE, socket_id);
1574         if (rxq == NULL)
1575                 return -ENOMEM;
1576
1577         /* Hw queues mapping based on firmware configuration */
1578         rxq->qidx = queue_idx;
1579         rxq->fl_qcidx = queue_idx * hw->stride_rx;
1580         rxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1);
1581         rxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx);
1582         rxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx);
1583
1584         /*
1585          * Tracking mbuf size for detecting a potential mbuf overflow due to
1586          * RX offset
1587          */
1588         rxq->mem_pool = mp;
1589         rxq->mbuf_size = rxq->mem_pool->elt_size;
1590         rxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);
1591         hw->flbufsz = rxq->mbuf_size;
1592
1593         rxq->rx_count = nb_desc;
1594         rxq->port_id = dev->data->port_id;
1595         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
1596         rxq->drop_en = rx_conf->rx_drop_en;
1597
1598         /*
1599          * Allocate RX ring hardware descriptors. A memzone large enough to
1600          * handle the maximum ring size is allocated in order to allow for
1601          * resizing in later calls to the queue setup function.
1602          */
1603         tz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
1604                                    sizeof(struct nfp_net_rx_desc) *
1605                                    NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN,
1606                                    socket_id);
1607
1608         if (tz == NULL) {
1609                 PMD_DRV_LOG(ERR, "Error allocating rx dma");
1610                 nfp_net_rx_queue_release(rxq);
1611                 return -ENOMEM;
1612         }
1613
1614         /* Saving physical and virtual addresses for the RX ring */
1615         rxq->dma = (uint64_t)tz->iova;
1616         rxq->rxds = (struct nfp_net_rx_desc *)tz->addr;
1617
1618         /* mbuf pointers array for referencing mbufs linked to RX descriptors */
1619         rxq->rxbufs = rte_zmalloc_socket("rxq->rxbufs",
1620                                          sizeof(*rxq->rxbufs) * nb_desc,
1621                                          RTE_CACHE_LINE_SIZE, socket_id);
1622         if (rxq->rxbufs == NULL) {
1623                 nfp_net_rx_queue_release(rxq);
1624                 return -ENOMEM;
1625         }
1626
1627         PMD_RX_LOG(DEBUG, "rxbufs=%p hw_ring=%p dma_addr=0x%" PRIx64,
1628                    rxq->rxbufs, rxq->rxds, (unsigned long int)rxq->dma);
1629
1630         nfp_net_reset_rx_queue(rxq);
1631
1632         dev->data->rx_queues[queue_idx] = rxq;
1633         rxq->hw = hw;
1634
1635         /*
1636          * Telling the HW about the physical address of the RX ring and number
1637          * of descriptors in log2 format
1638          */
1639         nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma);
1640         nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc));
1641
1642         return 0;
1643 }
1644
1645 static int
1646 nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq)
1647 {
1648         struct nfp_net_rx_buff *rxe = rxq->rxbufs;
1649         uint64_t dma_addr;
1650         unsigned i;
1651
1652         PMD_RX_LOG(DEBUG, "nfp_net_rx_fill_freelist for %u descriptors",
1653                    rxq->rx_count);
1654
1655         for (i = 0; i < rxq->rx_count; i++) {
1656                 struct nfp_net_rx_desc *rxd;
1657                 struct rte_mbuf *mbuf = rte_pktmbuf_alloc(rxq->mem_pool);
1658
1659                 if (mbuf == NULL) {
1660                         PMD_DRV_LOG(ERR, "RX mbuf alloc failed queue_id=%u",
1661                                 (unsigned)rxq->qidx);
1662                         return -ENOMEM;
1663                 }
1664
1665                 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(mbuf));
1666
1667                 rxd = &rxq->rxds[i];
1668                 rxd->fld.dd = 0;
1669                 rxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff;
1670                 rxd->fld.dma_addr_lo = dma_addr & 0xffffffff;
1671                 rxe[i].mbuf = mbuf;
1672                 PMD_RX_LOG(DEBUG, "[%d]: %" PRIx64, i, dma_addr);
1673         }
1674
1675         /* Make sure all writes are flushed before telling the hardware */
1676         rte_wmb();
1677
1678         /* Not advertising the whole ring as the firmware gets confused if so */
1679         PMD_RX_LOG(DEBUG, "Increment FL write pointer in %u",
1680                    rxq->rx_count - 1);
1681
1682         nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, rxq->rx_count - 1);
1683
1684         return 0;
1685 }
1686
1687 static int
1688 nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1689                        uint16_t nb_desc, unsigned int socket_id,
1690                        const struct rte_eth_txconf *tx_conf)
1691 {
1692         const struct rte_memzone *tz;
1693         struct nfp_net_txq *txq;
1694         uint16_t tx_free_thresh;
1695         struct nfp_net_hw *hw;
1696         uint32_t tx_desc_sz;
1697
1698         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1699
1700         PMD_INIT_FUNC_TRACE();
1701
1702         /* Validating number of descriptors */
1703         tx_desc_sz = nb_desc * sizeof(struct nfp_net_tx_desc);
1704         if (tx_desc_sz % NFP_ALIGN_RING_DESC != 0 ||
1705             nb_desc > NFP_NET_MAX_TX_DESC ||
1706             nb_desc < NFP_NET_MIN_TX_DESC) {
1707                 PMD_DRV_LOG(ERR, "Wrong nb_desc value");
1708                 return -EINVAL;
1709         }
1710
1711         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
1712                                     tx_conf->tx_free_thresh :
1713                                     DEFAULT_TX_FREE_THRESH);
1714
1715         if (tx_free_thresh > (nb_desc)) {
1716                 PMD_DRV_LOG(ERR,
1717                         "tx_free_thresh must be less than the number of TX "
1718                         "descriptors. (tx_free_thresh=%u port=%d "
1719                         "queue=%d)", (unsigned int)tx_free_thresh,
1720                         dev->data->port_id, (int)queue_idx);
1721                 return -(EINVAL);
1722         }
1723
1724         /*
1725          * Free memory prior to re-allocation if needed. This is the case after
1726          * calling nfp_net_stop
1727          */
1728         if (dev->data->tx_queues[queue_idx]) {
1729                 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
1730                            queue_idx);
1731                 nfp_net_tx_queue_release(dev->data->tx_queues[queue_idx]);
1732                 dev->data->tx_queues[queue_idx] = NULL;
1733         }
1734
1735         /* Allocating tx queue data structure */
1736         txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nfp_net_txq),
1737                                  RTE_CACHE_LINE_SIZE, socket_id);
1738         if (txq == NULL) {
1739                 PMD_DRV_LOG(ERR, "Error allocating tx dma");
1740                 return -ENOMEM;
1741         }
1742
1743         /*
1744          * Allocate TX ring hardware descriptors. A memzone large enough to
1745          * handle the maximum ring size is allocated in order to allow for
1746          * resizing in later calls to the queue setup function.
1747          */
1748         tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
1749                                    sizeof(struct nfp_net_tx_desc) *
1750                                    NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN,
1751                                    socket_id);
1752         if (tz == NULL) {
1753                 PMD_DRV_LOG(ERR, "Error allocating tx dma");
1754                 nfp_net_tx_queue_release(txq);
1755                 return -ENOMEM;
1756         }
1757
1758         txq->tx_count = nb_desc;
1759         txq->tx_free_thresh = tx_free_thresh;
1760         txq->tx_pthresh = tx_conf->tx_thresh.pthresh;
1761         txq->tx_hthresh = tx_conf->tx_thresh.hthresh;
1762         txq->tx_wthresh = tx_conf->tx_thresh.wthresh;
1763
1764         /* queue mapping based on firmware configuration */
1765         txq->qidx = queue_idx;
1766         txq->tx_qcidx = queue_idx * hw->stride_tx;
1767         txq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx);
1768
1769         txq->port_id = dev->data->port_id;
1770
1771         /* Saving physical and virtual addresses for the TX ring */
1772         txq->dma = (uint64_t)tz->iova;
1773         txq->txds = (struct nfp_net_tx_desc *)tz->addr;
1774
1775         /* mbuf pointers array for referencing mbufs linked to TX descriptors */
1776         txq->txbufs = rte_zmalloc_socket("txq->txbufs",
1777                                          sizeof(*txq->txbufs) * nb_desc,
1778                                          RTE_CACHE_LINE_SIZE, socket_id);
1779         if (txq->txbufs == NULL) {
1780                 nfp_net_tx_queue_release(txq);
1781                 return -ENOMEM;
1782         }
1783         PMD_TX_LOG(DEBUG, "txbufs=%p hw_ring=%p dma_addr=0x%" PRIx64,
1784                    txq->txbufs, txq->txds, (unsigned long int)txq->dma);
1785
1786         nfp_net_reset_tx_queue(txq);
1787
1788         dev->data->tx_queues[queue_idx] = txq;
1789         txq->hw = hw;
1790
1791         /*
1792          * Telling the HW about the physical address of the TX ring and number
1793          * of descriptors in log2 format
1794          */
1795         nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma);
1796         nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc));
1797
1798         return 0;
1799 }
1800
1801 /* nfp_net_tx_tso - Set TX descriptor for TSO */
1802 static inline void
1803 nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd,
1804                struct rte_mbuf *mb)
1805 {
1806         uint64_t ol_flags;
1807         struct nfp_net_hw *hw = txq->hw;
1808
1809         if (!(hw->cap & NFP_NET_CFG_CTRL_LSO_ANY))
1810                 goto clean_txd;
1811
1812         ol_flags = mb->ol_flags;
1813
1814         if (!(ol_flags & PKT_TX_TCP_SEG))
1815                 goto clean_txd;
1816
1817         txd->l3_offset = mb->l2_len;
1818         txd->l4_offset = mb->l2_len + mb->l3_len;
1819         txd->lso_hdrlen = mb->l2_len + mb->l3_len + mb->l4_len;
1820         txd->mss = rte_cpu_to_le_16(mb->tso_segsz);
1821         txd->flags = PCIE_DESC_TX_LSO;
1822         return;
1823
1824 clean_txd:
1825         txd->flags = 0;
1826         txd->l3_offset = 0;
1827         txd->l4_offset = 0;
1828         txd->lso_hdrlen = 0;
1829         txd->mss = 0;
1830 }
1831
1832 /* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */
1833 static inline void
1834 nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd,
1835                  struct rte_mbuf *mb)
1836 {
1837         uint64_t ol_flags;
1838         struct nfp_net_hw *hw = txq->hw;
1839
1840         if (!(hw->cap & NFP_NET_CFG_CTRL_TXCSUM))
1841                 return;
1842
1843         ol_flags = mb->ol_flags;
1844
1845         /* IPv6 does not need checksum */
1846         if (ol_flags & PKT_TX_IP_CKSUM)
1847                 txd->flags |= PCIE_DESC_TX_IP4_CSUM;
1848
1849         switch (ol_flags & PKT_TX_L4_MASK) {
1850         case PKT_TX_UDP_CKSUM:
1851                 txd->flags |= PCIE_DESC_TX_UDP_CSUM;
1852                 break;
1853         case PKT_TX_TCP_CKSUM:
1854                 txd->flags |= PCIE_DESC_TX_TCP_CSUM;
1855                 break;
1856         }
1857
1858         if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK))
1859                 txd->flags |= PCIE_DESC_TX_CSUM;
1860 }
1861
1862 /* nfp_net_rx_cksum - set mbuf checksum flags based on RX descriptor flags */
1863 static inline void
1864 nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,
1865                  struct rte_mbuf *mb)
1866 {
1867         struct nfp_net_hw *hw = rxq->hw;
1868
1869         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RXCSUM))
1870                 return;
1871
1872         /* If IPv4 and IP checksum error, fail */
1873         if (unlikely((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) &&
1874             !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK)))
1875                 mb->ol_flags |= PKT_RX_IP_CKSUM_BAD;
1876         else
1877                 mb->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
1878
1879         /* If neither UDP nor TCP return */
1880         if (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) &&
1881             !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM))
1882                 return;
1883
1884         if (likely(rxd->rxd.flags & PCIE_DESC_RX_L4_CSUM_OK))
1885                 mb->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
1886         else
1887                 mb->ol_flags |= PKT_RX_L4_CKSUM_BAD;
1888 }
1889
1890 #define NFP_HASH_OFFSET      ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4)
1891 #define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8)
1892
1893 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK)
1894
1895 /*
1896  * nfp_net_set_hash - Set mbuf hash data
1897  *
1898  * The RSS hash and hash-type are pre-pended to the packet data.
1899  * Extract and decode it and set the mbuf fields.
1900  */
1901 static inline void
1902 nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,
1903                  struct rte_mbuf *mbuf)
1904 {
1905         struct nfp_net_hw *hw = rxq->hw;
1906         uint8_t *meta_offset;
1907         uint32_t meta_info;
1908         uint32_t hash = 0;
1909         uint32_t hash_type = 0;
1910
1911         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
1912                 return;
1913
1914         /* this is true for new firmwares */
1915         if (likely(((hw->cap & NFP_NET_CFG_CTRL_RSS2) ||
1916             (NFD_CFG_MAJOR_VERSION_of(hw->ver) == 4)) &&
1917              NFP_DESC_META_LEN(rxd))) {
1918                 /*
1919                  * new metadata api:
1920                  * <----  32 bit  ----->
1921                  * m    field type word
1922                  * e     data field #2
1923                  * t     data field #1
1924                  * a     data field #0
1925                  * ====================
1926                  *    packet data
1927                  *
1928                  * Field type word contains up to 8 4bit field types
1929                  * A 4bit field type refers to a data field word
1930                  * A data field word can have several 4bit field types
1931                  */
1932                 meta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *);
1933                 meta_offset -= NFP_DESC_META_LEN(rxd);
1934                 meta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset);
1935                 meta_offset += 4;
1936                 /* NFP PMD just supports metadata for hashing */
1937                 switch (meta_info & NFP_NET_META_FIELD_MASK) {
1938                 case NFP_NET_META_HASH:
1939                         /* next field type is about the hash type */
1940                         meta_info >>= NFP_NET_META_FIELD_SIZE;
1941                         /* hash value is in the data field */
1942                         hash = rte_be_to_cpu_32(*(uint32_t *)meta_offset);
1943                         hash_type = meta_info & NFP_NET_META_FIELD_MASK;
1944                         break;
1945                 default:
1946                         /* Unsupported metadata can be a performance issue */
1947                         return;
1948                 }
1949         } else {
1950                 if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS))
1951                         return;
1952
1953                 hash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET);
1954                 hash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET);
1955         }
1956
1957         mbuf->hash.rss = hash;
1958         mbuf->ol_flags |= PKT_RX_RSS_HASH;
1959
1960         switch (hash_type) {
1961         case NFP_NET_RSS_IPV4:
1962                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV4;
1963                 break;
1964         case NFP_NET_RSS_IPV6:
1965                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6;
1966                 break;
1967         case NFP_NET_RSS_IPV6_EX:
1968                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
1969                 break;
1970         case NFP_NET_RSS_IPV4_TCP:
1971                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
1972                 break;
1973         case NFP_NET_RSS_IPV6_TCP:
1974                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
1975                 break;
1976         case NFP_NET_RSS_IPV4_UDP:
1977                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
1978                 break;
1979         case NFP_NET_RSS_IPV6_UDP:
1980                 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;
1981                 break;
1982         default:
1983                 mbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK;
1984         }
1985 }
1986
1987 static inline void
1988 nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq)
1989 {
1990         rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1991 }
1992
1993 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK)
1994
1995 /*
1996  * RX path design:
1997  *
1998  * There are some decisions to take:
1999  * 1) How to check DD RX descriptors bit
2000  * 2) How and when to allocate new mbufs
2001  *
2002  * Current implementation checks just one single DD bit each loop. As each
2003  * descriptor is 8 bytes, it is likely a good idea to check descriptors in
2004  * a single cache line instead. Tests with this change have not shown any
2005  * performance improvement but it requires further investigation. For example,
2006  * depending on which descriptor is next, the number of descriptors could be
2007  * less than 8 for just checking those in the same cache line. This implies
2008  * extra work which could be counterproductive by itself. Indeed, last firmware
2009  * changes are just doing this: writing several descriptors with the DD bit
2010  * for saving PCIe bandwidth and DMA operations from the NFP.
2011  *
2012  * Mbuf allocation is done when a new packet is received. Then the descriptor
2013  * is automatically linked with the new mbuf and the old one is given to the
2014  * user. The main drawback with this design is mbuf allocation is heavier than
2015  * using bulk allocations allowed by DPDK with rte_mempool_get_bulk. From the
2016  * cache point of view it does not seem allocating the mbuf early on as we are
2017  * doing now have any benefit at all. Again, tests with this change have not
2018  * shown any improvement. Also, rte_mempool_get_bulk returns all or nothing
2019  * so looking at the implications of this type of allocation should be studied
2020  * deeply
2021  */
2022
2023 static uint16_t
2024 nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2025 {
2026         struct nfp_net_rxq *rxq;
2027         struct nfp_net_rx_desc *rxds;
2028         struct nfp_net_rx_buff *rxb;
2029         struct nfp_net_hw *hw;
2030         struct rte_mbuf *mb;
2031         struct rte_mbuf *new_mb;
2032         uint16_t nb_hold;
2033         uint64_t dma_addr;
2034         int avail;
2035
2036         rxq = rx_queue;
2037         if (unlikely(rxq == NULL)) {
2038                 /*
2039                  * DPDK just checks the queue is lower than max queues
2040                  * enabled. But the queue needs to be configured
2041                  */
2042                 RTE_LOG_DP(ERR, PMD, "RX Bad queue\n");
2043                 return -EINVAL;
2044         }
2045
2046         hw = rxq->hw;
2047         avail = 0;
2048         nb_hold = 0;
2049
2050         while (avail < nb_pkts) {
2051                 rxb = &rxq->rxbufs[rxq->rd_p];
2052                 if (unlikely(rxb == NULL)) {
2053                         RTE_LOG_DP(ERR, PMD, "rxb does not exist!\n");
2054                         break;
2055                 }
2056
2057                 rxds = &rxq->rxds[rxq->rd_p];
2058                 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0)
2059                         break;
2060
2061                 /*
2062                  * Memory barrier to ensure that we won't do other
2063                  * reads before the DD bit.
2064                  */
2065                 rte_rmb();
2066
2067                 /*
2068                  * We got a packet. Let's alloc a new mbuf for refilling the
2069                  * free descriptor ring as soon as possible
2070                  */
2071                 new_mb = rte_pktmbuf_alloc(rxq->mem_pool);
2072                 if (unlikely(new_mb == NULL)) {
2073                         RTE_LOG_DP(DEBUG, PMD,
2074                         "RX mbuf alloc failed port_id=%u queue_id=%u\n",
2075                                 rxq->port_id, (unsigned int)rxq->qidx);
2076                         nfp_net_mbuf_alloc_failed(rxq);
2077                         break;
2078                 }
2079
2080                 nb_hold++;
2081
2082                 /*
2083                  * Grab the mbuf and refill the descriptor with the
2084                  * previously allocated mbuf
2085                  */
2086                 mb = rxb->mbuf;
2087                 rxb->mbuf = new_mb;
2088
2089                 PMD_RX_LOG(DEBUG, "Packet len: %u, mbuf_size: %u",
2090                            rxds->rxd.data_len, rxq->mbuf_size);
2091
2092                 /* Size of this segment */
2093                 mb->data_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds);
2094                 /* Size of the whole packet. We just support 1 segment */
2095                 mb->pkt_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds);
2096
2097                 if (unlikely((mb->data_len + hw->rx_offset) >
2098                              rxq->mbuf_size)) {
2099                         /*
2100                          * This should not happen and the user has the
2101                          * responsibility of avoiding it. But we have
2102                          * to give some info about the error
2103                          */
2104                         RTE_LOG_DP(ERR, PMD,
2105                                 "mbuf overflow likely due to the RX offset.\n"
2106                                 "\t\tYour mbuf size should have extra space for"
2107                                 " RX offset=%u bytes.\n"
2108                                 "\t\tCurrently you just have %u bytes available"
2109                                 " but the received packet is %u bytes long",
2110                                 hw->rx_offset,
2111                                 rxq->mbuf_size - hw->rx_offset,
2112                                 mb->data_len);
2113                         return -EINVAL;
2114                 }
2115
2116                 /* Filling the received mbuf with packet info */
2117                 if (hw->rx_offset)
2118                         mb->data_off = RTE_PKTMBUF_HEADROOM + hw->rx_offset;
2119                 else
2120                         mb->data_off = RTE_PKTMBUF_HEADROOM +
2121                                        NFP_DESC_META_LEN(rxds);
2122
2123                 /* No scatter mode supported */
2124                 mb->nb_segs = 1;
2125                 mb->next = NULL;
2126
2127                 mb->port = rxq->port_id;
2128
2129                 /* Checking the RSS flag */
2130                 nfp_net_set_hash(rxq, rxds, mb);
2131
2132                 /* Checking the checksum flag */
2133                 nfp_net_rx_cksum(rxq, rxds, mb);
2134
2135                 if ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) &&
2136                     (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) {
2137                         mb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan);
2138                         mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
2139                 }
2140
2141                 /* Adding the mbuf to the mbuf array passed by the app */
2142                 rx_pkts[avail++] = mb;
2143
2144                 /* Now resetting and updating the descriptor */
2145                 rxds->vals[0] = 0;
2146                 rxds->vals[1] = 0;
2147                 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb));
2148                 rxds->fld.dd = 0;
2149                 rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff;
2150                 rxds->fld.dma_addr_lo = dma_addr & 0xffffffff;
2151
2152                 rxq->rd_p++;
2153                 if (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/
2154                         rxq->rd_p = 0;
2155         }
2156
2157         if (nb_hold == 0)
2158                 return nb_hold;
2159
2160         PMD_RX_LOG(DEBUG, "RX  port_id=%u queue_id=%u, %d packets received",
2161                    rxq->port_id, (unsigned int)rxq->qidx, nb_hold);
2162
2163         nb_hold += rxq->nb_rx_hold;
2164
2165         /*
2166          * FL descriptors needs to be written before incrementing the
2167          * FL queue WR pointer
2168          */
2169         rte_wmb();
2170         if (nb_hold > rxq->rx_free_thresh) {
2171                 PMD_RX_LOG(DEBUG, "port=%u queue=%u nb_hold=%u avail=%u",
2172                            rxq->port_id, (unsigned int)rxq->qidx,
2173                            (unsigned)nb_hold, (unsigned)avail);
2174                 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold);
2175                 nb_hold = 0;
2176         }
2177         rxq->nb_rx_hold = nb_hold;
2178
2179         return avail;
2180 }
2181
2182 /*
2183  * nfp_net_tx_free_bufs - Check for descriptors with a complete
2184  * status
2185  * @txq: TX queue to work with
2186  * Returns number of descriptors freed
2187  */
2188 int
2189 nfp_net_tx_free_bufs(struct nfp_net_txq *txq)
2190 {
2191         uint32_t qcp_rd_p;
2192         int todo;
2193
2194         PMD_TX_LOG(DEBUG, "queue %u. Check for descriptor with a complete"
2195                    " status", txq->qidx);
2196
2197         /* Work out how many packets have been sent */
2198         qcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR);
2199
2200         if (qcp_rd_p == txq->rd_p) {
2201                 PMD_TX_LOG(DEBUG, "queue %u: It seems harrier is not sending "
2202                            "packets (%u, %u)", txq->qidx,
2203                            qcp_rd_p, txq->rd_p);
2204                 return 0;
2205         }
2206
2207         if (qcp_rd_p > txq->rd_p)
2208                 todo = qcp_rd_p - txq->rd_p;
2209         else
2210                 todo = qcp_rd_p + txq->tx_count - txq->rd_p;
2211
2212         PMD_TX_LOG(DEBUG, "qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u",
2213                    qcp_rd_p, txq->rd_p, txq->rd_p);
2214
2215         if (todo == 0)
2216                 return todo;
2217
2218         txq->rd_p += todo;
2219         if (unlikely(txq->rd_p >= txq->tx_count))
2220                 txq->rd_p -= txq->tx_count;
2221
2222         return todo;
2223 }
2224
2225 /* Leaving always free descriptors for avoiding wrapping confusion */
2226 static inline
2227 uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq)
2228 {
2229         if (txq->wr_p >= txq->rd_p)
2230                 return txq->tx_count - (txq->wr_p - txq->rd_p) - 8;
2231         else
2232                 return txq->rd_p - txq->wr_p - 8;
2233 }
2234
2235 /*
2236  * nfp_net_txq_full - Check if the TX queue free descriptors
2237  * is below tx_free_threshold
2238  *
2239  * @txq: TX queue to check
2240  *
2241  * This function uses the host copy* of read/write pointers
2242  */
2243 static inline
2244 uint32_t nfp_net_txq_full(struct nfp_net_txq *txq)
2245 {
2246         return (nfp_free_tx_desc(txq) < txq->tx_free_thresh);
2247 }
2248
2249 static uint16_t
2250 nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2251 {
2252         struct nfp_net_txq *txq;
2253         struct nfp_net_hw *hw;
2254         struct nfp_net_tx_desc *txds, txd;
2255         struct rte_mbuf *pkt;
2256         uint64_t dma_addr;
2257         int pkt_size, dma_size;
2258         uint16_t free_descs, issued_descs;
2259         struct rte_mbuf **lmbuf;
2260         int i;
2261
2262         txq = tx_queue;
2263         hw = txq->hw;
2264         txds = &txq->txds[txq->wr_p];
2265
2266         PMD_TX_LOG(DEBUG, "working for queue %u at pos %d and %u packets",
2267                    txq->qidx, txq->wr_p, nb_pkts);
2268
2269         if ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq)))
2270                 nfp_net_tx_free_bufs(txq);
2271
2272         free_descs = (uint16_t)nfp_free_tx_desc(txq);
2273         if (unlikely(free_descs == 0))
2274                 return 0;
2275
2276         pkt = *tx_pkts;
2277
2278         i = 0;
2279         issued_descs = 0;
2280         PMD_TX_LOG(DEBUG, "queue: %u. Sending %u packets",
2281                    txq->qidx, nb_pkts);
2282         /* Sending packets */
2283         while ((i < nb_pkts) && free_descs) {
2284                 /* Grabbing the mbuf linked to the current descriptor */
2285                 lmbuf = &txq->txbufs[txq->wr_p].mbuf;
2286                 /* Warming the cache for releasing the mbuf later on */
2287                 RTE_MBUF_PREFETCH_TO_FREE(*lmbuf);
2288
2289                 pkt = *(tx_pkts + i);
2290
2291                 if (unlikely((pkt->nb_segs > 1) &&
2292                              !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) {
2293                         PMD_INIT_LOG(INFO, "NFP_NET_CFG_CTRL_GATHER not set");
2294                         rte_panic("Multisegment packet unsupported\n");
2295                 }
2296
2297                 /* Checking if we have enough descriptors */
2298                 if (unlikely(pkt->nb_segs > free_descs))
2299                         goto xmit_end;
2300
2301                 /*
2302                  * Checksum and VLAN flags just in the first descriptor for a
2303                  * multisegment packet, but TSO info needs to be in all of them.
2304                  */
2305                 txd.data_len = pkt->pkt_len;
2306                 nfp_net_tx_tso(txq, &txd, pkt);
2307                 nfp_net_tx_cksum(txq, &txd, pkt);
2308
2309                 if ((pkt->ol_flags & PKT_TX_VLAN_PKT) &&
2310                     (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) {
2311                         txd.flags |= PCIE_DESC_TX_VLAN;
2312                         txd.vlan = pkt->vlan_tci;
2313                 }
2314
2315                 /*
2316                  * mbuf data_len is the data in one segment and pkt_len data
2317                  * in the whole packet. When the packet is just one segment,
2318                  * then data_len = pkt_len
2319                  */
2320                 pkt_size = pkt->pkt_len;
2321
2322                 while (pkt) {
2323                         /* Copying TSO, VLAN and cksum info */
2324                         *txds = txd;
2325
2326                         /* Releasing mbuf used by this descriptor previously*/
2327                         if (*lmbuf)
2328                                 rte_pktmbuf_free_seg(*lmbuf);
2329
2330                         /*
2331                          * Linking mbuf with descriptor for being released
2332                          * next time descriptor is used
2333                          */
2334                         *lmbuf = pkt;
2335
2336                         dma_size = pkt->data_len;
2337                         dma_addr = rte_mbuf_data_iova(pkt);
2338                         PMD_TX_LOG(DEBUG, "Working with mbuf at dma address:"
2339                                    "%" PRIx64 "", dma_addr);
2340
2341                         /* Filling descriptors fields */
2342                         txds->dma_len = dma_size;
2343                         txds->data_len = txd.data_len;
2344                         txds->dma_addr_hi = (dma_addr >> 32) & 0xff;
2345                         txds->dma_addr_lo = (dma_addr & 0xffffffff);
2346                         ASSERT(free_descs > 0);
2347                         free_descs--;
2348
2349                         txq->wr_p++;
2350                         if (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/
2351                                 txq->wr_p = 0;
2352
2353                         pkt_size -= dma_size;
2354
2355                         /*
2356                          * Making the EOP, packets with just one segment
2357                          * the priority
2358                          */
2359                         if (likely(!pkt_size))
2360                                 txds->offset_eop = PCIE_DESC_TX_EOP;
2361                         else
2362                                 txds->offset_eop = 0;
2363
2364                         pkt = pkt->next;
2365                         /* Referencing next free TX descriptor */
2366                         txds = &txq->txds[txq->wr_p];
2367                         lmbuf = &txq->txbufs[txq->wr_p].mbuf;
2368                         issued_descs++;
2369                 }
2370                 i++;
2371         }
2372
2373 xmit_end:
2374         /* Increment write pointers. Force memory write before we let HW know */
2375         rte_wmb();
2376         nfp_qcp_ptr_add(txq->qcp_q, NFP_QCP_WRITE_PTR, issued_descs);
2377
2378         return i;
2379 }
2380
2381 static int
2382 nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2383 {
2384         uint32_t new_ctrl, update;
2385         struct nfp_net_hw *hw;
2386         int ret;
2387
2388         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2389         new_ctrl = 0;
2390
2391         /* Enable vlan strip if it is not configured yet */
2392         if ((mask & ETH_VLAN_STRIP_OFFLOAD) &&
2393             !(hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN))
2394                 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_RXVLAN;
2395
2396         /* Disable vlan strip just if it is configured */
2397         if (!(mask & ETH_VLAN_STRIP_OFFLOAD) &&
2398             (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN))
2399                 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN;
2400
2401         if (new_ctrl == 0)
2402                 return 0;
2403
2404         update = NFP_NET_CFG_UPDATE_GEN;
2405
2406         ret = nfp_net_reconfig(hw, new_ctrl, update);
2407         if (!ret)
2408                 hw->ctrl = new_ctrl;
2409
2410         return ret;
2411 }
2412
2413 static int
2414 nfp_net_rss_reta_write(struct rte_eth_dev *dev,
2415                     struct rte_eth_rss_reta_entry64 *reta_conf,
2416                     uint16_t reta_size)
2417 {
2418         uint32_t reta, mask;
2419         int i, j;
2420         int idx, shift;
2421         struct nfp_net_hw *hw =
2422                 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2423
2424         if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) {
2425                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2426                         "(%d) doesn't match the number hardware can supported "
2427                         "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ);
2428                 return -EINVAL;
2429         }
2430
2431         /*
2432          * Update Redirection Table. There are 128 8bit-entries which can be
2433          * manage as 32 32bit-entries
2434          */
2435         for (i = 0; i < reta_size; i += 4) {
2436                 /* Handling 4 RSS entries per loop */
2437                 idx = i / RTE_RETA_GROUP_SIZE;
2438                 shift = i % RTE_RETA_GROUP_SIZE;
2439                 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF);
2440
2441                 if (!mask)
2442                         continue;
2443
2444                 reta = 0;
2445                 /* If all 4 entries were set, don't need read RETA register */
2446                 if (mask != 0xF)
2447                         reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + i);
2448
2449                 for (j = 0; j < 4; j++) {
2450                         if (!(mask & (0x1 << j)))
2451                                 continue;
2452                         if (mask != 0xF)
2453                                 /* Clearing the entry bits */
2454                                 reta &= ~(0xFF << (8 * j));
2455                         reta |= reta_conf[idx].reta[shift + j] << (8 * j);
2456                 }
2457                 nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift,
2458                               reta);
2459         }
2460         return 0;
2461 }
2462
2463 /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */
2464 static int
2465 nfp_net_reta_update(struct rte_eth_dev *dev,
2466                     struct rte_eth_rss_reta_entry64 *reta_conf,
2467                     uint16_t reta_size)
2468 {
2469         struct nfp_net_hw *hw =
2470                 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2471         uint32_t update;
2472         int ret;
2473
2474         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
2475                 return -EINVAL;
2476
2477         ret = nfp_net_rss_reta_write(dev, reta_conf, reta_size);
2478         if (ret != 0)
2479                 return ret;
2480
2481         update = NFP_NET_CFG_UPDATE_RSS;
2482
2483         if (nfp_net_reconfig(hw, hw->ctrl, update) < 0)
2484                 return -EIO;
2485
2486         return 0;
2487 }
2488
2489  /* Query Redirection Table(RETA) of Receive Side Scaling of Ethernet device. */
2490 static int
2491 nfp_net_reta_query(struct rte_eth_dev *dev,
2492                    struct rte_eth_rss_reta_entry64 *reta_conf,
2493                    uint16_t reta_size)
2494 {
2495         uint8_t i, j, mask;
2496         int idx, shift;
2497         uint32_t reta;
2498         struct nfp_net_hw *hw;
2499
2500         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2501
2502         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
2503                 return -EINVAL;
2504
2505         if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) {
2506                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2507                         "(%d) doesn't match the number hardware can supported "
2508                         "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ);
2509                 return -EINVAL;
2510         }
2511
2512         /*
2513          * Reading Redirection Table. There are 128 8bit-entries which can be
2514          * manage as 32 32bit-entries
2515          */
2516         for (i = 0; i < reta_size; i += 4) {
2517                 /* Handling 4 RSS entries per loop */
2518                 idx = i / RTE_RETA_GROUP_SIZE;
2519                 shift = i % RTE_RETA_GROUP_SIZE;
2520                 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF);
2521
2522                 if (!mask)
2523                         continue;
2524
2525                 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) +
2526                                     shift);
2527                 for (j = 0; j < 4; j++) {
2528                         if (!(mask & (0x1 << j)))
2529                                 continue;
2530                         reta_conf[idx].reta[shift + j] =
2531                                 (uint8_t)((reta >> (8 * j)) & 0xF);
2532                 }
2533         }
2534         return 0;
2535 }
2536
2537 static int
2538 nfp_net_rss_hash_write(struct rte_eth_dev *dev,
2539                         struct rte_eth_rss_conf *rss_conf)
2540 {
2541         struct nfp_net_hw *hw;
2542         uint64_t rss_hf;
2543         uint32_t cfg_rss_ctrl = 0;
2544         uint8_t key;
2545         int i;
2546
2547         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2548
2549         /* Writing the key byte a byte */
2550         for (i = 0; i < rss_conf->rss_key_len; i++) {
2551                 memcpy(&key, &rss_conf->rss_key[i], 1);
2552                 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY + i, key);
2553         }
2554
2555         rss_hf = rss_conf->rss_hf;
2556
2557         if (rss_hf & ETH_RSS_IPV4)
2558                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4;
2559
2560         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
2561                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_TCP;
2562
2563         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
2564                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_UDP;
2565
2566         if (rss_hf & ETH_RSS_IPV6)
2567                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6;
2568
2569         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
2570                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_TCP;
2571
2572         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
2573                 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_UDP;
2574
2575         cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK;
2576         cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ;
2577
2578         /* configuring where to apply the RSS hash */
2579         nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl);
2580
2581         /* Writing the key size */
2582         nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY_SZ, rss_conf->rss_key_len);
2583
2584         return 0;
2585 }
2586
2587 static int
2588 nfp_net_rss_hash_update(struct rte_eth_dev *dev,
2589                         struct rte_eth_rss_conf *rss_conf)
2590 {
2591         uint32_t update;
2592         uint64_t rss_hf;
2593         struct nfp_net_hw *hw;
2594
2595         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2596
2597         rss_hf = rss_conf->rss_hf;
2598
2599         /* Checking if RSS is enabled */
2600         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) {
2601                 if (rss_hf != 0) { /* Enable RSS? */
2602                         PMD_DRV_LOG(ERR, "RSS unsupported");
2603                         return -EINVAL;
2604                 }
2605                 return 0; /* Nothing to do */
2606         }
2607
2608         if (rss_conf->rss_key_len > NFP_NET_CFG_RSS_KEY_SZ) {
2609                 PMD_DRV_LOG(ERR, "hash key too long");
2610                 return -EINVAL;
2611         }
2612
2613         nfp_net_rss_hash_write(dev, rss_conf);
2614
2615         update = NFP_NET_CFG_UPDATE_RSS;
2616
2617         if (nfp_net_reconfig(hw, hw->ctrl, update) < 0)
2618                 return -EIO;
2619
2620         return 0;
2621 }
2622
2623 static int
2624 nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev,
2625                           struct rte_eth_rss_conf *rss_conf)
2626 {
2627         uint64_t rss_hf;
2628         uint32_t cfg_rss_ctrl;
2629         uint8_t key;
2630         int i;
2631         struct nfp_net_hw *hw;
2632
2633         hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2634
2635         if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))
2636                 return -EINVAL;
2637
2638         rss_hf = rss_conf->rss_hf;
2639         cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL);
2640
2641         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4)
2642                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP;
2643
2644         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP)
2645                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2646
2647         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_TCP)
2648                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
2649
2650         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_UDP)
2651                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2652
2653         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_UDP)
2654                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
2655
2656         if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6)
2657                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP;
2658
2659         /* Propagate current RSS hash functions to caller */
2660         rss_conf->rss_hf = rss_hf;
2661
2662         /* Reading the key size */
2663         rss_conf->rss_key_len = nn_cfg_readl(hw, NFP_NET_CFG_RSS_KEY_SZ);
2664
2665         /* Reading the key byte a byte */
2666         for (i = 0; i < rss_conf->rss_key_len; i++) {
2667                 key = nn_cfg_readb(hw, NFP_NET_CFG_RSS_KEY + i);
2668                 memcpy(&rss_conf->rss_key[i], &key, 1);
2669         }
2670
2671         return 0;
2672 }
2673
2674 static int
2675 nfp_net_rss_config_default(struct rte_eth_dev *dev)
2676 {
2677         struct rte_eth_conf *dev_conf;
2678         struct rte_eth_rss_conf rss_conf;
2679         struct rte_eth_rss_reta_entry64 nfp_reta_conf[2];
2680         uint16_t rx_queues = dev->data->nb_rx_queues;
2681         uint16_t queue;
2682         int i, j, ret;
2683
2684         PMD_DRV_LOG(INFO, "setting default RSS conf for %u queues",
2685                 rx_queues);
2686
2687         nfp_reta_conf[0].mask = ~0x0;
2688         nfp_reta_conf[1].mask = ~0x0;
2689
2690         queue = 0;
2691         for (i = 0; i < 0x40; i += 8) {
2692                 for (j = i; j < (i + 8); j++) {
2693                         nfp_reta_conf[0].reta[j] = queue;
2694                         nfp_reta_conf[1].reta[j] = queue++;
2695                         queue %= rx_queues;
2696                 }
2697         }
2698         ret = nfp_net_rss_reta_write(dev, nfp_reta_conf, 0x80);
2699         if (ret != 0)
2700                 return ret;
2701
2702         dev_conf = &dev->data->dev_conf;
2703         if (!dev_conf) {
2704                 PMD_DRV_LOG(INFO, "wrong rss conf");
2705                 return -EINVAL;
2706         }
2707         rss_conf = dev_conf->rx_adv_conf.rss_conf;
2708
2709         ret = nfp_net_rss_hash_write(dev, &rss_conf);
2710
2711         return ret;
2712 }
2713
2714
2715 /* Initialise and register driver with DPDK Application */
2716 static const struct eth_dev_ops nfp_net_eth_dev_ops = {
2717         .dev_configure          = nfp_net_configure,
2718         .dev_start              = nfp_net_start,
2719         .dev_stop               = nfp_net_stop,
2720         .dev_set_link_up        = nfp_net_set_link_up,
2721         .dev_set_link_down      = nfp_net_set_link_down,
2722         .dev_close              = nfp_net_close,
2723         .promiscuous_enable     = nfp_net_promisc_enable,
2724         .promiscuous_disable    = nfp_net_promisc_disable,
2725         .link_update            = nfp_net_link_update,
2726         .stats_get              = nfp_net_stats_get,
2727         .stats_reset            = nfp_net_stats_reset,
2728         .dev_infos_get          = nfp_net_infos_get,
2729         .dev_supported_ptypes_get = nfp_net_supported_ptypes_get,
2730         .mtu_set                = nfp_net_dev_mtu_set,
2731         .mac_addr_set           = nfp_set_mac_addr,
2732         .vlan_offload_set       = nfp_net_vlan_offload_set,
2733         .reta_update            = nfp_net_reta_update,
2734         .reta_query             = nfp_net_reta_query,
2735         .rss_hash_update        = nfp_net_rss_hash_update,
2736         .rss_hash_conf_get      = nfp_net_rss_hash_conf_get,
2737         .rx_queue_setup         = nfp_net_rx_queue_setup,
2738         .rx_queue_release       = nfp_net_rx_queue_release,
2739         .tx_queue_setup         = nfp_net_tx_queue_setup,
2740         .tx_queue_release       = nfp_net_tx_queue_release,
2741         .rx_queue_intr_enable   = nfp_rx_queue_intr_enable,
2742         .rx_queue_intr_disable  = nfp_rx_queue_intr_disable,
2743 };
2744
2745
2746 static int
2747 nfp_net_init(struct rte_eth_dev *eth_dev)
2748 {
2749         struct rte_pci_device *pci_dev;
2750         struct nfp_pf_dev *pf_dev;
2751         struct nfp_net_hw *hw;
2752
2753         uint64_t tx_bar_off = 0, rx_bar_off = 0;
2754         uint32_t start_q;
2755         int stride = 4;
2756         int port = 0;
2757         int err;
2758
2759         PMD_INIT_FUNC_TRACE();
2760
2761         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2762
2763         /* Use backpointer here to the PF of this eth_dev */
2764         pf_dev = NFP_NET_DEV_PRIVATE_TO_PF(eth_dev->data->dev_private);
2765
2766         /* NFP can not handle DMA addresses requiring more than 40 bits */
2767         if (rte_mem_check_dma_mask(40)) {
2768                 RTE_LOG(ERR, PMD, "device %s can not be used:",
2769                                    pci_dev->device.name);
2770                 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n");
2771                 return -ENODEV;
2772         };
2773
2774         if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) ||
2775             (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) {
2776                 port = ((struct nfp_net_hw *)eth_dev->data->dev_private)->idx;
2777                 if (port < 0 || port > 7) {
2778                         PMD_DRV_LOG(ERR, "Port value is wrong");
2779                         return -ENODEV;
2780                 }
2781
2782                 /* This points to the specific port private data */
2783                 PMD_INIT_LOG(DEBUG, "Working with physical port number %d",
2784                                     port);
2785
2786                 /* Use PF array of physical ports to get pointer to
2787                  * this specific port
2788                  */
2789                 hw = pf_dev->ports[port];
2790
2791         } else {
2792                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
2793         }
2794
2795         eth_dev->dev_ops = &nfp_net_eth_dev_ops;
2796         eth_dev->rx_queue_count = nfp_net_rx_queue_count;
2797         eth_dev->rx_pkt_burst = &nfp_net_recv_pkts;
2798         eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts;
2799
2800         /* For secondary processes, the primary has done all the work */
2801         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2802                 return 0;
2803
2804         rte_eth_copy_pci_info(eth_dev, pci_dev);
2805
2806         hw->device_id = pci_dev->id.device_id;
2807         hw->vendor_id = pci_dev->id.vendor_id;
2808         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
2809         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
2810
2811         PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u",
2812                      pci_dev->id.vendor_id, pci_dev->id.device_id,
2813                      pci_dev->addr.domain, pci_dev->addr.bus,
2814                      pci_dev->addr.devid, pci_dev->addr.function);
2815
2816         hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr;
2817         if (hw->ctrl_bar == NULL) {
2818                 PMD_DRV_LOG(ERR,
2819                         "hw->ctrl_bar is NULL. BAR0 not configured");
2820                 return -ENODEV;
2821         }
2822
2823         if (hw->is_phyport) {
2824                 if (port == 0) {
2825                         hw->ctrl_bar = pf_dev->ctrl_bar;
2826                 } else {
2827                         if (!pf_dev->ctrl_bar)
2828                                 return -ENODEV;
2829                         /* Use port offset in pf ctrl_bar for this
2830                          * ports control bar
2831                          */
2832                         hw->ctrl_bar = pf_dev->ctrl_bar +
2833                                        (port * NFP_PF_CSR_SLICE_SIZE);
2834                 }
2835         }
2836
2837         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar);
2838
2839         hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS);
2840         hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS);
2841
2842         /* Work out where in the BAR the queues start. */
2843         switch (pci_dev->id.device_id) {
2844         case PCI_DEVICE_ID_NFP4000_PF_NIC:
2845         case PCI_DEVICE_ID_NFP6000_PF_NIC:
2846         case PCI_DEVICE_ID_NFP6000_VF_NIC:
2847                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
2848                 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
2849                 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
2850                 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
2851                 break;
2852         default:
2853                 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
2854                 err = -ENODEV;
2855                 goto dev_err_ctrl_map;
2856         }
2857
2858         PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off);
2859         PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off);
2860
2861         if (hw->is_phyport) {
2862                 hw->tx_bar = pf_dev->hw_queues + tx_bar_off;
2863                 hw->rx_bar = pf_dev->hw_queues + rx_bar_off;
2864                 eth_dev->data->dev_private = hw;
2865         } else {
2866                 hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr +
2867                              tx_bar_off;
2868                 hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr +
2869                              rx_bar_off;
2870         }
2871
2872         PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p",
2873                      hw->ctrl_bar, hw->tx_bar, hw->rx_bar);
2874
2875         nfp_net_cfg_queue_setup(hw);
2876
2877         /* Get some of the read-only fields from the config BAR */
2878         hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION);
2879         hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP);
2880         hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU);
2881         hw->mtu = RTE_ETHER_MTU;
2882
2883         /* VLAN insertion is incompatible with LSOv2 */
2884         if (hw->cap & NFP_NET_CFG_CTRL_LSO2)
2885                 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN;
2886
2887         if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2)
2888                 hw->rx_offset = NFP_NET_RX_OFFSET;
2889         else
2890                 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR);
2891
2892         PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d",
2893                            NFD_CFG_MAJOR_VERSION_of(hw->ver),
2894                            NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu);
2895
2896         PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap,
2897                      hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "",
2898                      hw->cap & NFP_NET_CFG_CTRL_L2BC    ? "L2BCFILT " : "",
2899                      hw->cap & NFP_NET_CFG_CTRL_L2MC    ? "L2MCFILT " : "",
2900                      hw->cap & NFP_NET_CFG_CTRL_RXCSUM  ? "RXCSUM "  : "",
2901                      hw->cap & NFP_NET_CFG_CTRL_TXCSUM  ? "TXCSUM "  : "",
2902                      hw->cap & NFP_NET_CFG_CTRL_RXVLAN  ? "RXVLAN "  : "",
2903                      hw->cap & NFP_NET_CFG_CTRL_TXVLAN  ? "TXVLAN "  : "",
2904                      hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "",
2905                      hw->cap & NFP_NET_CFG_CTRL_GATHER  ? "GATHER "  : "",
2906                      hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR "  : "",
2907                      hw->cap & NFP_NET_CFG_CTRL_LSO     ? "TSO "     : "",
2908                      hw->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSOv2 "     : "",
2909                      hw->cap & NFP_NET_CFG_CTRL_RSS     ? "RSS "     : "",
2910                      hw->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSSv2 "     : "");
2911
2912         hw->ctrl = 0;
2913
2914         hw->stride_rx = stride;
2915         hw->stride_tx = stride;
2916
2917         PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u",
2918                      hw->max_rx_queues, hw->max_tx_queues);
2919
2920         /* Initializing spinlock for reconfigs */
2921         rte_spinlock_init(&hw->reconfig_lock);
2922
2923         /* Allocating memory for mac addr */
2924         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
2925                                                RTE_ETHER_ADDR_LEN, 0);
2926         if (eth_dev->data->mac_addrs == NULL) {
2927                 PMD_INIT_LOG(ERR, "Failed to space for MAC address");
2928                 err = -ENOMEM;
2929                 goto dev_err_queues_map;
2930         }
2931
2932         if (hw->is_phyport) {
2933                 nfp_net_pf_read_mac(pf_dev, port);
2934                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
2935         } else {
2936                 nfp_net_vf_read_mac(hw);
2937         }
2938
2939         if (!rte_is_valid_assigned_ether_addr(
2940                     (struct rte_ether_addr *)&hw->mac_addr)) {
2941                 PMD_INIT_LOG(INFO, "Using random mac address for port %d",
2942                                    port);
2943                 /* Using random mac addresses for VFs */
2944                 rte_eth_random_addr(&hw->mac_addr[0]);
2945                 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr);
2946         }
2947
2948         /* Copying mac address to DPDK eth_dev struct */
2949         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr,
2950                         &eth_dev->data->mac_addrs[0]);
2951
2952         if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR))
2953                 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR;
2954
2955         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2956
2957         PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x "
2958                      "mac=%02x:%02x:%02x:%02x:%02x:%02x",
2959                      eth_dev->data->port_id, pci_dev->id.vendor_id,
2960                      pci_dev->id.device_id,
2961                      hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
2962                      hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
2963
2964         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2965                 /* Registering LSC interrupt handler */
2966                 rte_intr_callback_register(&pci_dev->intr_handle,
2967                                            nfp_net_dev_interrupt_handler,
2968                                            (void *)eth_dev);
2969                 /* Telling the firmware about the LSC interrupt entry */
2970                 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
2971                 /* Recording current stats counters values */
2972                 nfp_net_stats_reset(eth_dev);
2973         }
2974
2975         return 0;
2976
2977 dev_err_queues_map:
2978                 nfp_cpp_area_free(hw->hwqueues_area);
2979 dev_err_ctrl_map:
2980                 nfp_cpp_area_free(hw->ctrl_area);
2981
2982         return err;
2983 }
2984
2985 #define NFP_CPP_MEMIO_BOUNDARY          (1 << 20)
2986
2987 /*
2988  * Serving a write request to NFP from host programs. The request
2989  * sends the write size and the CPP target. The bridge makes use
2990  * of CPP interface handler configured by the PMD setup.
2991  */
2992 static int
2993 nfp_cpp_bridge_serve_write(int sockfd, struct nfp_cpp *cpp)
2994 {
2995         struct nfp_cpp_area *area;
2996         off_t offset, nfp_offset;
2997         uint32_t cpp_id, pos, len;
2998         uint32_t tmpbuf[16];
2999         size_t count, curlen, totlen = 0;
3000         int err = 0;
3001
3002         PMD_CPP_LOG(DEBUG, "%s: offset size %zu, count_size: %zu\n", __func__,
3003                 sizeof(off_t), sizeof(size_t));
3004
3005         /* Reading the count param */
3006         err = recv(sockfd, &count, sizeof(off_t), 0);
3007         if (err != sizeof(off_t))
3008                 return -EINVAL;
3009
3010         curlen = count;
3011
3012         /* Reading the offset param */
3013         err = recv(sockfd, &offset, sizeof(off_t), 0);
3014         if (err != sizeof(off_t))
3015                 return -EINVAL;
3016
3017         /* Obtain target's CPP ID and offset in target */
3018         cpp_id = (offset >> 40) << 8;
3019         nfp_offset = offset & ((1ull << 40) - 1);
3020
3021         PMD_CPP_LOG(DEBUG, "%s: count %zu and offset %jd\n", __func__, count,
3022                 offset);
3023         PMD_CPP_LOG(DEBUG, "%s: cpp_id %08x and nfp_offset %jd\n", __func__,
3024                 cpp_id, nfp_offset);
3025
3026         /* Adjust length if not aligned */
3027         if (((nfp_offset + (off_t)count - 1) & ~(NFP_CPP_MEMIO_BOUNDARY - 1)) !=
3028             (nfp_offset & ~(NFP_CPP_MEMIO_BOUNDARY - 1))) {
3029                 curlen = NFP_CPP_MEMIO_BOUNDARY -
3030                         (nfp_offset & (NFP_CPP_MEMIO_BOUNDARY - 1));
3031         }
3032
3033         while (count > 0) {
3034                 /* configure a CPP PCIe2CPP BAR for mapping the CPP target */
3035                 area = nfp_cpp_area_alloc_with_name(cpp, cpp_id, "nfp.cdev",
3036                                                     nfp_offset, curlen);
3037                 if (!area) {
3038                         RTE_LOG(ERR, PMD, "%s: area alloc fail\n", __func__);
3039                         return -EIO;
3040                 }
3041
3042                 /* mapping the target */
3043                 err = nfp_cpp_area_acquire(area);
3044                 if (err < 0) {
3045                         RTE_LOG(ERR, PMD, "area acquire failed\n");
3046                         nfp_cpp_area_free(area);
3047                         return -EIO;
3048                 }
3049
3050                 for (pos = 0; pos < curlen; pos += len) {
3051                         len = curlen - pos;
3052                         if (len > sizeof(tmpbuf))
3053                                 len = sizeof(tmpbuf);
3054
3055                         PMD_CPP_LOG(DEBUG, "%s: Receive %u of %zu\n", __func__,
3056                                            len, count);
3057                         err = recv(sockfd, tmpbuf, len, MSG_WAITALL);
3058                         if (err != (int)len) {
3059                                 RTE_LOG(ERR, PMD,
3060                                         "%s: error when receiving, %d of %zu\n",
3061                                         __func__, err, count);
3062                                 nfp_cpp_area_release(area);
3063                                 nfp_cpp_area_free(area);
3064                                 return -EIO;
3065                         }
3066                         err = nfp_cpp_area_write(area, pos, tmpbuf, len);
3067                         if (err < 0) {
3068                                 RTE_LOG(ERR, PMD, "nfp_cpp_area_write error\n");
3069                                 nfp_cpp_area_release(area);
3070                                 nfp_cpp_area_free(area);
3071                                 return -EIO;
3072                         }
3073                 }
3074
3075                 nfp_offset += pos;
3076                 totlen += pos;
3077                 nfp_cpp_area_release(area);
3078                 nfp_cpp_area_free(area);
3079
3080                 count -= pos;
3081                 curlen = (count > NFP_CPP_MEMIO_BOUNDARY) ?
3082                          NFP_CPP_MEMIO_BOUNDARY : count;
3083         }
3084
3085         return 0;
3086 }
3087
3088 /*
3089  * Serving a read request to NFP from host programs. The request
3090  * sends the read size and the CPP target. The bridge makes use
3091  * of CPP interface handler configured by the PMD setup. The read
3092  * data is sent to the requester using the same socket.
3093  */
3094 static int
3095 nfp_cpp_bridge_serve_read(int sockfd, struct nfp_cpp *cpp)
3096 {
3097         struct nfp_cpp_area *area;
3098         off_t offset, nfp_offset;
3099         uint32_t cpp_id, pos, len;
3100         uint32_t tmpbuf[16];
3101         size_t count, curlen, totlen = 0;
3102         int err = 0;
3103
3104         PMD_CPP_LOG(DEBUG, "%s: offset size %zu, count_size: %zu\n", __func__,
3105                 sizeof(off_t), sizeof(size_t));
3106
3107         /* Reading the count param */
3108         err = recv(sockfd, &count, sizeof(off_t), 0);
3109         if (err != sizeof(off_t))
3110                 return -EINVAL;
3111
3112         curlen = count;
3113
3114         /* Reading the offset param */
3115         err = recv(sockfd, &offset, sizeof(off_t), 0);
3116         if (err != sizeof(off_t))
3117                 return -EINVAL;
3118
3119         /* Obtain target's CPP ID and offset in target */
3120         cpp_id = (offset >> 40) << 8;
3121         nfp_offset = offset & ((1ull << 40) - 1);
3122
3123         PMD_CPP_LOG(DEBUG, "%s: count %zu and offset %jd\n", __func__, count,
3124                            offset);
3125         PMD_CPP_LOG(DEBUG, "%s: cpp_id %08x and nfp_offset %jd\n", __func__,
3126                            cpp_id, nfp_offset);
3127
3128         /* Adjust length if not aligned */
3129         if (((nfp_offset + (off_t)count - 1) & ~(NFP_CPP_MEMIO_BOUNDARY - 1)) !=
3130             (nfp_offset & ~(NFP_CPP_MEMIO_BOUNDARY - 1))) {
3131                 curlen = NFP_CPP_MEMIO_BOUNDARY -
3132                         (nfp_offset & (NFP_CPP_MEMIO_BOUNDARY - 1));
3133         }
3134
3135         while (count > 0) {
3136                 area = nfp_cpp_area_alloc_with_name(cpp, cpp_id, "nfp.cdev",
3137                                                     nfp_offset, curlen);
3138                 if (!area) {
3139                         RTE_LOG(ERR, PMD, "%s: area alloc failed\n", __func__);
3140                         return -EIO;
3141                 }
3142
3143                 err = nfp_cpp_area_acquire(area);
3144                 if (err < 0) {
3145                         RTE_LOG(ERR, PMD, "area acquire failed\n");
3146                         nfp_cpp_area_free(area);
3147                         return -EIO;
3148                 }
3149
3150                 for (pos = 0; pos < curlen; pos += len) {
3151                         len = curlen - pos;
3152                         if (len > sizeof(tmpbuf))
3153                                 len = sizeof(tmpbuf);
3154
3155                         err = nfp_cpp_area_read(area, pos, tmpbuf, len);
3156                         if (err < 0) {
3157                                 RTE_LOG(ERR, PMD, "nfp_cpp_area_read error\n");
3158                                 nfp_cpp_area_release(area);
3159                                 nfp_cpp_area_free(area);
3160                                 return -EIO;
3161                         }
3162                         PMD_CPP_LOG(DEBUG, "%s: sending %u of %zu\n", __func__,
3163                                            len, count);
3164
3165                         err = send(sockfd, tmpbuf, len, 0);
3166                         if (err != (int)len) {
3167                                 RTE_LOG(ERR, PMD,
3168                                         "%s: error when sending: %d of %zu\n",
3169                                         __func__, err, count);
3170                                 nfp_cpp_area_release(area);
3171                                 nfp_cpp_area_free(area);
3172                                 return -EIO;
3173                         }
3174                 }
3175
3176                 nfp_offset += pos;
3177                 totlen += pos;
3178                 nfp_cpp_area_release(area);
3179                 nfp_cpp_area_free(area);
3180
3181                 count -= pos;
3182                 curlen = (count > NFP_CPP_MEMIO_BOUNDARY) ?
3183                         NFP_CPP_MEMIO_BOUNDARY : count;
3184         }
3185         return 0;
3186 }
3187
3188 #define NFP_IOCTL 'n'
3189 #define NFP_IOCTL_CPP_IDENTIFICATION _IOW(NFP_IOCTL, 0x8f, uint32_t)
3190 /*
3191  * Serving a ioctl command from host NFP tools. This usually goes to
3192  * a kernel driver char driver but it is not available when the PF is
3193  * bound to the PMD. Currently just one ioctl command is served and it
3194  * does not require any CPP access at all.
3195  */
3196 static int
3197 nfp_cpp_bridge_serve_ioctl(int sockfd, struct nfp_cpp *cpp)
3198 {
3199         uint32_t cmd, ident_size, tmp;
3200         int err;
3201
3202         /* Reading now the IOCTL command */
3203         err = recv(sockfd, &cmd, 4, 0);
3204         if (err != 4) {
3205                 RTE_LOG(ERR, PMD, "%s: read error from socket\n", __func__);
3206                 return -EIO;
3207         }
3208
3209         /* Only supporting NFP_IOCTL_CPP_IDENTIFICATION */
3210         if (cmd != NFP_IOCTL_CPP_IDENTIFICATION) {
3211                 RTE_LOG(ERR, PMD, "%s: unknown cmd %d\n", __func__, cmd);
3212                 return -EINVAL;
3213         }
3214
3215         err = recv(sockfd, &ident_size, 4, 0);
3216         if (err != 4) {
3217                 RTE_LOG(ERR, PMD, "%s: read error from socket\n", __func__);
3218                 return -EIO;
3219         }
3220
3221         tmp = nfp_cpp_model(cpp);
3222
3223         PMD_CPP_LOG(DEBUG, "%s: sending NFP model %08x\n", __func__, tmp);
3224
3225         err = send(sockfd, &tmp, 4, 0);
3226         if (err != 4) {
3227                 RTE_LOG(ERR, PMD, "%s: error writing to socket\n", __func__);
3228                 return -EIO;
3229         }
3230
3231         tmp = cpp->interface;
3232
3233         PMD_CPP_LOG(DEBUG, "%s: sending NFP interface %08x\n", __func__, tmp);
3234
3235         err = send(sockfd, &tmp, 4, 0);
3236         if (err != 4) {
3237                 RTE_LOG(ERR, PMD, "%s: error writing to socket\n", __func__);
3238                 return -EIO;
3239         }
3240
3241         return 0;
3242 }
3243
3244 #define NFP_BRIDGE_OP_READ      20
3245 #define NFP_BRIDGE_OP_WRITE     30
3246 #define NFP_BRIDGE_OP_IOCTL     40
3247
3248 /*
3249  * This is the code to be executed by a service core. The CPP bridge interface
3250  * is based on a unix socket and requests usually received by a kernel char
3251  * driver, read, write and ioctl, are handled by the CPP bridge. NFP host tools
3252  * can be executed with a wrapper library and LD_LIBRARY being completely
3253  * unaware of the CPP bridge performing the NFP kernel char driver for CPP
3254  * accesses.
3255  */
3256 static int32_t
3257 nfp_cpp_bridge_service_func(void *args)
3258 {
3259         struct sockaddr address;
3260         struct nfp_cpp *cpp = args;
3261         int sockfd, datafd, op, ret;
3262
3263         unlink("/tmp/nfp_cpp");
3264         sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
3265         if (sockfd < 0) {
3266                 RTE_LOG(ERR, PMD, "%s: socket creation error. Service failed\n",
3267                         __func__);
3268                 return -EIO;
3269         }
3270
3271         memset(&address, 0, sizeof(struct sockaddr));
3272
3273         address.sa_family = AF_UNIX;
3274         strcpy(address.sa_data, "/tmp/nfp_cpp");
3275
3276         ret = bind(sockfd, (const struct sockaddr *)&address,
3277                    sizeof(struct sockaddr));
3278         if (ret < 0) {
3279                 RTE_LOG(ERR, PMD, "%s: bind error (%d). Service failed\n",
3280                                   __func__, errno);
3281                 close(sockfd);
3282                 return ret;
3283         }
3284
3285         ret = listen(sockfd, 20);
3286         if (ret < 0) {
3287                 RTE_LOG(ERR, PMD, "%s: listen error(%d). Service failed\n",
3288                                   __func__, errno);
3289                 close(sockfd);
3290                 return ret;
3291         }
3292
3293         for (;;) {
3294                 datafd = accept(sockfd, NULL, NULL);
3295                 if (datafd < 0) {
3296                         RTE_LOG(ERR, PMD, "%s: accept call error (%d)\n",
3297                                           __func__, errno);
3298                         RTE_LOG(ERR, PMD, "%s: service failed\n", __func__);
3299                         close(sockfd);
3300                         return -EIO;
3301                 }
3302
3303                 while (1) {
3304                         ret = recv(datafd, &op, 4, 0);
3305                         if (ret <= 0) {
3306                                 PMD_CPP_LOG(DEBUG, "%s: socket close\n",
3307                                                    __func__);
3308                                 break;
3309                         }
3310
3311                         PMD_CPP_LOG(DEBUG, "%s: getting op %u\n", __func__, op);
3312
3313                         if (op == NFP_BRIDGE_OP_READ)
3314                                 nfp_cpp_bridge_serve_read(datafd, cpp);
3315
3316                         if (op == NFP_BRIDGE_OP_WRITE)
3317                                 nfp_cpp_bridge_serve_write(datafd, cpp);
3318
3319                         if (op == NFP_BRIDGE_OP_IOCTL)
3320                                 nfp_cpp_bridge_serve_ioctl(datafd, cpp);
3321
3322                         if (op == 0)
3323                                 break;
3324                 }
3325                 close(datafd);
3326         }
3327         close(sockfd);
3328
3329         return 0;
3330 }
3331
3332 #define DEFAULT_FW_PATH       "/lib/firmware/netronome"
3333
3334 static int
3335 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
3336 {
3337         struct nfp_cpp *cpp = nsp->cpp;
3338         int fw_f;
3339         char *fw_buf;
3340         char fw_name[125];
3341         char serial[40];
3342         struct stat file_stat;
3343         off_t fsize, bytes;
3344
3345         /* Looking for firmware file in order of priority */
3346
3347         /* First try to find a firmware image specific for this device */
3348         snprintf(serial, sizeof(serial),
3349                         "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x",
3350                 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3],
3351                 cpp->serial[4], cpp->serial[5], cpp->interface >> 8,
3352                 cpp->interface & 0xff);
3353
3354         snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
3355                         serial);
3356
3357         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
3358         fw_f = open(fw_name, O_RDONLY);
3359         if (fw_f >= 0)
3360                 goto read_fw;
3361
3362         /* Then try the PCI name */
3363         snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
3364                         dev->device.name);
3365
3366         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
3367         fw_f = open(fw_name, O_RDONLY);
3368         if (fw_f >= 0)
3369                 goto read_fw;
3370
3371         /* Finally try the card type and media */
3372         snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
3373         PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
3374         fw_f = open(fw_name, O_RDONLY);
3375         if (fw_f < 0) {
3376                 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
3377                 return -ENOENT;
3378         }
3379
3380 read_fw:
3381         if (fstat(fw_f, &file_stat) < 0) {
3382                 PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
3383                 close(fw_f);
3384                 return -ENOENT;
3385         }
3386
3387         fsize = file_stat.st_size;
3388         PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
3389                             fw_name, (uint64_t)fsize);
3390
3391         fw_buf = malloc((size_t)fsize);
3392         if (!fw_buf) {
3393                 PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
3394                 close(fw_f);
3395                 return -ENOMEM;
3396         }
3397         memset(fw_buf, 0, fsize);
3398
3399         bytes = read(fw_f, fw_buf, fsize);
3400         if (bytes != fsize) {
3401                 PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
3402                                    "Just %" PRIu64 " of %" PRIu64 " bytes read",
3403                                    (uint64_t)bytes, (uint64_t)fsize);
3404                 free(fw_buf);
3405                 close(fw_f);
3406                 return -EIO;
3407         }
3408
3409         PMD_DRV_LOG(INFO, "Uploading the firmware ...");
3410         nfp_nsp_load_fw(nsp, fw_buf, bytes);
3411         PMD_DRV_LOG(INFO, "Done");
3412
3413         free(fw_buf);
3414         close(fw_f);
3415
3416         return 0;
3417 }
3418
3419 static int
3420 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp,
3421              struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo)
3422 {
3423         struct nfp_nsp *nsp;
3424         const char *nfp_fw_model;
3425         char card_desc[100];
3426         int err = 0;
3427
3428         nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno");
3429
3430         if (nfp_fw_model) {
3431                 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model);
3432         } else {
3433                 PMD_DRV_LOG(ERR, "firmware model NOT found");
3434                 return -EIO;
3435         }
3436
3437         if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) {
3438                 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u",
3439                        nfp_eth_table->count);
3440                 return -EIO;
3441         }
3442
3443         PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports",
3444                            nfp_eth_table->count);
3445
3446         PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed);
3447
3448         snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw",
3449                         nfp_fw_model, nfp_eth_table->count,
3450                         nfp_eth_table->ports[0].speed / 1000);
3451
3452         nsp = nfp_nsp_open(cpp);
3453         if (!nsp) {
3454                 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle");
3455                 return -EIO;
3456         }
3457
3458         nfp_nsp_device_soft_reset(nsp);
3459         err = nfp_fw_upload(dev, nsp, card_desc);
3460
3461         nfp_nsp_close(nsp);
3462         return err;
3463 }
3464
3465 static int nfp_init_phyports(struct nfp_pf_dev *pf_dev)
3466 {
3467         struct nfp_net_hw *hw;
3468         struct rte_eth_dev *eth_dev;
3469         int ret = 0;
3470         int i;
3471
3472         /* Loop through all physical ports on PF */
3473         for (i = 0; i < pf_dev->total_phyports; i++) {
3474                 const unsigned int numa_node = rte_socket_id();
3475                 char port_name[RTE_ETH_NAME_MAX_LEN];
3476
3477                 snprintf(port_name, sizeof(port_name), "%s_port%d",
3478                          pf_dev->pci_dev->device.name, i);
3479
3480                 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3481                         eth_dev = rte_eth_dev_attach_secondary(port_name);
3482                         if (!eth_dev) {
3483                                 RTE_LOG(ERR, EAL,
3484                                 "secondary process attach failed, "
3485                                 "ethdev doesn't exist");
3486                                 ret = -ENODEV;
3487                                 goto error;
3488                         }
3489
3490                         eth_dev->process_private = pf_dev->cpp;
3491                         goto nfp_net_init;
3492                 }
3493
3494                 /* First port has already been initialized */
3495                 if (i == 0) {
3496                         eth_dev = pf_dev->eth_dev;
3497                         goto skip_dev_alloc;
3498                 }
3499
3500                 /* Allocate a eth_dev for remaining ports */
3501                 eth_dev = rte_eth_dev_allocate(port_name);
3502                 if (!eth_dev) {
3503                         ret = -ENODEV;
3504                         goto port_cleanup;
3505                 }
3506
3507                 /* Allocate memory for remaining ports */
3508                 eth_dev->data->dev_private =
3509                         rte_zmalloc_socket(port_name, sizeof(struct nfp_net_hw),
3510                                            RTE_CACHE_LINE_SIZE, numa_node);
3511                 if (!eth_dev->data->dev_private) {
3512                         ret = -ENOMEM;
3513                         rte_eth_dev_release_port(eth_dev);
3514                         goto port_cleanup;
3515                 }
3516
3517 skip_dev_alloc:
3518                 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
3519
3520                 /* Add this device to the PF's array of physical ports */
3521                 pf_dev->ports[i] = hw;
3522
3523                 hw->pf_dev = pf_dev;
3524                 hw->cpp = pf_dev->cpp;
3525                 hw->eth_dev = eth_dev;
3526                 hw->idx = i;
3527                 hw->is_phyport = true;
3528
3529 nfp_net_init:
3530                 eth_dev->device = &pf_dev->pci_dev->device;
3531
3532                 /* ctrl/tx/rx BAR mappings and remaining init happens in
3533                  * nfp_net_init
3534                  */
3535                 ret = nfp_net_init(eth_dev);
3536
3537                 if (ret) {
3538                         ret = -ENODEV;
3539                         goto port_cleanup;
3540                 }
3541
3542                 rte_eth_dev_probing_finish(eth_dev);
3543
3544         } /* End loop, all ports on this PF */
3545         return 0;
3546
3547 port_cleanup:
3548         for (i = 0; i < pf_dev->total_phyports; i++) {
3549                 if (pf_dev->ports[i] && pf_dev->ports[i]->eth_dev) {
3550                         struct rte_eth_dev *tmp_dev;
3551                         tmp_dev = pf_dev->ports[i]->eth_dev;
3552                         rte_eth_dev_release_port(tmp_dev);
3553                         pf_dev->ports[i] = NULL;
3554                 }
3555         }
3556 error:
3557         return ret;
3558 }
3559
3560 static int nfp_pf_init(struct rte_eth_dev *eth_dev)
3561 {
3562         struct rte_pci_device *pci_dev;
3563         struct nfp_net_hw *hw = NULL;
3564         struct nfp_pf_dev *pf_dev = NULL;
3565         struct nfp_cpp *cpp;
3566         struct nfp_hwinfo *hwinfo;
3567         struct nfp_rtsym_table *sym_tbl;
3568         struct nfp_eth_table *nfp_eth_table = NULL;
3569         struct rte_service_spec service;
3570         char name[RTE_ETH_NAME_MAX_LEN];
3571         int total_ports;
3572         int ret = -ENODEV;
3573         int err;
3574
3575         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
3576         hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev);
3577
3578         if (!pci_dev)
3579                 return ret;
3580
3581         /*
3582          * When device bound to UIO, the device could be used, by mistake,
3583          * by two DPDK apps, and the UIO driver does not avoid it. This
3584          * could lead to a serious problem when configuring the NFP CPP
3585          * interface. Here we avoid this telling to the CPP init code to
3586          * use a lock file if UIO is being used.
3587          */
3588         if (pci_dev->kdrv == RTE_PCI_KDRV_VFIO)
3589                 cpp = nfp_cpp_from_device_name(pci_dev, 0);
3590         else
3591                 cpp = nfp_cpp_from_device_name(pci_dev, 1);
3592
3593         if (!cpp) {
3594                 PMD_INIT_LOG(ERR, "A CPP handle can not be obtained");
3595                 ret = -EIO;
3596                 goto error;
3597         }
3598
3599         hwinfo = nfp_hwinfo_read(cpp);
3600         if (!hwinfo) {
3601                 PMD_INIT_LOG(ERR, "Error reading hwinfo table");
3602                 ret = -EIO;
3603                 goto error;
3604         }
3605
3606         nfp_eth_table = nfp_eth_read_ports(cpp);
3607         if (!nfp_eth_table) {
3608                 PMD_INIT_LOG(ERR, "Error reading NFP ethernet table");
3609                 ret = -EIO;
3610                 goto hwinfo_cleanup;
3611         }
3612
3613         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3614                 if (nfp_fw_setup(pci_dev, cpp, nfp_eth_table, hwinfo)) {
3615                         PMD_INIT_LOG(ERR, "Error when uploading firmware");
3616                         ret = -EIO;
3617                         goto eth_table_cleanup;
3618                 }
3619         }
3620
3621         /* Now the symbol table should be there */
3622         sym_tbl = nfp_rtsym_table_read(cpp);
3623         if (!sym_tbl) {
3624                 PMD_INIT_LOG(ERR, "Something is wrong with the firmware"
3625                                 " symbol table");
3626                 ret = -EIO;
3627                 goto eth_table_cleanup;
3628         }
3629
3630         total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err);
3631         if (total_ports != (int)nfp_eth_table->count) {
3632                 PMD_DRV_LOG(ERR, "Inconsistent number of ports");
3633                 ret = -EIO;
3634                 goto sym_tbl_cleanup;
3635         }
3636
3637         PMD_INIT_LOG(INFO, "Total physical ports: %d", total_ports);
3638
3639         if (total_ports <= 0 || total_ports > 8) {
3640                 PMD_INIT_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value");
3641                 ret = -ENODEV;
3642                 goto sym_tbl_cleanup;
3643         }
3644         /* Allocate memory for the PF "device" */
3645         snprintf(name, sizeof(name), "nfp_pf%d", eth_dev->data->port_id);
3646         pf_dev = rte_zmalloc(name, sizeof(*pf_dev), 0);
3647         if (!pf_dev) {
3648                 ret = -ENOMEM;
3649                 goto sym_tbl_cleanup;
3650         }
3651
3652         /* Populate the newly created PF device */
3653         pf_dev->cpp = cpp;
3654         pf_dev->hwinfo = hwinfo;
3655         pf_dev->sym_tbl = sym_tbl;
3656         pf_dev->total_phyports = total_ports;
3657
3658         if (total_ports > 1)
3659                 pf_dev->multiport = true;
3660
3661         pf_dev->pci_dev = pci_dev;
3662
3663         /* The first eth_dev is part of the PF struct */
3664         pf_dev->eth_dev = eth_dev;
3665
3666         /* Map the symbol table */
3667         pf_dev->ctrl_bar = nfp_rtsym_map(pf_dev->sym_tbl, "_pf0_net_bar0",
3668                                      pf_dev->total_phyports * 32768,
3669                                      &pf_dev->ctrl_area);
3670         if (!pf_dev->ctrl_bar) {
3671                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for _pf0_net_ctrl_bar");
3672                 ret = -EIO;
3673                 goto pf_cleanup;
3674         }
3675
3676         PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar);
3677
3678         /* configure access to tx/rx vNIC BARs */
3679         pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0,
3680                                               NFP_PCIE_QUEUE(0),
3681                                               NFP_QCP_QUEUE_AREA_SZ,
3682                                               &pf_dev->hwqueues_area);
3683         if (!pf_dev->hw_queues) {
3684                 PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
3685                 ret = -EIO;
3686                 goto ctrl_area_cleanup;
3687         }
3688
3689         PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", pf_dev->hw_queues);
3690
3691         /* Initialize and prep physical ports now
3692          * This will loop through all physical ports
3693          */
3694         ret = nfp_init_phyports(pf_dev);
3695         if (ret) {
3696                 PMD_INIT_LOG(ERR, "Could not create physical ports");
3697                 goto hwqueues_cleanup;
3698         }
3699
3700         /*
3701          * The rte_service needs to be created just once per PMD.
3702          * And the cpp handler needs to be linked to the service.
3703          * Secondary processes will be used for debugging DPDK apps
3704          * when requiring to use the CPP interface for accessing NFP
3705          * components. And the cpp handler for secondary processes is
3706          * available at this point.
3707          */
3708         memset(&service, 0, sizeof(struct rte_service_spec));
3709         snprintf(service.name, sizeof(service.name), "nfp_cpp_service");
3710         service.callback = nfp_cpp_bridge_service_func;
3711         service.callback_userdata = (void *)cpp;
3712
3713         if (rte_service_component_register(&service,
3714                                            &hw->nfp_cpp_service_id))
3715                 RTE_LOG(ERR, PMD, "NFP CPP bridge service register() failed");
3716         else
3717                 RTE_LOG(DEBUG, PMD, "NFP CPP bridge service registered");
3718
3719         return 0;
3720
3721 hwqueues_cleanup:
3722         nfp_cpp_area_free(pf_dev->hwqueues_area);
3723 ctrl_area_cleanup:
3724         nfp_cpp_area_free(pf_dev->ctrl_area);
3725 pf_cleanup:
3726         rte_free(pf_dev);
3727 sym_tbl_cleanup:
3728         free(sym_tbl);
3729 eth_table_cleanup:
3730         free(nfp_eth_table);
3731 hwinfo_cleanup:
3732         free(hwinfo);
3733 error:
3734         return ret;
3735 }
3736
3737 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3738                             struct rte_pci_device *dev)
3739 {
3740         return rte_eth_dev_pci_generic_probe(dev,
3741                 sizeof(struct nfp_net_hw), nfp_pf_init);
3742 }
3743
3744 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = {
3745         {
3746                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
3747                                PCI_DEVICE_ID_NFP4000_PF_NIC)
3748         },
3749         {
3750                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
3751                                PCI_DEVICE_ID_NFP6000_PF_NIC)
3752         },
3753         {
3754                 .vendor_id = 0,
3755         },
3756 };
3757
3758 static const struct rte_pci_id pci_id_nfp_vf_net_map[] = {
3759         {
3760                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
3761                                PCI_DEVICE_ID_NFP6000_VF_NIC)
3762         },
3763         {
3764                 .vendor_id = 0,
3765         },
3766 };
3767
3768 static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3769         struct rte_pci_device *pci_dev)
3770 {
3771         return rte_eth_dev_pci_generic_probe(pci_dev,
3772                 sizeof(struct nfp_net_adapter), nfp_net_init);
3773 }
3774
3775 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev)
3776 {
3777         struct rte_eth_dev *eth_dev;
3778
3779         eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
3780         if (eth_dev == NULL)
3781                 return 0; /* port already released */
3782         if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) ||
3783             (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC))
3784                 return -ENOTSUP;
3785
3786         return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
3787 }
3788
3789 static struct rte_pci_driver rte_nfp_net_pf_pmd = {
3790         .id_table = pci_id_nfp_pf_net_map,
3791         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3792         .probe = nfp_pf_pci_probe,
3793         .remove = eth_nfp_pci_remove,
3794 };
3795
3796 static struct rte_pci_driver rte_nfp_net_vf_pmd = {
3797         .id_table = pci_id_nfp_vf_net_map,
3798         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3799         .probe = eth_nfp_pci_probe,
3800         .remove = eth_nfp_pci_remove,
3801 };
3802
3803 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd);
3804 RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd);
3805 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map);
3806 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map);
3807 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio");
3808 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio");
3809 RTE_LOG_REGISTER(nfp_logtype_init, pmd.net.nfp.init, NOTICE);
3810 RTE_LOG_REGISTER(nfp_logtype_driver, pmd.net.nfp.driver, NOTICE);
3811 /*
3812  * Local variables:
3813  * c-file-style: "Linux"
3814  * indent-tabs-mode: t
3815  * End:
3816  */