drivers/net/ngbe/ngbe_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd.
   3  * Copyright(c) 2010-2017 Intel Corporation
   4  */
   5
   6 #include <errno.h>
   7 #include <rte_common.h>
   8 #include <ethdev_pci.h>
   9
  10 #include <rte_alarm.h>
  11
  12 #include "ngbe_logs.h"
  13 #include "ngbe.h"
  14 #include "ngbe_ethdev.h"
  15 #include "ngbe_rxtx.h"
  16 #include "ngbe_regs_group.h"
  17
  18 static const struct reg_info ngbe_regs_general[] = {
  19         {NGBE_RST, 1, 1, "NGBE_RST"},
  20         {NGBE_STAT, 1, 1, "NGBE_STAT"},
  21         {NGBE_PORTCTL, 1, 1, "NGBE_PORTCTL"},
  22         {NGBE_GPIODATA, 1, 1, "NGBE_GPIODATA"},
  23         {NGBE_GPIOCTL, 1, 1, "NGBE_GPIOCTL"},
  24         {NGBE_LEDCTL, 1, 1, "NGBE_LEDCTL"},
  25         {0, 0, 0, ""}
  26 };
  27
  28 static const struct reg_info ngbe_regs_nvm[] = {
  29         {0, 0, 0, ""}
  30 };
  31
  32 static const struct reg_info ngbe_regs_interrupt[] = {
  33         {0, 0, 0, ""}
  34 };
  35
  36 static const struct reg_info ngbe_regs_fctl_others[] = {
  37         {0, 0, 0, ""}
  38 };
  39
  40 static const struct reg_info ngbe_regs_rxdma[] = {
  41         {0, 0, 0, ""}
  42 };
  43
  44 static const struct reg_info ngbe_regs_rx[] = {
  45         {0, 0, 0, ""}
  46 };
  47
  48 static struct reg_info ngbe_regs_tx[] = {
  49         {0, 0, 0, ""}
  50 };
  51
  52 static const struct reg_info ngbe_regs_wakeup[] = {
  53         {0, 0, 0, ""}
  54 };
  55
  56 static const struct reg_info ngbe_regs_mac[] = {
  57         {0, 0, 0, ""}
  58 };
  59
  60 static const struct reg_info ngbe_regs_diagnostic[] = {
  61         {0, 0, 0, ""},
  62 };
  63
  64 /* PF registers */
  65 static const struct reg_info *ngbe_regs_others[] = {
  66                                 ngbe_regs_general,
  67                                 ngbe_regs_nvm,
  68                                 ngbe_regs_interrupt,
  69                                 ngbe_regs_fctl_others,
  70                                 ngbe_regs_rxdma,
  71                                 ngbe_regs_rx,
  72                                 ngbe_regs_tx,
  73                                 ngbe_regs_wakeup,
  74                                 ngbe_regs_mac,
  75                                 ngbe_regs_diagnostic,
  76                                 NULL};
  77
  78 static int ngbe_dev_close(struct rte_eth_dev *dev);
  79 static int ngbe_dev_link_update(struct rte_eth_dev *dev,
  80                                 int wait_to_complete);
  81 static int ngbe_dev_stats_reset(struct rte_eth_dev *dev);
  82 static void ngbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
  83 static void ngbe_vlan_hw_strip_disable(struct rte_eth_dev *dev,
  84                                         uint16_t queue);
  85
  86 static void ngbe_dev_link_status_print(struct rte_eth_dev *dev);
  87 static int ngbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
  88 static int ngbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev);
  89 static int ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev);
  90 static int ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
  91 static void ngbe_dev_interrupt_handler(void *param);
  92 static void ngbe_configure_msix(struct rte_eth_dev *dev);
  93
  94 #define NGBE_SET_HWSTRIP(h, q) do {\
  95                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
  96                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
  97                 (h)->bitmap[idx] |= 1 << bit;\
  98         } while (0)
  99
 100 #define NGBE_CLEAR_HWSTRIP(h, q) do {\
 101                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 102                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 103                 (h)->bitmap[idx] &= ~(1 << bit);\
 104         } while (0)
 105
 106 #define NGBE_GET_HWSTRIP(h, q, r) do {\
 107                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 108                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 109                 (r) = (h)->bitmap[idx] >> bit & 1;\
 110         } while (0)
 111
 112 /*
 113  * The set of PCI devices this driver supports
 114  */
 115 static const struct rte_pci_id pci_id_ngbe_map[] = {
 116         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A2) },
 117         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A2S) },
 118         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A4) },
 119         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A4S) },
 120         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL2) },
 121         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S) },
 122         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL4) },
 123         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S) },
 124         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860NCSI) },
 125         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A1) },
 126         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A1L) },
 127         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W) },
 128         { .vendor_id = 0, /* sentinel */ },
 129 };
 130
 131 static const struct rte_eth_desc_lim rx_desc_lim = {
 132         .nb_max = NGBE_RING_DESC_MAX,
 133         .nb_min = NGBE_RING_DESC_MIN,
 134         .nb_align = NGBE_RXD_ALIGN,
 135 };
 136
 137 static const struct rte_eth_desc_lim tx_desc_lim = {
 138         .nb_max = NGBE_RING_DESC_MAX,
 139         .nb_min = NGBE_RING_DESC_MIN,
 140         .nb_align = NGBE_TXD_ALIGN,
 141         .nb_seg_max = NGBE_TX_MAX_SEG,
 142         .nb_mtu_seg_max = NGBE_TX_MAX_SEG,
 143 };
 144
 145 static const struct eth_dev_ops ngbe_eth_dev_ops;
 146
 147 #define HW_XSTAT(m) {#m, offsetof(struct ngbe_hw_stats, m)}
 148 #define HW_XSTAT_NAME(m, n) {n, offsetof(struct ngbe_hw_stats, m)}
 149 static const struct rte_ngbe_xstats_name_off rte_ngbe_stats_strings[] = {
 150         /* MNG RxTx */
 151         HW_XSTAT(mng_bmc2host_packets),
 152         HW_XSTAT(mng_host2bmc_packets),
 153         /* Basic RxTx */
 154         HW_XSTAT(rx_packets),
 155         HW_XSTAT(tx_packets),
 156         HW_XSTAT(rx_bytes),
 157         HW_XSTAT(tx_bytes),
 158         HW_XSTAT(rx_total_bytes),
 159         HW_XSTAT(rx_total_packets),
 160         HW_XSTAT(tx_total_packets),
 161         HW_XSTAT(rx_total_missed_packets),
 162         HW_XSTAT(rx_broadcast_packets),
 163         HW_XSTAT(rx_multicast_packets),
 164         HW_XSTAT(rx_management_packets),
 165         HW_XSTAT(tx_management_packets),
 166         HW_XSTAT(rx_management_dropped),
 167
 168         /* Basic Error */
 169         HW_XSTAT(rx_crc_errors),
 170         HW_XSTAT(rx_illegal_byte_errors),
 171         HW_XSTAT(rx_error_bytes),
 172         HW_XSTAT(rx_mac_short_packet_dropped),
 173         HW_XSTAT(rx_length_errors),
 174         HW_XSTAT(rx_undersize_errors),
 175         HW_XSTAT(rx_fragment_errors),
 176         HW_XSTAT(rx_oversize_errors),
 177         HW_XSTAT(rx_jabber_errors),
 178         HW_XSTAT(rx_l3_l4_xsum_error),
 179         HW_XSTAT(mac_local_errors),
 180         HW_XSTAT(mac_remote_errors),
 181
 182         /* MACSEC */
 183         HW_XSTAT(tx_macsec_pkts_untagged),
 184         HW_XSTAT(tx_macsec_pkts_encrypted),
 185         HW_XSTAT(tx_macsec_pkts_protected),
 186         HW_XSTAT(tx_macsec_octets_encrypted),
 187         HW_XSTAT(tx_macsec_octets_protected),
 188         HW_XSTAT(rx_macsec_pkts_untagged),
 189         HW_XSTAT(rx_macsec_pkts_badtag),
 190         HW_XSTAT(rx_macsec_pkts_nosci),
 191         HW_XSTAT(rx_macsec_pkts_unknownsci),
 192         HW_XSTAT(rx_macsec_octets_decrypted),
 193         HW_XSTAT(rx_macsec_octets_validated),
 194         HW_XSTAT(rx_macsec_sc_pkts_unchecked),
 195         HW_XSTAT(rx_macsec_sc_pkts_delayed),
 196         HW_XSTAT(rx_macsec_sc_pkts_late),
 197         HW_XSTAT(rx_macsec_sa_pkts_ok),
 198         HW_XSTAT(rx_macsec_sa_pkts_invalid),
 199         HW_XSTAT(rx_macsec_sa_pkts_notvalid),
 200         HW_XSTAT(rx_macsec_sa_pkts_unusedsa),
 201         HW_XSTAT(rx_macsec_sa_pkts_notusingsa),
 202
 203         /* MAC RxTx */
 204         HW_XSTAT(rx_size_64_packets),
 205         HW_XSTAT(rx_size_65_to_127_packets),
 206         HW_XSTAT(rx_size_128_to_255_packets),
 207         HW_XSTAT(rx_size_256_to_511_packets),
 208         HW_XSTAT(rx_size_512_to_1023_packets),
 209         HW_XSTAT(rx_size_1024_to_max_packets),
 210         HW_XSTAT(tx_size_64_packets),
 211         HW_XSTAT(tx_size_65_to_127_packets),
 212         HW_XSTAT(tx_size_128_to_255_packets),
 213         HW_XSTAT(tx_size_256_to_511_packets),
 214         HW_XSTAT(tx_size_512_to_1023_packets),
 215         HW_XSTAT(tx_size_1024_to_max_packets),
 216
 217         /* Flow Control */
 218         HW_XSTAT(tx_xon_packets),
 219         HW_XSTAT(rx_xon_packets),
 220         HW_XSTAT(tx_xoff_packets),
 221         HW_XSTAT(rx_xoff_packets),
 222
 223         HW_XSTAT_NAME(tx_xon_packets, "tx_flow_control_xon_packets"),
 224         HW_XSTAT_NAME(rx_xon_packets, "rx_flow_control_xon_packets"),
 225         HW_XSTAT_NAME(tx_xoff_packets, "tx_flow_control_xoff_packets"),
 226         HW_XSTAT_NAME(rx_xoff_packets, "rx_flow_control_xoff_packets"),
 227 };
 228
 229 #define NGBE_NB_HW_STATS (sizeof(rte_ngbe_stats_strings) / \
 230                            sizeof(rte_ngbe_stats_strings[0]))
 231
 232 /* Per-queue statistics */
 233 #define QP_XSTAT(m) {#m, offsetof(struct ngbe_hw_stats, qp[0].m)}
 234 static const struct rte_ngbe_xstats_name_off rte_ngbe_qp_strings[] = {
 235         QP_XSTAT(rx_qp_packets),
 236         QP_XSTAT(tx_qp_packets),
 237         QP_XSTAT(rx_qp_bytes),
 238         QP_XSTAT(tx_qp_bytes),
 239         QP_XSTAT(rx_qp_mc_packets),
 240 };
 241
 242 #define NGBE_NB_QP_STATS (sizeof(rte_ngbe_qp_strings) / \
 243                            sizeof(rte_ngbe_qp_strings[0]))
 244
 245 static inline int32_t
 246 ngbe_pf_reset_hw(struct ngbe_hw *hw)
 247 {
 248         uint32_t ctrl_ext;
 249         int32_t status;
 250
 251         status = hw->mac.reset_hw(hw);
 252
 253         ctrl_ext = rd32(hw, NGBE_PORTCTL);
 254         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 255         ctrl_ext |= NGBE_PORTCTL_RSTDONE;
 256         wr32(hw, NGBE_PORTCTL, ctrl_ext);
 257         ngbe_flush(hw);
 258
 259         if (status == NGBE_ERR_SFP_NOT_PRESENT)
 260                 status = 0;
 261         return status;
 262 }
 263
 264 static inline void
 265 ngbe_enable_intr(struct rte_eth_dev *dev)
 266 {
 267         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 268         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 269
 270         wr32(hw, NGBE_IENMISC, intr->mask_misc);
 271         wr32(hw, NGBE_IMC(0), intr->mask & BIT_MASK32);
 272         ngbe_flush(hw);
 273 }
 274
 275 static void
 276 ngbe_disable_intr(struct ngbe_hw *hw)
 277 {
 278         PMD_INIT_FUNC_TRACE();
 279
 280         wr32(hw, NGBE_IMS(0), NGBE_IMS_MASK);
 281         ngbe_flush(hw);
 282 }
 283
 284 /*
 285  * Ensure that all locks are released before first NVM or PHY access
 286  */
 287 static void
 288 ngbe_swfw_lock_reset(struct ngbe_hw *hw)
 289 {
 290         uint16_t mask;
 291
 292         /*
 293          * These ones are more tricky since they are common to all ports; but
 294          * swfw_sync retries last long enough (1s) to be almost sure that if
 295          * lock can not be taken it is due to an improper lock of the
 296          * semaphore.
 297          */
 298         mask = NGBE_MNGSEM_SWPHY |
 299                NGBE_MNGSEM_SWMBX |
 300                NGBE_MNGSEM_SWFLASH;
 301         if (hw->mac.acquire_swfw_sync(hw, mask) < 0)
 302                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 303
 304         hw->mac.release_swfw_sync(hw, mask);
 305 }
 306
 307 static int
 308 eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 309 {
 310         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 311         struct ngbe_hw *hw = ngbe_dev_hw(eth_dev);
 312         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(eth_dev);
 313         struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(eth_dev);
 314         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 315         const struct rte_memzone *mz;
 316         uint32_t ctrl_ext;
 317         int err, ret;
 318
 319         PMD_INIT_FUNC_TRACE();
 320
 321         eth_dev->dev_ops = &ngbe_eth_dev_ops;
 322         eth_dev->rx_queue_count       = ngbe_dev_rx_queue_count;
 323         eth_dev->rx_descriptor_status = ngbe_dev_rx_descriptor_status;
 324         eth_dev->tx_descriptor_status = ngbe_dev_tx_descriptor_status;
 325         eth_dev->rx_pkt_burst = &ngbe_recv_pkts;
 326         eth_dev->tx_pkt_burst = &ngbe_xmit_pkts;
 327         eth_dev->tx_pkt_prepare = &ngbe_prep_pkts;
 328
 329         /*
 330          * For secondary processes, we don't initialise any further as primary
 331          * has already done this work. Only check we don't need a different
 332          * Rx and Tx function.
 333          */
 334         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 335                 struct ngbe_tx_queue *txq;
 336                 /* Tx queue function in primary, set by last queue initialized
 337                  * Tx queue may not initialized by primary process
 338                  */
 339                 if (eth_dev->data->tx_queues) {
 340                         uint16_t nb_tx_queues = eth_dev->data->nb_tx_queues;
 341                         txq = eth_dev->data->tx_queues[nb_tx_queues - 1];
 342                         ngbe_set_tx_function(eth_dev, txq);
 343                 } else {
 344                         /* Use default Tx function if we get here */
 345                         PMD_INIT_LOG(NOTICE,
 346                                 "No Tx queues configured yet. Using default Tx function.");
 347                 }
 348
 349                 ngbe_set_rx_function(eth_dev);
 350
 351                 return 0;
 352         }
 353
 354         rte_eth_copy_pci_info(eth_dev, pci_dev);
 355         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 356
 357         /* Vendor and Device ID need to be set before init of shared code */
 358         hw->device_id = pci_dev->id.device_id;
 359         hw->vendor_id = pci_dev->id.vendor_id;
 360         hw->sub_system_id = pci_dev->id.subsystem_device_id;
 361         ngbe_map_device_id(hw);
 362         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 363
 364         /* Reserve memory for interrupt status block */
 365         mz = rte_eth_dma_zone_reserve(eth_dev, "ngbe_driver", -1,
 366                 NGBE_ISB_SIZE, NGBE_ALIGN, SOCKET_ID_ANY);
 367         if (mz == NULL)
 368                 return -ENOMEM;
 369
 370         hw->isb_dma = TMZ_PADDR(mz);
 371         hw->isb_mem = TMZ_VADDR(mz);
 372
 373         /* Initialize the shared code (base driver) */
 374         err = ngbe_init_shared_code(hw);
 375         if (err != 0) {
 376                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", err);
 377                 return -EIO;
 378         }
 379
 380         /* Unlock any pending hardware semaphore */
 381         ngbe_swfw_lock_reset(hw);
 382
 383         /* Get Hardware Flow Control setting */
 384         hw->fc.requested_mode = ngbe_fc_full;
 385         hw->fc.current_mode = ngbe_fc_full;
 386         hw->fc.pause_time = NGBE_FC_PAUSE_TIME;
 387         hw->fc.low_water = NGBE_FC_XON_LOTH;
 388         hw->fc.high_water = NGBE_FC_XOFF_HITH;
 389         hw->fc.send_xon = 1;
 390
 391         err = hw->rom.init_params(hw);
 392         if (err != 0) {
 393                 PMD_INIT_LOG(ERR, "The EEPROM init failed: %d", err);
 394                 return -EIO;
 395         }
 396
 397         /* Make sure we have a good EEPROM before we read from it */
 398         err = hw->rom.validate_checksum(hw, NULL);
 399         if (err != 0) {
 400                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", err);
 401                 return -EIO;
 402         }
 403
 404         err = hw->mac.init_hw(hw);
 405         if (err != 0) {
 406                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", err);
 407                 return -EIO;
 408         }
 409
 410         /* Reset the hw statistics */
 411         ngbe_dev_stats_reset(eth_dev);
 412
 413         /* disable interrupt */
 414         ngbe_disable_intr(hw);
 415
 416         /* Allocate memory for storing MAC addresses */
 417         eth_dev->data->mac_addrs = rte_zmalloc("ngbe", RTE_ETHER_ADDR_LEN *
 418                                                hw->mac.num_rar_entries, 0);
 419         if (eth_dev->data->mac_addrs == NULL) {
 420                 PMD_INIT_LOG(ERR,
 421                              "Failed to allocate %u bytes needed to store MAC addresses",
 422                              RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 423                 return -ENOMEM;
 424         }
 425
 426         /* Copy the permanent MAC address */
 427         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr,
 428                         &eth_dev->data->mac_addrs[0]);
 429
 430         /* Allocate memory for storing hash filter MAC addresses */
 431         eth_dev->data->hash_mac_addrs = rte_zmalloc("ngbe",
 432                         RTE_ETHER_ADDR_LEN * NGBE_VMDQ_NUM_UC_MAC, 0);
 433         if (eth_dev->data->hash_mac_addrs == NULL) {
 434                 PMD_INIT_LOG(ERR,
 435                              "Failed to allocate %d bytes needed to store MAC addresses",
 436                              RTE_ETHER_ADDR_LEN * NGBE_VMDQ_NUM_UC_MAC);
 437                 rte_free(eth_dev->data->mac_addrs);
 438                 eth_dev->data->mac_addrs = NULL;
 439                 return -ENOMEM;
 440         }
 441
 442         /* initialize the vfta */
 443         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 444
 445         /* initialize the hw strip bitmap*/
 446         memset(hwstrip, 0, sizeof(*hwstrip));
 447
 448         /* initialize PF if max_vfs not zero */
 449         ret = ngbe_pf_host_init(eth_dev);
 450         if (ret) {
 451                 rte_free(eth_dev->data->mac_addrs);
 452                 eth_dev->data->mac_addrs = NULL;
 453                 rte_free(eth_dev->data->hash_mac_addrs);
 454                 eth_dev->data->hash_mac_addrs = NULL;
 455                 return ret;
 456         }
 457
 458         ctrl_ext = rd32(hw, NGBE_PORTCTL);
 459         /* let hardware know driver is loaded */
 460         ctrl_ext |= NGBE_PORTCTL_DRVLOAD;
 461         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 462         ctrl_ext |= NGBE_PORTCTL_RSTDONE;
 463         wr32(hw, NGBE_PORTCTL, ctrl_ext);
 464         ngbe_flush(hw);
 465
 466         PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 467                         (int)hw->mac.type, (int)hw->phy.type);
 468
 469         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 470                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 471                      pci_dev->id.device_id);
 472
 473         rte_intr_callback_register(intr_handle,
 474                                    ngbe_dev_interrupt_handler, eth_dev);
 475
 476         /* enable uio/vfio intr/eventfd mapping */
 477         rte_intr_enable(intr_handle);
 478
 479         /* enable support intr */
 480         ngbe_enable_intr(eth_dev);
 481
 482         return 0;
 483 }
 484
 485 static int
 486 eth_ngbe_dev_uninit(struct rte_eth_dev *eth_dev)
 487 {
 488         PMD_INIT_FUNC_TRACE();
 489
 490         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 491                 return 0;
 492
 493         ngbe_dev_close(eth_dev);
 494
 495         return 0;
 496 }
 497
 498 static int
 499 eth_ngbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 500                 struct rte_pci_device *pci_dev)
 501 {
 502         return rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
 503                         sizeof(struct ngbe_adapter),
 504                         eth_dev_pci_specific_init, pci_dev,
 505                         eth_ngbe_dev_init, NULL);
 506 }
 507
 508 static int eth_ngbe_pci_remove(struct rte_pci_device *pci_dev)
 509 {
 510         struct rte_eth_dev *ethdev;
 511
 512         ethdev = rte_eth_dev_allocated(pci_dev->device.name);
 513         if (ethdev == NULL)
 514                 return 0;
 515
 516         return rte_eth_dev_destroy(ethdev, eth_ngbe_dev_uninit);
 517 }
 518
 519 static struct rte_pci_driver rte_ngbe_pmd = {
 520         .id_table = pci_id_ngbe_map,
 521         .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
 522                      RTE_PCI_DRV_INTR_LSC,
 523         .probe = eth_ngbe_pci_probe,
 524         .remove = eth_ngbe_pci_remove,
 525 };
 526
 527 static int
 528 ngbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 529 {
 530         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 531         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(dev);
 532         uint32_t vfta;
 533         uint32_t vid_idx;
 534         uint32_t vid_bit;
 535
 536         vid_idx = (uint32_t)((vlan_id >> 5) & 0x7F);
 537         vid_bit = (uint32_t)(1 << (vlan_id & 0x1F));
 538         vfta = rd32(hw, NGBE_VLANTBL(vid_idx));
 539         if (on)
 540                 vfta |= vid_bit;
 541         else
 542                 vfta &= ~vid_bit;
 543         wr32(hw, NGBE_VLANTBL(vid_idx), vfta);
 544
 545         /* update local VFTA copy */
 546         shadow_vfta->vfta[vid_idx] = vfta;
 547
 548         return 0;
 549 }
 550
 551 static void
 552 ngbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 553 {
 554         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 555         struct ngbe_rx_queue *rxq;
 556         bool restart;
 557         uint32_t rxcfg, rxbal, rxbah;
 558
 559         if (on)
 560                 ngbe_vlan_hw_strip_enable(dev, queue);
 561         else
 562                 ngbe_vlan_hw_strip_disable(dev, queue);
 563
 564         rxq = dev->data->rx_queues[queue];
 565         rxbal = rd32(hw, NGBE_RXBAL(rxq->reg_idx));
 566         rxbah = rd32(hw, NGBE_RXBAH(rxq->reg_idx));
 567         rxcfg = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
 568         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) {
 569                 restart = (rxcfg & NGBE_RXCFG_ENA) &&
 570                         !(rxcfg & NGBE_RXCFG_VLAN);
 571                 rxcfg |= NGBE_RXCFG_VLAN;
 572         } else {
 573                 restart = (rxcfg & NGBE_RXCFG_ENA) &&
 574                         (rxcfg & NGBE_RXCFG_VLAN);
 575                 rxcfg &= ~NGBE_RXCFG_VLAN;
 576         }
 577         rxcfg &= ~NGBE_RXCFG_ENA;
 578
 579         if (restart) {
 580                 /* set vlan strip for ring */
 581                 ngbe_dev_rx_queue_stop(dev, queue);
 582                 wr32(hw, NGBE_RXBAL(rxq->reg_idx), rxbal);
 583                 wr32(hw, NGBE_RXBAH(rxq->reg_idx), rxbah);
 584                 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxcfg);
 585                 ngbe_dev_rx_queue_start(dev, queue);
 586         }
 587 }
 588
 589 static int
 590 ngbe_vlan_tpid_set(struct rte_eth_dev *dev,
 591                     enum rte_vlan_type vlan_type,
 592                     uint16_t tpid)
 593 {
 594         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 595         int ret = 0;
 596         uint32_t portctrl, vlan_ext, qinq;
 597
 598         portctrl = rd32(hw, NGBE_PORTCTL);
 599
 600         vlan_ext = (portctrl & NGBE_PORTCTL_VLANEXT);
 601         qinq = vlan_ext && (portctrl & NGBE_PORTCTL_QINQ);
 602         switch (vlan_type) {
 603         case RTE_ETH_VLAN_TYPE_INNER:
 604                 if (vlan_ext) {
 605                         wr32m(hw, NGBE_VLANCTL,
 606                                 NGBE_VLANCTL_TPID_MASK,
 607                                 NGBE_VLANCTL_TPID(tpid));
 608                         wr32m(hw, NGBE_DMATXCTRL,
 609                                 NGBE_DMATXCTRL_TPID_MASK,
 610                                 NGBE_DMATXCTRL_TPID(tpid));
 611                 } else {
 612                         ret = -ENOTSUP;
 613                         PMD_DRV_LOG(ERR,
 614                                 "Inner type is not supported by single VLAN");
 615                 }
 616
 617                 if (qinq) {
 618                         wr32m(hw, NGBE_TAGTPID(0),
 619                                 NGBE_TAGTPID_LSB_MASK,
 620                                 NGBE_TAGTPID_LSB(tpid));
 621                 }
 622                 break;
 623         case RTE_ETH_VLAN_TYPE_OUTER:
 624                 if (vlan_ext) {
 625                         /* Only the high 16-bits is valid */
 626                         wr32m(hw, NGBE_EXTAG,
 627                                 NGBE_EXTAG_VLAN_MASK,
 628                                 NGBE_EXTAG_VLAN(tpid));
 629                 } else {
 630                         wr32m(hw, NGBE_VLANCTL,
 631                                 NGBE_VLANCTL_TPID_MASK,
 632                                 NGBE_VLANCTL_TPID(tpid));
 633                         wr32m(hw, NGBE_DMATXCTRL,
 634                                 NGBE_DMATXCTRL_TPID_MASK,
 635                                 NGBE_DMATXCTRL_TPID(tpid));
 636                 }
 637
 638                 if (qinq) {
 639                         wr32m(hw, NGBE_TAGTPID(0),
 640                                 NGBE_TAGTPID_MSB_MASK,
 641                                 NGBE_TAGTPID_MSB(tpid));
 642                 }
 643                 break;
 644         default:
 645                 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type);
 646                 return -EINVAL;
 647         }
 648
 649         return ret;
 650 }
 651
 652 void
 653 ngbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 654 {
 655         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 656         uint32_t vlnctrl;
 657
 658         PMD_INIT_FUNC_TRACE();
 659
 660         /* Filter Table Disable */
 661         vlnctrl = rd32(hw, NGBE_VLANCTL);
 662         vlnctrl &= ~NGBE_VLANCTL_VFE;
 663         wr32(hw, NGBE_VLANCTL, vlnctrl);
 664 }
 665
 666 void
 667 ngbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 668 {
 669         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 670         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(dev);
 671         uint32_t vlnctrl;
 672         uint16_t i;
 673
 674         PMD_INIT_FUNC_TRACE();
 675
 676         /* Filter Table Enable */
 677         vlnctrl = rd32(hw, NGBE_VLANCTL);
 678         vlnctrl &= ~NGBE_VLANCTL_CFIENA;
 679         vlnctrl |= NGBE_VLANCTL_VFE;
 680         wr32(hw, NGBE_VLANCTL, vlnctrl);
 681
 682         /* write whatever is in local vfta copy */
 683         for (i = 0; i < NGBE_VFTA_SIZE; i++)
 684                 wr32(hw, NGBE_VLANTBL(i), shadow_vfta->vfta[i]);
 685 }
 686
 687 void
 688 ngbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 689 {
 690         struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(dev);
 691         struct ngbe_rx_queue *rxq;
 692
 693         if (queue >= NGBE_MAX_RX_QUEUE_NUM)
 694                 return;
 695
 696         if (on)
 697                 NGBE_SET_HWSTRIP(hwstrip, queue);
 698         else
 699                 NGBE_CLEAR_HWSTRIP(hwstrip, queue);
 700
 701         if (queue >= dev->data->nb_rx_queues)
 702                 return;
 703
 704         rxq = dev->data->rx_queues[queue];
 705
 706         if (on) {
 707                 rxq->vlan_flags = RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
 708                 rxq->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 709         } else {
 710                 rxq->vlan_flags = RTE_MBUF_F_RX_VLAN;
 711                 rxq->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 712         }
 713 }
 714
 715 static void
 716 ngbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 717 {
 718         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 719         uint32_t ctrl;
 720
 721         PMD_INIT_FUNC_TRACE();
 722
 723         ctrl = rd32(hw, NGBE_RXCFG(queue));
 724         ctrl &= ~NGBE_RXCFG_VLAN;
 725         wr32(hw, NGBE_RXCFG(queue), ctrl);
 726
 727         /* record those setting for HW strip per queue */
 728         ngbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 729 }
 730
 731 static void
 732 ngbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 733 {
 734         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 735         uint32_t ctrl;
 736
 737         PMD_INIT_FUNC_TRACE();
 738
 739         ctrl = rd32(hw, NGBE_RXCFG(queue));
 740         ctrl |= NGBE_RXCFG_VLAN;
 741         wr32(hw, NGBE_RXCFG(queue), ctrl);
 742
 743         /* record those setting for HW strip per queue */
 744         ngbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 745 }
 746
 747 static void
 748 ngbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
 749 {
 750         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 751         uint32_t ctrl;
 752
 753         PMD_INIT_FUNC_TRACE();
 754
 755         ctrl = rd32(hw, NGBE_PORTCTL);
 756         ctrl &= ~NGBE_PORTCTL_VLANEXT;
 757         ctrl &= ~NGBE_PORTCTL_QINQ;
 758         wr32(hw, NGBE_PORTCTL, ctrl);
 759 }
 760
 761 static void
 762 ngbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
 763 {
 764         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 765         uint32_t ctrl;
 766
 767         PMD_INIT_FUNC_TRACE();
 768
 769         ctrl  = rd32(hw, NGBE_PORTCTL);
 770         ctrl |= NGBE_PORTCTL_VLANEXT | NGBE_PORTCTL_QINQ;
 771         wr32(hw, NGBE_PORTCTL, ctrl);
 772 }
 773
 774 static void
 775 ngbe_qinq_hw_strip_disable(struct rte_eth_dev *dev)
 776 {
 777         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 778         uint32_t ctrl;
 779
 780         PMD_INIT_FUNC_TRACE();
 781
 782         ctrl = rd32(hw, NGBE_PORTCTL);
 783         ctrl &= ~NGBE_PORTCTL_QINQ;
 784         wr32(hw, NGBE_PORTCTL, ctrl);
 785 }
 786
 787 static void
 788 ngbe_qinq_hw_strip_enable(struct rte_eth_dev *dev)
 789 {
 790         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 791         uint32_t ctrl;
 792
 793         PMD_INIT_FUNC_TRACE();
 794
 795         ctrl  = rd32(hw, NGBE_PORTCTL);
 796         ctrl |= NGBE_PORTCTL_QINQ | NGBE_PORTCTL_VLANEXT;
 797         wr32(hw, NGBE_PORTCTL, ctrl);
 798 }
 799
 800 void
 801 ngbe_vlan_hw_strip_config(struct rte_eth_dev *dev)
 802 {
 803         struct ngbe_rx_queue *rxq;
 804         uint16_t i;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 809                 rxq = dev->data->rx_queues[i];
 810
 811                 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 812                         ngbe_vlan_hw_strip_enable(dev, i);
 813                 else
 814                         ngbe_vlan_hw_strip_disable(dev, i);
 815         }
 816 }
 817
 818 void
 819 ngbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, int mask)
 820 {
 821         uint16_t i;
 822         struct rte_eth_rxmode *rxmode;
 823         struct ngbe_rx_queue *rxq;
 824
 825         if (mask & RTE_ETH_VLAN_STRIP_MASK) {
 826                 rxmode = &dev->data->dev_conf.rxmode;
 827                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 828                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 829                                 rxq = dev->data->rx_queues[i];
 830                                 rxq->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 831                         }
 832                 else
 833                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 834                                 rxq = dev->data->rx_queues[i];
 835                                 rxq->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 836                         }
 837         }
 838 }
 839
 840 static int
 841 ngbe_vlan_offload_config(struct rte_eth_dev *dev, int mask)
 842 {
 843         struct rte_eth_rxmode *rxmode;
 844         rxmode = &dev->data->dev_conf.rxmode;
 845
 846         if (mask & RTE_ETH_VLAN_STRIP_MASK)
 847                 ngbe_vlan_hw_strip_config(dev);
 848
 849         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
 850                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER)
 851                         ngbe_vlan_hw_filter_enable(dev);
 852                 else
 853                         ngbe_vlan_hw_filter_disable(dev);
 854         }
 855
 856         if (mask & RTE_ETH_VLAN_EXTEND_MASK) {
 857                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_EXTEND)
 858                         ngbe_vlan_hw_extend_enable(dev);
 859                 else
 860                         ngbe_vlan_hw_extend_disable(dev);
 861         }
 862
 863         if (mask & RTE_ETH_QINQ_STRIP_MASK) {
 864                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP)
 865                         ngbe_qinq_hw_strip_enable(dev);
 866                 else
 867                         ngbe_qinq_hw_strip_disable(dev);
 868         }
 869
 870         return 0;
 871 }
 872
 873 static int
 874 ngbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
 875 {
 876         ngbe_config_vlan_strip_on_all_queues(dev, mask);
 877
 878         ngbe_vlan_offload_config(dev, mask);
 879
 880         return 0;
 881 }
 882
 883 static int
 884 ngbe_dev_configure(struct rte_eth_dev *dev)
 885 {
 886         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 887         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
 888
 889         PMD_INIT_FUNC_TRACE();
 890
 891         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
 892                 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
 893
 894         /* set flag to update link status after init */
 895         intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
 896
 897         /*
 898          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
 899          * allocation Rx preconditions we will reset it.
 900          */
 901         adapter->rx_bulk_alloc_allowed = true;
 902
 903         return 0;
 904 }
 905
 906 static void
 907 ngbe_dev_phy_intr_setup(struct rte_eth_dev *dev)
 908 {
 909         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 910         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 911
 912         wr32(hw, NGBE_GPIODIR, NGBE_GPIODIR_DDR(1));
 913         wr32(hw, NGBE_GPIOINTEN, NGBE_GPIOINTEN_INT(3));
 914         wr32(hw, NGBE_GPIOINTTYPE, NGBE_GPIOINTTYPE_LEVEL(0));
 915         if (hw->phy.type == ngbe_phy_yt8521s_sfi)
 916                 wr32(hw, NGBE_GPIOINTPOL, NGBE_GPIOINTPOL_ACT(0));
 917         else
 918                 wr32(hw, NGBE_GPIOINTPOL, NGBE_GPIOINTPOL_ACT(3));
 919
 920         intr->mask_misc |= NGBE_ICRMISC_GPIO;
 921 }
 922
 923 /*
 924  * Configure device link speed and setup link.
 925  * It returns 0 on success.
 926  */
 927 static int
 928 ngbe_dev_start(struct rte_eth_dev *dev)
 929 {
 930         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 931         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
 932         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 933         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 934         uint32_t intr_vector = 0;
 935         int err;
 936         bool link_up = false, negotiate = false;
 937         uint32_t speed = 0;
 938         uint32_t allowed_speeds = 0;
 939         int mask = 0;
 940         int status;
 941         uint32_t *link_speeds;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Stop the link setup handler before resetting the HW. */
 946         rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev);
 947
 948         /* disable uio/vfio intr/eventfd mapping */
 949         rte_intr_disable(intr_handle);
 950
 951         /* stop adapter */
 952         hw->adapter_stopped = 0;
 953
 954         /* reinitialize adapter, this calls reset and start */
 955         hw->nb_rx_queues = dev->data->nb_rx_queues;
 956         hw->nb_tx_queues = dev->data->nb_tx_queues;
 957         status = ngbe_pf_reset_hw(hw);
 958         if (status != 0)
 959                 return -1;
 960         hw->mac.start_hw(hw);
 961         hw->mac.get_link_status = true;
 962
 963         ngbe_set_pcie_master(hw, true);
 964
 965         /* configure PF module if SRIOV enabled */
 966         ngbe_pf_host_configure(dev);
 967
 968         ngbe_dev_phy_intr_setup(dev);
 969
 970         /* check and configure queue intr-vector mapping */
 971         if ((rte_intr_cap_multiple(intr_handle) ||
 972              !RTE_ETH_DEV_SRIOV(dev).active) &&
 973             dev->data->dev_conf.intr_conf.rxq != 0) {
 974                 intr_vector = dev->data->nb_rx_queues;
 975                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 976                         return -1;
 977         }
 978
 979         if (rte_intr_dp_is_en(intr_handle)) {
 980                 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec",
 981                                                    dev->data->nb_rx_queues)) {
 982                         PMD_INIT_LOG(ERR,
 983                                      "Failed to allocate %d rx_queues intr_vec",
 984                                      dev->data->nb_rx_queues);
 985                         return -ENOMEM;
 986                 }
 987         }
 988
 989         /* configure MSI-X for sleep until Rx interrupt */
 990         ngbe_configure_msix(dev);
 991
 992         /* initialize transmission unit */
 993         ngbe_dev_tx_init(dev);
 994
 995         /* This can fail when allocating mbufs for descriptor rings */
 996         err = ngbe_dev_rx_init(dev);
 997         if (err != 0) {
 998                 PMD_INIT_LOG(ERR, "Unable to initialize Rx hardware");
 999                 goto error;
1000         }
1001
1002         mask = RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK |
1003                 RTE_ETH_VLAN_EXTEND_MASK;
1004         err = ngbe_vlan_offload_config(dev, mask);
1005         if (err != 0) {
1006                 PMD_INIT_LOG(ERR, "Unable to set VLAN offload");
1007                 goto error;
1008         }
1009
1010         hw->mac.setup_pba(hw);
1011         ngbe_configure_port(dev);
1012
1013         err = ngbe_dev_rxtx_start(dev);
1014         if (err < 0) {
1015                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1016                 goto error;
1017         }
1018
1019         /* Skip link setup if loopback mode is enabled. */
1020         if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
1021                 goto skip_link_setup;
1022
1023         err = hw->mac.check_link(hw, &speed, &link_up, 0);
1024         if (err != 0)
1025                 goto error;
1026         dev->data->dev_link.link_status = link_up;
1027
1028         link_speeds = &dev->data->dev_conf.link_speeds;
1029         if (*link_speeds == RTE_ETH_LINK_SPEED_AUTONEG)
1030                 negotiate = true;
1031
1032         err = hw->mac.get_link_capabilities(hw, &speed, &negotiate);
1033         if (err != 0)
1034                 goto error;
1035
1036         allowed_speeds = 0;
1037         if (hw->mac.default_speeds & NGBE_LINK_SPEED_1GB_FULL)
1038                 allowed_speeds |= RTE_ETH_LINK_SPEED_1G;
1039         if (hw->mac.default_speeds & NGBE_LINK_SPEED_100M_FULL)
1040                 allowed_speeds |= RTE_ETH_LINK_SPEED_100M;
1041         if (hw->mac.default_speeds & NGBE_LINK_SPEED_10M_FULL)
1042                 allowed_speeds |= RTE_ETH_LINK_SPEED_10M;
1043
1044         if (*link_speeds & ~allowed_speeds) {
1045                 PMD_INIT_LOG(ERR, "Invalid link setting");
1046                 goto error;
1047         }
1048
1049         speed = 0x0;
1050         if (*link_speeds == RTE_ETH_LINK_SPEED_AUTONEG) {
1051                 speed = hw->mac.default_speeds;
1052         } else {
1053                 if (*link_speeds & RTE_ETH_LINK_SPEED_1G)
1054                         speed |= NGBE_LINK_SPEED_1GB_FULL;
1055                 if (*link_speeds & RTE_ETH_LINK_SPEED_100M)
1056                         speed |= NGBE_LINK_SPEED_100M_FULL;
1057                 if (*link_speeds & RTE_ETH_LINK_SPEED_10M)
1058                         speed |= NGBE_LINK_SPEED_10M_FULL;
1059         }
1060
1061         err = hw->phy.init_hw(hw);
1062         if (err != 0) {
1063                 PMD_INIT_LOG(ERR, "PHY init failed");
1064                 goto error;
1065         }
1066         err = hw->mac.setup_link(hw, speed, link_up);
1067         if (err != 0)
1068                 goto error;
1069
1070 skip_link_setup:
1071
1072         if (rte_intr_allow_others(intr_handle)) {
1073                 ngbe_dev_misc_interrupt_setup(dev);
1074                 /* check if lsc interrupt is enabled */
1075                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1076                         ngbe_dev_lsc_interrupt_setup(dev, TRUE);
1077                 else
1078                         ngbe_dev_lsc_interrupt_setup(dev, FALSE);
1079                 ngbe_dev_macsec_interrupt_setup(dev);
1080                 ngbe_set_ivar_map(hw, -1, 1, NGBE_MISC_VEC_ID);
1081         } else {
1082                 rte_intr_callback_unregister(intr_handle,
1083                                              ngbe_dev_interrupt_handler, dev);
1084                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1085                         PMD_INIT_LOG(INFO,
1086                                      "LSC won't enable because of no intr multiplex");
1087         }
1088
1089         /* check if rxq interrupt is enabled */
1090         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1091             rte_intr_dp_is_en(intr_handle))
1092                 ngbe_dev_rxq_interrupt_setup(dev);
1093
1094         /* enable UIO/VFIO intr/eventfd mapping */
1095         rte_intr_enable(intr_handle);
1096
1097         /* resume enabled intr since HW reset */
1098         ngbe_enable_intr(dev);
1099
1100         if ((hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_M88E1512_SFP ||
1101                 (hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_YT8521S_SFP) {
1102                 /* gpio0 is used to power on/off control*/
1103                 wr32(hw, NGBE_GPIODATA, 0);
1104         }
1105
1106         /*
1107          * Update link status right before return, because it may
1108          * start link configuration process in a separate thread.
1109          */
1110         ngbe_dev_link_update(dev, 0);
1111
1112         ngbe_read_stats_registers(hw, hw_stats);
1113         hw->offset_loaded = 1;
1114
1115         return 0;
1116
1117 error:
1118         PMD_INIT_LOG(ERR, "failure in dev start: %d", err);
1119         ngbe_dev_clear_queues(dev);
1120         return -EIO;
1121 }
1122
1123 /*
1124  * Stop device: disable rx and tx functions to allow for reconfiguring.
1125  */
1126 static int
1127 ngbe_dev_stop(struct rte_eth_dev *dev)
1128 {
1129         struct rte_eth_link link;
1130         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
1131         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1132         struct ngbe_vf_info *vfinfo = *NGBE_DEV_VFDATA(dev);
1133         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1134         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1135         int vf;
1136
1137         if (hw->adapter_stopped)
1138                 return 0;
1139
1140         PMD_INIT_FUNC_TRACE();
1141
1142         rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev);
1143
1144         if ((hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_M88E1512_SFP ||
1145                 (hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_YT8521S_SFP) {
1146                 /* gpio0 is used to power on/off control*/
1147                 wr32(hw, NGBE_GPIODATA, NGBE_GPIOBIT_0);
1148         }
1149
1150         /* disable interrupts */
1151         ngbe_disable_intr(hw);
1152
1153         /* reset the NIC */
1154         ngbe_pf_reset_hw(hw);
1155         hw->adapter_stopped = 0;
1156
1157         /* stop adapter */
1158         ngbe_stop_hw(hw);
1159
1160         for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++)
1161                 vfinfo[vf].clear_to_send = false;
1162
1163         ngbe_dev_clear_queues(dev);
1164
1165         /* Clear stored conf */
1166         dev->data->scattered_rx = 0;
1167
1168         /* Clear recorded link status */
1169         memset(&link, 0, sizeof(link));
1170         rte_eth_linkstatus_set(dev, &link);
1171
1172         if (!rte_intr_allow_others(intr_handle))
1173                 /* resume to the default handler */
1174                 rte_intr_callback_register(intr_handle,
1175                                            ngbe_dev_interrupt_handler,
1176                                            (void *)dev);
1177
1178         /* Clean datapath event and queue/vec mapping */
1179         rte_intr_efd_disable(intr_handle);
1180         rte_intr_vec_list_free(intr_handle);
1181
1182         ngbe_set_pcie_master(hw, true);
1183
1184         adapter->rss_reta_updated = 0;
1185
1186         hw->adapter_stopped = true;
1187         dev->data->dev_started = 0;
1188
1189         return 0;
1190 }
1191
1192 /*
1193  * Reset and stop device.
1194  */
1195 static int
1196 ngbe_dev_close(struct rte_eth_dev *dev)
1197 {
1198         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1199         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1200         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1201         int retries = 0;
1202         int ret;
1203
1204         PMD_INIT_FUNC_TRACE();
1205
1206         ngbe_pf_reset_hw(hw);
1207
1208         ngbe_dev_stop(dev);
1209
1210         ngbe_dev_free_queues(dev);
1211
1212         ngbe_set_pcie_master(hw, false);
1213
1214         /* reprogram the RAR[0] in case user changed it. */
1215         ngbe_set_rar(hw, 0, hw->mac.addr, 0, true);
1216
1217         /* Unlock any pending hardware semaphore */
1218         ngbe_swfw_lock_reset(hw);
1219
1220         /* disable uio intr before callback unregister */
1221         rte_intr_disable(intr_handle);
1222
1223         do {
1224                 ret = rte_intr_callback_unregister(intr_handle,
1225                                 ngbe_dev_interrupt_handler, dev);
1226                 if (ret >= 0 || ret == -ENOENT) {
1227                         break;
1228                 } else if (ret != -EAGAIN) {
1229                         PMD_INIT_LOG(ERR,
1230                                 "intr callback unregister failed: %d",
1231                                 ret);
1232                 }
1233                 rte_delay_ms(100);
1234         } while (retries++ < (10 + NGBE_LINK_UP_TIME));
1235
1236         /* uninitialize PF if max_vfs not zero */
1237         ngbe_pf_host_uninit(dev);
1238
1239         rte_free(dev->data->mac_addrs);
1240         dev->data->mac_addrs = NULL;
1241
1242         rte_free(dev->data->hash_mac_addrs);
1243         dev->data->hash_mac_addrs = NULL;
1244
1245         return ret;
1246 }
1247
1248 /*
1249  * Reset PF device.
1250  */
1251 static int
1252 ngbe_dev_reset(struct rte_eth_dev *dev)
1253 {
1254         int ret;
1255
1256         /* When a DPDK PMD PF begin to reset PF port, it should notify all
1257          * its VF to make them align with it. The detailed notification
1258          * mechanism is PMD specific. As to ngbe PF, it is rather complex.
1259          * To avoid unexpected behavior in VF, currently reset of PF with
1260          * SR-IOV activation is not supported. It might be supported later.
1261          */
1262         if (dev->data->sriov.active)
1263                 return -ENOTSUP;
1264
1265         ret = eth_ngbe_dev_uninit(dev);
1266         if (ret != 0)
1267                 return ret;
1268
1269         ret = eth_ngbe_dev_init(dev, NULL);
1270
1271         return ret;
1272 }
1273
1274 #define UPDATE_QP_COUNTER_32bit(reg, last_counter, counter)     \
1275         {                                                       \
1276                 uint32_t current_counter = rd32(hw, reg);       \
1277                 if (current_counter < last_counter)             \
1278                         current_counter += 0x100000000LL;       \
1279                 if (!hw->offset_loaded)                         \
1280                         last_counter = current_counter;         \
1281                 counter = current_counter - last_counter;       \
1282                 counter &= 0xFFFFFFFFLL;                        \
1283         }
1284
1285 #define UPDATE_QP_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
1286         {                                                                \
1287                 uint64_t current_counter_lsb = rd32(hw, reg_lsb);        \
1288                 uint64_t current_counter_msb = rd32(hw, reg_msb);        \
1289                 uint64_t current_counter = (current_counter_msb << 32) | \
1290                         current_counter_lsb;                             \
1291                 if (current_counter < last_counter)                      \
1292                         current_counter += 0x1000000000LL;               \
1293                 if (!hw->offset_loaded)                                  \
1294                         last_counter = current_counter;                  \
1295                 counter = current_counter - last_counter;                \
1296                 counter &= 0xFFFFFFFFFLL;                                \
1297         }
1298
1299 void
1300 ngbe_read_stats_registers(struct ngbe_hw *hw,
1301                            struct ngbe_hw_stats *hw_stats)
1302 {
1303         unsigned int i;
1304
1305         /* QP Stats */
1306         for (i = 0; i < hw->nb_rx_queues; i++) {
1307                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXPKT(i),
1308                         hw->qp_last[i].rx_qp_packets,
1309                         hw_stats->qp[i].rx_qp_packets);
1310                 UPDATE_QP_COUNTER_36bit(NGBE_QPRXOCTL(i), NGBE_QPRXOCTH(i),
1311                         hw->qp_last[i].rx_qp_bytes,
1312                         hw_stats->qp[i].rx_qp_bytes);
1313                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXMPKT(i),
1314                         hw->qp_last[i].rx_qp_mc_packets,
1315                         hw_stats->qp[i].rx_qp_mc_packets);
1316                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXBPKT(i),
1317                         hw->qp_last[i].rx_qp_bc_packets,
1318                         hw_stats->qp[i].rx_qp_bc_packets);
1319         }
1320
1321         for (i = 0; i < hw->nb_tx_queues; i++) {
1322                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXPKT(i),
1323                         hw->qp_last[i].tx_qp_packets,
1324                         hw_stats->qp[i].tx_qp_packets);
1325                 UPDATE_QP_COUNTER_36bit(NGBE_QPTXOCTL(i), NGBE_QPTXOCTH(i),
1326                         hw->qp_last[i].tx_qp_bytes,
1327                         hw_stats->qp[i].tx_qp_bytes);
1328                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXMPKT(i),
1329                         hw->qp_last[i].tx_qp_mc_packets,
1330                         hw_stats->qp[i].tx_qp_mc_packets);
1331                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXBPKT(i),
1332                         hw->qp_last[i].tx_qp_bc_packets,
1333                         hw_stats->qp[i].tx_qp_bc_packets);
1334         }
1335
1336         /* PB Stats */
1337         hw_stats->rx_up_dropped += rd32(hw, NGBE_PBRXMISS);
1338         hw_stats->rdb_pkt_cnt += rd32(hw, NGBE_PBRXPKT);
1339         hw_stats->rdb_repli_cnt += rd32(hw, NGBE_PBRXREP);
1340         hw_stats->rdb_drp_cnt += rd32(hw, NGBE_PBRXDROP);
1341         hw_stats->tx_xoff_packets += rd32(hw, NGBE_PBTXLNKXOFF);
1342         hw_stats->tx_xon_packets += rd32(hw, NGBE_PBTXLNKXON);
1343
1344         hw_stats->rx_xon_packets += rd32(hw, NGBE_PBRXLNKXON);
1345         hw_stats->rx_xoff_packets += rd32(hw, NGBE_PBRXLNKXOFF);
1346
1347         /* DMA Stats */
1348         hw_stats->rx_drop_packets += rd32(hw, NGBE_DMARXDROP);
1349         hw_stats->tx_drop_packets += rd32(hw, NGBE_DMATXDROP);
1350         hw_stats->rx_dma_drop += rd32(hw, NGBE_DMARXDROP);
1351         hw_stats->tx_secdrp_packets += rd32(hw, NGBE_DMATXSECDROP);
1352         hw_stats->rx_packets += rd32(hw, NGBE_DMARXPKT);
1353         hw_stats->tx_packets += rd32(hw, NGBE_DMATXPKT);
1354         hw_stats->rx_bytes += rd64(hw, NGBE_DMARXOCTL);
1355         hw_stats->tx_bytes += rd64(hw, NGBE_DMATXOCTL);
1356
1357         /* MAC Stats */
1358         hw_stats->rx_crc_errors += rd64(hw, NGBE_MACRXERRCRCL);
1359         hw_stats->rx_multicast_packets += rd64(hw, NGBE_MACRXMPKTL);
1360         hw_stats->tx_multicast_packets += rd64(hw, NGBE_MACTXMPKTL);
1361
1362         hw_stats->rx_total_packets += rd64(hw, NGBE_MACRXPKTL);
1363         hw_stats->tx_total_packets += rd64(hw, NGBE_MACTXPKTL);
1364         hw_stats->rx_total_bytes += rd64(hw, NGBE_MACRXGBOCTL);
1365
1366         hw_stats->rx_broadcast_packets += rd64(hw, NGBE_MACRXOCTL);
1367         hw_stats->tx_broadcast_packets += rd32(hw, NGBE_MACTXOCTL);
1368
1369         hw_stats->rx_size_64_packets += rd64(hw, NGBE_MACRX1TO64L);
1370         hw_stats->rx_size_65_to_127_packets += rd64(hw, NGBE_MACRX65TO127L);
1371         hw_stats->rx_size_128_to_255_packets += rd64(hw, NGBE_MACRX128TO255L);
1372         hw_stats->rx_size_256_to_511_packets += rd64(hw, NGBE_MACRX256TO511L);
1373         hw_stats->rx_size_512_to_1023_packets +=
1374                         rd64(hw, NGBE_MACRX512TO1023L);
1375         hw_stats->rx_size_1024_to_max_packets +=
1376                         rd64(hw, NGBE_MACRX1024TOMAXL);
1377         hw_stats->tx_size_64_packets += rd64(hw, NGBE_MACTX1TO64L);
1378         hw_stats->tx_size_65_to_127_packets += rd64(hw, NGBE_MACTX65TO127L);
1379         hw_stats->tx_size_128_to_255_packets += rd64(hw, NGBE_MACTX128TO255L);
1380         hw_stats->tx_size_256_to_511_packets += rd64(hw, NGBE_MACTX256TO511L);
1381         hw_stats->tx_size_512_to_1023_packets +=
1382                         rd64(hw, NGBE_MACTX512TO1023L);
1383         hw_stats->tx_size_1024_to_max_packets +=
1384                         rd64(hw, NGBE_MACTX1024TOMAXL);
1385
1386         hw_stats->rx_undersize_errors += rd64(hw, NGBE_MACRXERRLENL);
1387         hw_stats->rx_oversize_errors += rd32(hw, NGBE_MACRXOVERSIZE);
1388         hw_stats->rx_jabber_errors += rd32(hw, NGBE_MACRXJABBER);
1389
1390         /* MNG Stats */
1391         hw_stats->mng_bmc2host_packets = rd32(hw, NGBE_MNGBMC2OS);
1392         hw_stats->mng_host2bmc_packets = rd32(hw, NGBE_MNGOS2BMC);
1393         hw_stats->rx_management_packets = rd32(hw, NGBE_DMARXMNG);
1394         hw_stats->tx_management_packets = rd32(hw, NGBE_DMATXMNG);
1395
1396         /* MACsec Stats */
1397         hw_stats->tx_macsec_pkts_untagged += rd32(hw, NGBE_LSECTX_UTPKT);
1398         hw_stats->tx_macsec_pkts_encrypted +=
1399                         rd32(hw, NGBE_LSECTX_ENCPKT);
1400         hw_stats->tx_macsec_pkts_protected +=
1401                         rd32(hw, NGBE_LSECTX_PROTPKT);
1402         hw_stats->tx_macsec_octets_encrypted +=
1403                         rd32(hw, NGBE_LSECTX_ENCOCT);
1404         hw_stats->tx_macsec_octets_protected +=
1405                         rd32(hw, NGBE_LSECTX_PROTOCT);
1406         hw_stats->rx_macsec_pkts_untagged += rd32(hw, NGBE_LSECRX_UTPKT);
1407         hw_stats->rx_macsec_pkts_badtag += rd32(hw, NGBE_LSECRX_BTPKT);
1408         hw_stats->rx_macsec_pkts_nosci += rd32(hw, NGBE_LSECRX_NOSCIPKT);
1409         hw_stats->rx_macsec_pkts_unknownsci += rd32(hw, NGBE_LSECRX_UNSCIPKT);
1410         hw_stats->rx_macsec_octets_decrypted += rd32(hw, NGBE_LSECRX_DECOCT);
1411         hw_stats->rx_macsec_octets_validated += rd32(hw, NGBE_LSECRX_VLDOCT);
1412         hw_stats->rx_macsec_sc_pkts_unchecked +=
1413                         rd32(hw, NGBE_LSECRX_UNCHKPKT);
1414         hw_stats->rx_macsec_sc_pkts_delayed += rd32(hw, NGBE_LSECRX_DLYPKT);
1415         hw_stats->rx_macsec_sc_pkts_late += rd32(hw, NGBE_LSECRX_LATEPKT);
1416         for (i = 0; i < 2; i++) {
1417                 hw_stats->rx_macsec_sa_pkts_ok +=
1418                         rd32(hw, NGBE_LSECRX_OKPKT(i));
1419                 hw_stats->rx_macsec_sa_pkts_invalid +=
1420                         rd32(hw, NGBE_LSECRX_INVPKT(i));
1421                 hw_stats->rx_macsec_sa_pkts_notvalid +=
1422                         rd32(hw, NGBE_LSECRX_BADPKT(i));
1423         }
1424         for (i = 0; i < 4; i++) {
1425                 hw_stats->rx_macsec_sa_pkts_unusedsa +=
1426                         rd32(hw, NGBE_LSECRX_INVSAPKT(i));
1427                 hw_stats->rx_macsec_sa_pkts_notusingsa +=
1428                         rd32(hw, NGBE_LSECRX_BADSAPKT(i));
1429         }
1430         hw_stats->rx_total_missed_packets =
1431                         hw_stats->rx_up_dropped;
1432 }
1433
1434 static int
1435 ngbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1436 {
1437         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1438         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1439         struct ngbe_stat_mappings *stat_mappings =
1440                         NGBE_DEV_STAT_MAPPINGS(dev);
1441         uint32_t i, j;
1442
1443         ngbe_read_stats_registers(hw, hw_stats);
1444
1445         if (stats == NULL)
1446                 return -EINVAL;
1447
1448         /* Fill out the rte_eth_stats statistics structure */
1449         stats->ipackets = hw_stats->rx_packets;
1450         stats->ibytes = hw_stats->rx_bytes;
1451         stats->opackets = hw_stats->tx_packets;
1452         stats->obytes = hw_stats->tx_bytes;
1453
1454         memset(&stats->q_ipackets, 0, sizeof(stats->q_ipackets));
1455         memset(&stats->q_opackets, 0, sizeof(stats->q_opackets));
1456         memset(&stats->q_ibytes, 0, sizeof(stats->q_ibytes));
1457         memset(&stats->q_obytes, 0, sizeof(stats->q_obytes));
1458         memset(&stats->q_errors, 0, sizeof(stats->q_errors));
1459         for (i = 0; i < NGBE_MAX_QP; i++) {
1460                 uint32_t n = i / NB_QMAP_FIELDS_PER_QSM_REG;
1461                 uint32_t offset = (i % NB_QMAP_FIELDS_PER_QSM_REG) * 8;
1462                 uint32_t q_map;
1463
1464                 q_map = (stat_mappings->rqsm[n] >> offset)
1465                                 & QMAP_FIELD_RESERVED_BITS_MASK;
1466                 j = (q_map < RTE_ETHDEV_QUEUE_STAT_CNTRS
1467                      ? q_map : q_map % RTE_ETHDEV_QUEUE_STAT_CNTRS);
1468                 stats->q_ipackets[j] += hw_stats->qp[i].rx_qp_packets;
1469                 stats->q_ibytes[j] += hw_stats->qp[i].rx_qp_bytes;
1470
1471                 q_map = (stat_mappings->tqsm[n] >> offset)
1472                                 & QMAP_FIELD_RESERVED_BITS_MASK;
1473                 j = (q_map < RTE_ETHDEV_QUEUE_STAT_CNTRS
1474                      ? q_map : q_map % RTE_ETHDEV_QUEUE_STAT_CNTRS);
1475                 stats->q_opackets[j] += hw_stats->qp[i].tx_qp_packets;
1476                 stats->q_obytes[j] += hw_stats->qp[i].tx_qp_bytes;
1477         }
1478
1479         /* Rx Errors */
1480         stats->imissed  = hw_stats->rx_total_missed_packets +
1481                           hw_stats->rx_dma_drop;
1482         stats->ierrors  = hw_stats->rx_crc_errors +
1483                           hw_stats->rx_mac_short_packet_dropped +
1484                           hw_stats->rx_length_errors +
1485                           hw_stats->rx_undersize_errors +
1486                           hw_stats->rx_oversize_errors +
1487                           hw_stats->rx_illegal_byte_errors +
1488                           hw_stats->rx_error_bytes +
1489                           hw_stats->rx_fragment_errors;
1490
1491         /* Tx Errors */
1492         stats->oerrors  = 0;
1493         return 0;
1494 }
1495
1496 static int
1497 ngbe_dev_stats_reset(struct rte_eth_dev *dev)
1498 {
1499         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1500         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1501
1502         /* HW registers are cleared on read */
1503         hw->offset_loaded = 0;
1504         ngbe_dev_stats_get(dev, NULL);
1505         hw->offset_loaded = 1;
1506
1507         /* Reset software totals */
1508         memset(hw_stats, 0, sizeof(*hw_stats));
1509
1510         return 0;
1511 }
1512
1513 /* This function calculates the number of xstats based on the current config */
1514 static unsigned
1515 ngbe_xstats_calc_num(struct rte_eth_dev *dev)
1516 {
1517         int nb_queues = max(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
1518         return NGBE_NB_HW_STATS +
1519                NGBE_NB_QP_STATS * nb_queues;
1520 }
1521
1522 static inline int
1523 ngbe_get_name_by_id(uint32_t id, char *name, uint32_t size)
1524 {
1525         int nb, st;
1526
1527         /* Extended stats from ngbe_hw_stats */
1528         if (id < NGBE_NB_HW_STATS) {
1529                 snprintf(name, size, "[hw]%s",
1530                         rte_ngbe_stats_strings[id].name);
1531                 return 0;
1532         }
1533         id -= NGBE_NB_HW_STATS;
1534
1535         /* Queue Stats */
1536         if (id < NGBE_NB_QP_STATS * NGBE_MAX_QP) {
1537                 nb = id / NGBE_NB_QP_STATS;
1538                 st = id % NGBE_NB_QP_STATS;
1539                 snprintf(name, size, "[q%u]%s", nb,
1540                         rte_ngbe_qp_strings[st].name);
1541                 return 0;
1542         }
1543         id -= NGBE_NB_QP_STATS * NGBE_MAX_QP;
1544
1545         return -(int)(id + 1);
1546 }
1547
1548 static inline int
1549 ngbe_get_offset_by_id(uint32_t id, uint32_t *offset)
1550 {
1551         int nb, st;
1552
1553         /* Extended stats from ngbe_hw_stats */
1554         if (id < NGBE_NB_HW_STATS) {
1555                 *offset = rte_ngbe_stats_strings[id].offset;
1556                 return 0;
1557         }
1558         id -= NGBE_NB_HW_STATS;
1559
1560         /* Queue Stats */
1561         if (id < NGBE_NB_QP_STATS * NGBE_MAX_QP) {
1562                 nb = id / NGBE_NB_QP_STATS;
1563                 st = id % NGBE_NB_QP_STATS;
1564                 *offset = rte_ngbe_qp_strings[st].offset +
1565                         nb * (NGBE_NB_QP_STATS * sizeof(uint64_t));
1566                 return 0;
1567         }
1568
1569         return -1;
1570 }
1571
1572 static int ngbe_dev_xstats_get_names(struct rte_eth_dev *dev,
1573         struct rte_eth_xstat_name *xstats_names, unsigned int limit)
1574 {
1575         unsigned int i, count;
1576
1577         count = ngbe_xstats_calc_num(dev);
1578         if (xstats_names == NULL)
1579                 return count;
1580
1581         /* Note: limit >= cnt_stats checked upstream
1582          * in rte_eth_xstats_names()
1583          */
1584         limit = min(limit, count);
1585
1586         /* Extended stats from ngbe_hw_stats */
1587         for (i = 0; i < limit; i++) {
1588                 if (ngbe_get_name_by_id(i, xstats_names[i].name,
1589                         sizeof(xstats_names[i].name))) {
1590                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1591                         break;
1592                 }
1593         }
1594
1595         return i;
1596 }
1597
1598 static int ngbe_dev_xstats_get_names_by_id(struct rte_eth_dev *dev,
1599         const uint64_t *ids,
1600         struct rte_eth_xstat_name *xstats_names,
1601         unsigned int limit)
1602 {
1603         unsigned int i;
1604
1605         if (ids == NULL)
1606                 return ngbe_dev_xstats_get_names(dev, xstats_names, limit);
1607
1608         for (i = 0; i < limit; i++) {
1609                 if (ngbe_get_name_by_id(ids[i], xstats_names[i].name,
1610                                 sizeof(xstats_names[i].name))) {
1611                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1612                         return -1;
1613                 }
1614         }
1615
1616         return i;
1617 }
1618
1619 static int
1620 ngbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1621                                          unsigned int limit)
1622 {
1623         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1624         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1625         unsigned int i, count;
1626
1627         ngbe_read_stats_registers(hw, hw_stats);
1628
1629         /* If this is a reset xstats is NULL, and we have cleared the
1630          * registers by reading them.
1631          */
1632         count = ngbe_xstats_calc_num(dev);
1633         if (xstats == NULL)
1634                 return count;
1635
1636         limit = min(limit, ngbe_xstats_calc_num(dev));
1637
1638         /* Extended stats from ngbe_hw_stats */
1639         for (i = 0; i < limit; i++) {
1640                 uint32_t offset = 0;
1641
1642                 if (ngbe_get_offset_by_id(i, &offset)) {
1643                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1644                         break;
1645                 }
1646                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + offset);
1647                 xstats[i].id = i;
1648         }
1649
1650         return i;
1651 }
1652
1653 static int
1654 ngbe_dev_xstats_get_(struct rte_eth_dev *dev, uint64_t *values,
1655                                          unsigned int limit)
1656 {
1657         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1658         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1659         unsigned int i, count;
1660
1661         ngbe_read_stats_registers(hw, hw_stats);
1662
1663         /* If this is a reset xstats is NULL, and we have cleared the
1664          * registers by reading them.
1665          */
1666         count = ngbe_xstats_calc_num(dev);
1667         if (values == NULL)
1668                 return count;
1669
1670         limit = min(limit, ngbe_xstats_calc_num(dev));
1671
1672         /* Extended stats from ngbe_hw_stats */
1673         for (i = 0; i < limit; i++) {
1674                 uint32_t offset;
1675
1676                 if (ngbe_get_offset_by_id(i, &offset)) {
1677                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1678                         break;
1679                 }
1680                 values[i] = *(uint64_t *)(((char *)hw_stats) + offset);
1681         }
1682
1683         return i;
1684 }
1685
1686 static int
1687 ngbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1688                 uint64_t *values, unsigned int limit)
1689 {
1690         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1691         unsigned int i;
1692
1693         if (ids == NULL)
1694                 return ngbe_dev_xstats_get_(dev, values, limit);
1695
1696         for (i = 0; i < limit; i++) {
1697                 uint32_t offset;
1698
1699                 if (ngbe_get_offset_by_id(ids[i], &offset)) {
1700                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1701                         break;
1702                 }
1703                 values[i] = *(uint64_t *)(((char *)hw_stats) + offset);
1704         }
1705
1706         return i;
1707 }
1708
1709 static int
1710 ngbe_dev_xstats_reset(struct rte_eth_dev *dev)
1711 {
1712         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1713         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1714
1715         /* HW registers are cleared on read */
1716         hw->offset_loaded = 0;
1717         ngbe_read_stats_registers(hw, hw_stats);
1718         hw->offset_loaded = 1;
1719
1720         /* Reset software totals */
1721         memset(hw_stats, 0, sizeof(*hw_stats));
1722
1723         return 0;
1724 }
1725
1726 static int
1727 ngbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
1728 {
1729         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1730         int ret;
1731
1732         ret = snprintf(fw_version, fw_size, "0x%08x", hw->eeprom_id);
1733
1734         if (ret < 0)
1735                 return -EINVAL;
1736
1737         ret += 1; /* add the size of '\0' */
1738         if (fw_size < (size_t)ret)
1739                 return ret;
1740
1741         return 0;
1742 }
1743
1744 static int
1745 ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1746 {
1747         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1748         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1749
1750         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1751         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1752         dev_info->min_rx_bufsize = 1024;
1753         dev_info->max_rx_pktlen = 15872;
1754         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1755         dev_info->max_hash_mac_addrs = NGBE_VMDQ_NUM_UC_MAC;
1756         dev_info->max_vfs = pci_dev->max_vfs;
1757         dev_info->rx_queue_offload_capa = ngbe_get_rx_queue_offloads(dev);
1758         dev_info->rx_offload_capa = (ngbe_get_rx_port_offloads(dev) |
1759                                      dev_info->rx_queue_offload_capa);
1760         dev_info->tx_queue_offload_capa = 0;
1761         dev_info->tx_offload_capa = ngbe_get_tx_port_offloads(dev);
1762
1763         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1764                 .rx_thresh = {
1765                         .pthresh = NGBE_DEFAULT_RX_PTHRESH,
1766                         .hthresh = NGBE_DEFAULT_RX_HTHRESH,
1767                         .wthresh = NGBE_DEFAULT_RX_WTHRESH,
1768                 },
1769                 .rx_free_thresh = NGBE_DEFAULT_RX_FREE_THRESH,
1770                 .rx_drop_en = 0,
1771                 .offloads = 0,
1772         };
1773
1774         dev_info->default_txconf = (struct rte_eth_txconf) {
1775                 .tx_thresh = {
1776                         .pthresh = NGBE_DEFAULT_TX_PTHRESH,
1777                         .hthresh = NGBE_DEFAULT_TX_HTHRESH,
1778                         .wthresh = NGBE_DEFAULT_TX_WTHRESH,
1779                 },
1780                 .tx_free_thresh = NGBE_DEFAULT_TX_FREE_THRESH,
1781                 .offloads = 0,
1782         };
1783
1784         dev_info->rx_desc_lim = rx_desc_lim;
1785         dev_info->tx_desc_lim = tx_desc_lim;
1786
1787         dev_info->hash_key_size = NGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
1788         dev_info->reta_size = RTE_ETH_RSS_RETA_SIZE_128;
1789         dev_info->flow_type_rss_offloads = NGBE_RSS_OFFLOAD_ALL;
1790
1791         dev_info->speed_capa = RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_100M |
1792                                 RTE_ETH_LINK_SPEED_10M;
1793
1794         /* Driver-preferred Rx/Tx parameters */
1795         dev_info->default_rxportconf.burst_size = 32;
1796         dev_info->default_txportconf.burst_size = 32;
1797         dev_info->default_rxportconf.nb_queues = 1;
1798         dev_info->default_txportconf.nb_queues = 1;
1799         dev_info->default_rxportconf.ring_size = 256;
1800         dev_info->default_txportconf.ring_size = 256;
1801
1802         return 0;
1803 }
1804
1805 const uint32_t *
1806 ngbe_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1807 {
1808         if (dev->rx_pkt_burst == ngbe_recv_pkts ||
1809             dev->rx_pkt_burst == ngbe_recv_pkts_sc_single_alloc ||
1810             dev->rx_pkt_burst == ngbe_recv_pkts_sc_bulk_alloc ||
1811             dev->rx_pkt_burst == ngbe_recv_pkts_bulk_alloc)
1812                 return ngbe_get_supported_ptypes();
1813
1814         return NULL;
1815 }
1816
1817 void
1818 ngbe_dev_setup_link_alarm_handler(void *param)
1819 {
1820         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1821         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1822         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1823         u32 speed;
1824         bool autoneg = false;
1825
1826         speed = hw->phy.autoneg_advertised;
1827         if (!speed)
1828                 hw->mac.get_link_capabilities(hw, &speed, &autoneg);
1829
1830         hw->mac.setup_link(hw, speed, true);
1831
1832         intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
1833 }
1834
1835 /* return 0 means link status changed, -1 means not changed */
1836 int
1837 ngbe_dev_link_update_share(struct rte_eth_dev *dev,
1838                             int wait_to_complete)
1839 {
1840         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1841         struct rte_eth_link link;
1842         u32 link_speed = NGBE_LINK_SPEED_UNKNOWN;
1843         u32 lan_speed = 0;
1844         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1845         bool link_up;
1846         int err;
1847         int wait = 1;
1848
1849         memset(&link, 0, sizeof(link));
1850         link.link_status = RTE_ETH_LINK_DOWN;
1851         link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1852         link.link_duplex = RTE_ETH_LINK_HALF_DUPLEX;
1853         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
1854                         ~RTE_ETH_LINK_SPEED_AUTONEG);
1855
1856         hw->mac.get_link_status = true;
1857
1858         if (intr->flags & NGBE_FLAG_NEED_LINK_CONFIG)
1859                 return rte_eth_linkstatus_set(dev, &link);
1860
1861         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1862         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1863                 wait = 0;
1864
1865         err = hw->mac.check_link(hw, &link_speed, &link_up, wait);
1866         if (err != 0) {
1867                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1868                 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1869                 return rte_eth_linkstatus_set(dev, &link);
1870         }
1871
1872         if (!link_up) {
1873                 if (hw->phy.media_type == ngbe_media_type_fiber &&
1874                         hw->phy.type != ngbe_phy_mvl_sfi) {
1875                         intr->flags |= NGBE_FLAG_NEED_LINK_CONFIG;
1876                         rte_eal_alarm_set(10,
1877                                 ngbe_dev_setup_link_alarm_handler, dev);
1878                 }
1879
1880                 return rte_eth_linkstatus_set(dev, &link);
1881         }
1882
1883         intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
1884         link.link_status = RTE_ETH_LINK_UP;
1885         link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1886
1887         switch (link_speed) {
1888         default:
1889         case NGBE_LINK_SPEED_UNKNOWN:
1890                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1891                 break;
1892
1893         case NGBE_LINK_SPEED_10M_FULL:
1894                 link.link_speed = RTE_ETH_SPEED_NUM_10M;
1895                 lan_speed = 0;
1896                 break;
1897
1898         case NGBE_LINK_SPEED_100M_FULL:
1899                 link.link_speed = RTE_ETH_SPEED_NUM_100M;
1900                 lan_speed = 1;
1901                 break;
1902
1903         case NGBE_LINK_SPEED_1GB_FULL:
1904                 link.link_speed = RTE_ETH_SPEED_NUM_1G;
1905                 lan_speed = 2;
1906                 break;
1907         }
1908
1909         if (hw->is_pf) {
1910                 wr32m(hw, NGBE_LAN_SPEED, NGBE_LAN_SPEED_MASK, lan_speed);
1911                 if (link_speed & (NGBE_LINK_SPEED_1GB_FULL |
1912                                 NGBE_LINK_SPEED_100M_FULL |
1913                                 NGBE_LINK_SPEED_10M_FULL)) {
1914                         wr32m(hw, NGBE_MACTXCFG, NGBE_MACTXCFG_SPEED_MASK,
1915                                 NGBE_MACTXCFG_SPEED_1G | NGBE_MACTXCFG_TE);
1916                 }
1917         }
1918
1919         return rte_eth_linkstatus_set(dev, &link);
1920 }
1921
1922 static int
1923 ngbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1924 {
1925         return ngbe_dev_link_update_share(dev, wait_to_complete);
1926 }
1927
1928 static int
1929 ngbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1930 {
1931         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1932         uint32_t fctrl;
1933
1934         fctrl = rd32(hw, NGBE_PSRCTL);
1935         fctrl |= (NGBE_PSRCTL_UCP | NGBE_PSRCTL_MCP);
1936         wr32(hw, NGBE_PSRCTL, fctrl);
1937
1938         return 0;
1939 }
1940
1941 static int
1942 ngbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1943 {
1944         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1945         uint32_t fctrl;
1946
1947         fctrl = rd32(hw, NGBE_PSRCTL);
1948         fctrl &= (~NGBE_PSRCTL_UCP);
1949         if (dev->data->all_multicast == 1)
1950                 fctrl |= NGBE_PSRCTL_MCP;
1951         else
1952                 fctrl &= (~NGBE_PSRCTL_MCP);
1953         wr32(hw, NGBE_PSRCTL, fctrl);
1954
1955         return 0;
1956 }
1957
1958 static int
1959 ngbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1960 {
1961         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1962         uint32_t fctrl;
1963
1964         fctrl = rd32(hw, NGBE_PSRCTL);
1965         fctrl |= NGBE_PSRCTL_MCP;
1966         wr32(hw, NGBE_PSRCTL, fctrl);
1967
1968         return 0;
1969 }
1970
1971 static int
1972 ngbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1973 {
1974         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1975         uint32_t fctrl;
1976
1977         if (dev->data->promiscuous == 1)
1978                 return 0; /* must remain in all_multicast mode */
1979
1980         fctrl = rd32(hw, NGBE_PSRCTL);
1981         fctrl &= (~NGBE_PSRCTL_MCP);
1982         wr32(hw, NGBE_PSRCTL, fctrl);
1983
1984         return 0;
1985 }
1986
1987 /**
1988  * It clears the interrupt causes and enables the interrupt.
1989  * It will be called once only during NIC initialized.
1990  *
1991  * @param dev
1992  *  Pointer to struct rte_eth_dev.
1993  * @param on
1994  *  Enable or Disable.
1995  *
1996  * @return
1997  *  - On success, zero.
1998  *  - On failure, a negative value.
1999  */
2000 static int
2001 ngbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
2002 {
2003         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2004
2005         ngbe_dev_link_status_print(dev);
2006         if (on != 0) {
2007                 intr->mask_misc |= NGBE_ICRMISC_PHY;
2008                 intr->mask_misc |= NGBE_ICRMISC_GPIO;
2009         } else {
2010                 intr->mask_misc &= ~NGBE_ICRMISC_PHY;
2011                 intr->mask_misc &= ~NGBE_ICRMISC_GPIO;
2012         }
2013
2014         return 0;
2015 }
2016
2017 /**
2018  * It clears the interrupt causes and enables the interrupt.
2019  * It will be called once only during NIC initialized.
2020  *
2021  * @param dev
2022  *  Pointer to struct rte_eth_dev.
2023  *
2024  * @return
2025  *  - On success, zero.
2026  *  - On failure, a negative value.
2027  */
2028 static int
2029 ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev)
2030 {
2031         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2032         u64 mask;
2033
2034         mask = NGBE_ICR_MASK;
2035         mask &= (1ULL << NGBE_MISC_VEC_ID);
2036         intr->mask |= mask;
2037         intr->mask_misc |= NGBE_ICRMISC_GPIO;
2038
2039         return 0;
2040 }
2041
2042 /**
2043  * It clears the interrupt causes and enables the interrupt.
2044  * It will be called once only during NIC initialized.
2045  *
2046  * @param dev
2047  *  Pointer to struct rte_eth_dev.
2048  *
2049  * @return
2050  *  - On success, zero.
2051  *  - On failure, a negative value.
2052  */
2053 static int
2054 ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2055 {
2056         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2057         u64 mask;
2058
2059         mask = NGBE_ICR_MASK;
2060         mask &= ~((1ULL << NGBE_RX_VEC_START) - 1);
2061         intr->mask |= mask;
2062
2063         return 0;
2064 }
2065
2066 /**
2067  * It clears the interrupt causes and enables the interrupt.
2068  * It will be called once only during NIC initialized.
2069  *
2070  * @param dev
2071  *  Pointer to struct rte_eth_dev.
2072  *
2073  * @return
2074  *  - On success, zero.
2075  *  - On failure, a negative value.
2076  */
2077 static int
2078 ngbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev)
2079 {
2080         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2081
2082         intr->mask_misc |= NGBE_ICRMISC_LNKSEC;
2083
2084         return 0;
2085 }
2086
2087 /*
2088  * It reads ICR and sets flag for the link_update.
2089  *
2090  * @param dev
2091  *  Pointer to struct rte_eth_dev.
2092  *
2093  * @return
2094  *  - On success, zero.
2095  *  - On failure, a negative value.
2096  */
2097 static int
2098 ngbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2099 {
2100         uint32_t eicr;
2101         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2102         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2103
2104         /* read-on-clear nic registers here */
2105         eicr = ((u32 *)hw->isb_mem)[NGBE_ISB_MISC];
2106         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
2107
2108         intr->flags = 0;
2109
2110         /* set flag for async link update */
2111         if (eicr & NGBE_ICRMISC_PHY)
2112                 intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
2113
2114         if (eicr & NGBE_ICRMISC_VFMBX)
2115                 intr->flags |= NGBE_FLAG_MAILBOX;
2116
2117         if (eicr & NGBE_ICRMISC_LNKSEC)
2118                 intr->flags |= NGBE_FLAG_MACSEC;
2119
2120         if (eicr & NGBE_ICRMISC_GPIO)
2121                 intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
2122
2123         ((u32 *)hw->isb_mem)[NGBE_ISB_MISC] = 0;
2124
2125         return 0;
2126 }
2127
2128 /**
2129  * It gets and then prints the link status.
2130  *
2131  * @param dev
2132  *  Pointer to struct rte_eth_dev.
2133  *
2134  * @return
2135  *  - On success, zero.
2136  *  - On failure, a negative value.
2137  */
2138 static void
2139 ngbe_dev_link_status_print(struct rte_eth_dev *dev)
2140 {
2141         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2142         struct rte_eth_link link;
2143
2144         rte_eth_linkstatus_get(dev, &link);
2145
2146         if (link.link_status == RTE_ETH_LINK_UP) {
2147                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2148                                         (int)(dev->data->port_id),
2149                                         (unsigned int)link.link_speed,
2150                         link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX ?
2151                                         "full-duplex" : "half-duplex");
2152         } else {
2153                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2154                                 (int)(dev->data->port_id));
2155         }
2156         PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
2157                                 pci_dev->addr.domain,
2158                                 pci_dev->addr.bus,
2159                                 pci_dev->addr.devid,
2160                                 pci_dev->addr.function);
2161 }
2162
2163 /*
2164  * It executes link_update after knowing an interrupt occurred.
2165  *
2166  * @param dev
2167  *  Pointer to struct rte_eth_dev.
2168  *
2169  * @return
2170  *  - On success, zero.
2171  *  - On failure, a negative value.
2172  */
2173 static int
2174 ngbe_dev_interrupt_action(struct rte_eth_dev *dev)
2175 {
2176         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2177
2178         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2179
2180         if (intr->flags & NGBE_FLAG_MAILBOX) {
2181                 ngbe_pf_mbx_process(dev);
2182                 intr->flags &= ~NGBE_FLAG_MAILBOX;
2183         }
2184
2185         if (intr->flags & NGBE_FLAG_NEED_LINK_UPDATE) {
2186                 struct rte_eth_link link;
2187
2188                 /*get the link status before link update, for predicting later*/
2189                 rte_eth_linkstatus_get(dev, &link);
2190
2191                 ngbe_dev_link_update(dev, 0);
2192                 intr->flags &= ~NGBE_FLAG_NEED_LINK_UPDATE;
2193                 ngbe_dev_link_status_print(dev);
2194                 if (dev->data->dev_link.link_speed != link.link_speed)
2195                         rte_eth_dev_callback_process(dev,
2196                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2197         }
2198
2199         PMD_DRV_LOG(DEBUG, "enable intr immediately");
2200         ngbe_enable_intr(dev);
2201
2202         return 0;
2203 }
2204
2205 /**
2206  * Interrupt handler triggered by NIC  for handling
2207  * specific interrupt.
2208  *
2209  * @param param
2210  *  The address of parameter (struct rte_eth_dev *) registered before.
2211  */
2212 static void
2213 ngbe_dev_interrupt_handler(void *param)
2214 {
2215         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2216
2217         ngbe_dev_interrupt_get_status(dev);
2218         ngbe_dev_interrupt_action(dev);
2219 }
2220
2221 static int
2222 ngbe_dev_led_on(struct rte_eth_dev *dev)
2223 {
2224         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2225         return hw->mac.led_on(hw, 0) == 0 ? 0 : -ENOTSUP;
2226 }
2227
2228 static int
2229 ngbe_dev_led_off(struct rte_eth_dev *dev)
2230 {
2231         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2232         return hw->mac.led_off(hw, 0) == 0 ? 0 : -ENOTSUP;
2233 }
2234
2235 static int
2236 ngbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2237 {
2238         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2239         uint32_t mflcn_reg;
2240         uint32_t fccfg_reg;
2241         int rx_pause;
2242         int tx_pause;
2243
2244         fc_conf->pause_time = hw->fc.pause_time;
2245         fc_conf->high_water = hw->fc.high_water;
2246         fc_conf->low_water = hw->fc.low_water;
2247         fc_conf->send_xon = hw->fc.send_xon;
2248         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2249
2250         /*
2251          * Return rx_pause status according to actual setting of
2252          * RXFCCFG register.
2253          */
2254         mflcn_reg = rd32(hw, NGBE_RXFCCFG);
2255         if (mflcn_reg & NGBE_RXFCCFG_FC)
2256                 rx_pause = 1;
2257         else
2258                 rx_pause = 0;
2259
2260         /*
2261          * Return tx_pause status according to actual setting of
2262          * TXFCCFG register.
2263          */
2264         fccfg_reg = rd32(hw, NGBE_TXFCCFG);
2265         if (fccfg_reg & NGBE_TXFCCFG_FC)
2266                 tx_pause = 1;
2267         else
2268                 tx_pause = 0;
2269
2270         if (rx_pause && tx_pause)
2271                 fc_conf->mode = RTE_ETH_FC_FULL;
2272         else if (rx_pause)
2273                 fc_conf->mode = RTE_ETH_FC_RX_PAUSE;
2274         else if (tx_pause)
2275                 fc_conf->mode = RTE_ETH_FC_TX_PAUSE;
2276         else
2277                 fc_conf->mode = RTE_ETH_FC_NONE;
2278
2279         return 0;
2280 }
2281
2282 static int
2283 ngbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2284 {
2285         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2286         int err;
2287         uint32_t rx_buf_size;
2288         uint32_t max_high_water;
2289         enum ngbe_fc_mode rte_fcmode_2_ngbe_fcmode[] = {
2290                 ngbe_fc_none,
2291                 ngbe_fc_rx_pause,
2292                 ngbe_fc_tx_pause,
2293                 ngbe_fc_full
2294         };
2295
2296         PMD_INIT_FUNC_TRACE();
2297
2298         rx_buf_size = rd32(hw, NGBE_PBRXSIZE);
2299         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2300
2301         /*
2302          * At least reserve one Ethernet frame for watermark
2303          * high_water/low_water in kilo bytes for ngbe
2304          */
2305         max_high_water = (rx_buf_size - RTE_ETHER_MAX_LEN) >> 10;
2306         if (fc_conf->high_water > max_high_water ||
2307             fc_conf->high_water < fc_conf->low_water) {
2308                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2309                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2310                 return -EINVAL;
2311         }
2312
2313         hw->fc.requested_mode = rte_fcmode_2_ngbe_fcmode[fc_conf->mode];
2314         hw->fc.pause_time     = fc_conf->pause_time;
2315         hw->fc.high_water     = fc_conf->high_water;
2316         hw->fc.low_water      = fc_conf->low_water;
2317         hw->fc.send_xon       = fc_conf->send_xon;
2318         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
2319
2320         err = hw->mac.fc_enable(hw);
2321
2322         /* Not negotiated is not an error case */
2323         if (err == 0 || err == NGBE_ERR_FC_NOT_NEGOTIATED) {
2324                 wr32m(hw, NGBE_MACRXFLT, NGBE_MACRXFLT_CTL_MASK,
2325                       (fc_conf->mac_ctrl_frame_fwd
2326                        ? NGBE_MACRXFLT_CTL_NOPS : NGBE_MACRXFLT_CTL_DROP));
2327                 ngbe_flush(hw);
2328
2329                 return 0;
2330         }
2331
2332         PMD_INIT_LOG(ERR, "ngbe_fc_enable = 0x%x", err);
2333         return -EIO;
2334 }
2335
2336 int
2337 ngbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2338                           struct rte_eth_rss_reta_entry64 *reta_conf,
2339                           uint16_t reta_size)
2340 {
2341         uint8_t i, j, mask;
2342         uint32_t reta;
2343         uint16_t idx, shift;
2344         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2345         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2346
2347         PMD_INIT_FUNC_TRACE();
2348
2349         if (!hw->is_pf) {
2350                 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
2351                         "NIC.");
2352                 return -ENOTSUP;
2353         }
2354
2355         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2356                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2357                         "(%d) doesn't match the number hardware can supported "
2358                         "(%d)", reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2359                 return -EINVAL;
2360         }
2361
2362         for (i = 0; i < reta_size; i += 4) {
2363                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2364                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2365                 mask = (uint8_t)RS64(reta_conf[idx].mask, shift, 0xF);
2366                 if (!mask)
2367                         continue;
2368
2369                 reta = rd32a(hw, NGBE_REG_RSSTBL, i >> 2);
2370                 for (j = 0; j < 4; j++) {
2371                         if (RS8(mask, j, 0x1)) {
2372                                 reta  &= ~(MS32(8 * j, 0xFF));
2373                                 reta |= LS32(reta_conf[idx].reta[shift + j],
2374                                                 8 * j, 0xFF);
2375                         }
2376                 }
2377                 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta);
2378         }
2379         adapter->rss_reta_updated = 1;
2380
2381         return 0;
2382 }
2383
2384 int
2385 ngbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2386                          struct rte_eth_rss_reta_entry64 *reta_conf,
2387                          uint16_t reta_size)
2388 {
2389         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2390         uint8_t i, j, mask;
2391         uint32_t reta;
2392         uint16_t idx, shift;
2393
2394         PMD_INIT_FUNC_TRACE();
2395
2396         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2397                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2398                         "(%d) doesn't match the number hardware can supported "
2399                         "(%d)", reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2400                 return -EINVAL;
2401         }
2402
2403         for (i = 0; i < reta_size; i += 4) {
2404                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2405                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2406                 mask = (uint8_t)RS64(reta_conf[idx].mask, shift, 0xF);
2407                 if (!mask)
2408                         continue;
2409
2410                 reta = rd32a(hw, NGBE_REG_RSSTBL, i >> 2);
2411                 for (j = 0; j < 4; j++) {
2412                         if (RS8(mask, j, 0x1))
2413                                 reta_conf[idx].reta[shift + j] =
2414                                         (uint16_t)RS32(reta, 8 * j, 0xFF);
2415                 }
2416         }
2417
2418         return 0;
2419 }
2420
2421 static int
2422 ngbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
2423                                 uint32_t index, uint32_t pool)
2424 {
2425         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2426         uint32_t enable_addr = 1;
2427
2428         return ngbe_set_rar(hw, index, mac_addr->addr_bytes,
2429                              pool, enable_addr);
2430 }
2431
2432 static void
2433 ngbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2434 {
2435         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2436
2437         ngbe_clear_rar(hw, index);
2438 }
2439
2440 static int
2441 ngbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
2442 {
2443         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2444
2445         ngbe_remove_rar(dev, 0);
2446         ngbe_add_rar(dev, addr, 0, pci_dev->max_vfs);
2447
2448         return 0;
2449 }
2450
2451 static int
2452 ngbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
2453 {
2454         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2455         uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 4;
2456         struct rte_eth_dev_data *dev_data = dev->data;
2457
2458         /* If device is started, refuse mtu that requires the support of
2459          * scattered packets when this feature has not been enabled before.
2460          */
2461         if (dev_data->dev_started && !dev_data->scattered_rx &&
2462             (frame_size + 2 * RTE_VLAN_HLEN >
2463              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
2464                 PMD_INIT_LOG(ERR, "Stop port first.");
2465                 return -EINVAL;
2466         }
2467
2468         if (hw->mode)
2469                 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2470                         NGBE_FRAME_SIZE_MAX);
2471         else
2472                 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2473                         NGBE_FRMSZ_MAX(frame_size));
2474
2475         return 0;
2476 }
2477
2478 static uint32_t
2479 ngbe_uta_vector(struct ngbe_hw *hw, struct rte_ether_addr *uc_addr)
2480 {
2481         uint32_t vector = 0;
2482
2483         switch (hw->mac.mc_filter_type) {
2484         case 0:   /* use bits [47:36] of the address */
2485                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2486                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2487                 break;
2488         case 1:   /* use bits [46:35] of the address */
2489                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2490                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2491                 break;
2492         case 2:   /* use bits [45:34] of the address */
2493                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2494                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2495                 break;
2496         case 3:   /* use bits [43:32] of the address */
2497                 vector = ((uc_addr->addr_bytes[4]) |
2498                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2499                 break;
2500         default:  /* Invalid mc_filter_type */
2501                 break;
2502         }
2503
2504         /* vector can only be 12-bits or boundary will be exceeded */
2505         vector &= 0xFFF;
2506         return vector;
2507 }
2508
2509 static int
2510 ngbe_uc_hash_table_set(struct rte_eth_dev *dev,
2511                         struct rte_ether_addr *mac_addr, uint8_t on)
2512 {
2513         uint32_t vector;
2514         uint32_t uta_idx;
2515         uint32_t reg_val;
2516         uint32_t uta_mask;
2517         uint32_t psrctl;
2518
2519         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2520         struct ngbe_uta_info *uta_info = NGBE_DEV_UTA_INFO(dev);
2521
2522         vector = ngbe_uta_vector(hw, mac_addr);
2523         uta_idx = (vector >> 5) & 0x7F;
2524         uta_mask = 0x1UL << (vector & 0x1F);
2525
2526         if (!!on == !!(uta_info->uta_shadow[uta_idx] & uta_mask))
2527                 return 0;
2528
2529         reg_val = rd32(hw, NGBE_UCADDRTBL(uta_idx));
2530         if (on) {
2531                 uta_info->uta_in_use++;
2532                 reg_val |= uta_mask;
2533                 uta_info->uta_shadow[uta_idx] |= uta_mask;
2534         } else {
2535                 uta_info->uta_in_use--;
2536                 reg_val &= ~uta_mask;
2537                 uta_info->uta_shadow[uta_idx] &= ~uta_mask;
2538         }
2539
2540         wr32(hw, NGBE_UCADDRTBL(uta_idx), reg_val);
2541
2542         psrctl = rd32(hw, NGBE_PSRCTL);
2543         if (uta_info->uta_in_use > 0)
2544                 psrctl |= NGBE_PSRCTL_UCHFENA;
2545         else
2546                 psrctl &= ~NGBE_PSRCTL_UCHFENA;
2547
2548         psrctl &= ~NGBE_PSRCTL_ADHF12_MASK;
2549         psrctl |= NGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
2550         wr32(hw, NGBE_PSRCTL, psrctl);
2551
2552         return 0;
2553 }
2554
2555 static int
2556 ngbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2557 {
2558         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2559         struct ngbe_uta_info *uta_info = NGBE_DEV_UTA_INFO(dev);
2560         uint32_t psrctl;
2561         int i;
2562
2563         if (on) {
2564                 for (i = 0; i < RTE_ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2565                         uta_info->uta_shadow[i] = ~0;
2566                         wr32(hw, NGBE_UCADDRTBL(i), ~0);
2567                 }
2568         } else {
2569                 for (i = 0; i < RTE_ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2570                         uta_info->uta_shadow[i] = 0;
2571                         wr32(hw, NGBE_UCADDRTBL(i), 0);
2572                 }
2573         }
2574
2575         psrctl = rd32(hw, NGBE_PSRCTL);
2576         if (on)
2577                 psrctl |= NGBE_PSRCTL_UCHFENA;
2578         else
2579                 psrctl &= ~NGBE_PSRCTL_UCHFENA;
2580
2581         psrctl &= ~NGBE_PSRCTL_ADHF12_MASK;
2582         psrctl |= NGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
2583         wr32(hw, NGBE_PSRCTL, psrctl);
2584
2585         return 0;
2586 }
2587
2588 /**
2589  * Set the IVAR registers, mapping interrupt causes to vectors
2590  * @param hw
2591  *  pointer to ngbe_hw struct
2592  * @direction
2593  *  0 for Rx, 1 for Tx, -1 for other causes
2594  * @queue
2595  *  queue to map the corresponding interrupt to
2596  * @msix_vector
2597  *  the vector to map to the corresponding queue
2598  */
2599 void
2600 ngbe_set_ivar_map(struct ngbe_hw *hw, int8_t direction,
2601                    uint8_t queue, uint8_t msix_vector)
2602 {
2603         uint32_t tmp, idx;
2604
2605         if (direction == -1) {
2606                 /* other causes */
2607                 msix_vector |= NGBE_IVARMISC_VLD;
2608                 idx = 0;
2609                 tmp = rd32(hw, NGBE_IVARMISC);
2610                 tmp &= ~(0xFF << idx);
2611                 tmp |= (msix_vector << idx);
2612                 wr32(hw, NGBE_IVARMISC, tmp);
2613         } else {
2614                 /* rx or tx causes */
2615                 /* Workaround for ICR lost */
2616                 idx = ((16 * (queue & 1)) + (8 * direction));
2617                 tmp = rd32(hw, NGBE_IVAR(queue >> 1));
2618                 tmp &= ~(0xFF << idx);
2619                 tmp |= (msix_vector << idx);
2620                 wr32(hw, NGBE_IVAR(queue >> 1), tmp);
2621         }
2622 }
2623
2624 /**
2625  * Sets up the hardware to properly generate MSI-X interrupts
2626  * @hw
2627  *  board private structure
2628  */
2629 static void
2630 ngbe_configure_msix(struct rte_eth_dev *dev)
2631 {
2632         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2633         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
2634         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2635         uint32_t queue_id, base = NGBE_MISC_VEC_ID;
2636         uint32_t vec = NGBE_MISC_VEC_ID;
2637         uint32_t gpie;
2638
2639         /*
2640          * Won't configure MSI-X register if no mapping is done
2641          * between intr vector and event fd
2642          * but if MSI-X has been enabled already, need to configure
2643          * auto clean, auto mask and throttling.
2644          */
2645         gpie = rd32(hw, NGBE_GPIE);
2646         if (!rte_intr_dp_is_en(intr_handle) &&
2647             !(gpie & NGBE_GPIE_MSIX))
2648                 return;
2649
2650         if (rte_intr_allow_others(intr_handle)) {
2651                 base = NGBE_RX_VEC_START;
2652                 vec = base;
2653         }
2654
2655         /* setup GPIE for MSI-X mode */
2656         gpie = rd32(hw, NGBE_GPIE);
2657         gpie |= NGBE_GPIE_MSIX;
2658         wr32(hw, NGBE_GPIE, gpie);
2659
2660         /* Populate the IVAR table and set the ITR values to the
2661          * corresponding register.
2662          */
2663         if (rte_intr_dp_is_en(intr_handle)) {
2664                 for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
2665                         queue_id++) {
2666                         /* by default, 1:1 mapping */
2667                         ngbe_set_ivar_map(hw, 0, queue_id, vec);
2668                         rte_intr_vec_list_index_set(intr_handle,
2669                                                            queue_id, vec);
2670                         if (vec < base + rte_intr_nb_efd_get(intr_handle)
2671                             - 1)
2672                                 vec++;
2673                 }
2674
2675                 ngbe_set_ivar_map(hw, -1, 1, NGBE_MISC_VEC_ID);
2676         }
2677         wr32(hw, NGBE_ITR(NGBE_MISC_VEC_ID),
2678                         NGBE_ITR_IVAL_1G(NGBE_QUEUE_ITR_INTERVAL_DEFAULT)
2679                         | NGBE_ITR_WRDSA);
2680 }
2681
2682 static u8 *
2683 ngbe_dev_addr_list_itr(__rte_unused struct ngbe_hw *hw,
2684                         u8 **mc_addr_ptr, u32 *vmdq)
2685 {
2686         u8 *mc_addr;
2687
2688         *vmdq = 0;
2689         mc_addr = *mc_addr_ptr;
2690         *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr));
2691         return mc_addr;
2692 }
2693
2694 int
2695 ngbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
2696                           struct rte_ether_addr *mc_addr_set,
2697                           uint32_t nb_mc_addr)
2698 {
2699         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2700         u8 *mc_addr_list;
2701
2702         mc_addr_list = (u8 *)mc_addr_set;
2703         return hw->mac.update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
2704                                          ngbe_dev_addr_list_itr, TRUE);
2705 }
2706
2707 static uint64_t
2708 ngbe_read_systime_cyclecounter(struct rte_eth_dev *dev)
2709 {
2710         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2711         uint64_t systime_cycles;
2712
2713         systime_cycles = (uint64_t)rd32(hw, NGBE_TSTIMEL);
2714         systime_cycles |= (uint64_t)rd32(hw, NGBE_TSTIMEH) << 32;
2715
2716         return systime_cycles;
2717 }
2718
2719 static uint64_t
2720 ngbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
2721 {
2722         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2723         uint64_t rx_tstamp_cycles;
2724
2725         /* TSRXSTMPL stores ns and TSRXSTMPH stores seconds. */
2726         rx_tstamp_cycles = (uint64_t)rd32(hw, NGBE_TSRXSTMPL);
2727         rx_tstamp_cycles |= (uint64_t)rd32(hw, NGBE_TSRXSTMPH) << 32;
2728
2729         return rx_tstamp_cycles;
2730 }
2731
2732 static uint64_t
2733 ngbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
2734 {
2735         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2736         uint64_t tx_tstamp_cycles;
2737
2738         /* TSTXSTMPL stores ns and TSTXSTMPH stores seconds. */
2739         tx_tstamp_cycles = (uint64_t)rd32(hw, NGBE_TSTXSTMPL);
2740         tx_tstamp_cycles |= (uint64_t)rd32(hw, NGBE_TSTXSTMPH) << 32;
2741
2742         return tx_tstamp_cycles;
2743 }
2744
2745 static void
2746 ngbe_start_timecounters(struct rte_eth_dev *dev)
2747 {
2748         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2749         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2750         uint32_t incval = 0;
2751         uint32_t shift = 0;
2752
2753         incval = NGBE_INCVAL_1GB;
2754         shift = NGBE_INCVAL_SHIFT_1GB;
2755
2756         wr32(hw, NGBE_TSTIMEINC, NGBE_TSTIMEINC_IV(incval));
2757
2758         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
2759         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
2760         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
2761
2762         adapter->systime_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2763         adapter->systime_tc.cc_shift = shift;
2764         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
2765
2766         adapter->rx_tstamp_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2767         adapter->rx_tstamp_tc.cc_shift = shift;
2768         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
2769
2770         adapter->tx_tstamp_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2771         adapter->tx_tstamp_tc.cc_shift = shift;
2772         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
2773 }
2774
2775 static int
2776 ngbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
2777 {
2778         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2779
2780         adapter->systime_tc.nsec += delta;
2781         adapter->rx_tstamp_tc.nsec += delta;
2782         adapter->tx_tstamp_tc.nsec += delta;
2783
2784         return 0;
2785 }
2786
2787 static int
2788 ngbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
2789 {
2790         uint64_t ns;
2791         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2792
2793         ns = rte_timespec_to_ns(ts);
2794         /* Set the timecounters to a new value. */
2795         adapter->systime_tc.nsec = ns;
2796         adapter->rx_tstamp_tc.nsec = ns;
2797         adapter->tx_tstamp_tc.nsec = ns;
2798
2799         return 0;
2800 }
2801
2802 static int
2803 ngbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
2804 {
2805         uint64_t ns, systime_cycles;
2806         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2807
2808         systime_cycles = ngbe_read_systime_cyclecounter(dev);
2809         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
2810         *ts = rte_ns_to_timespec(ns);
2811
2812         return 0;
2813 }
2814
2815 static int
2816 ngbe_timesync_enable(struct rte_eth_dev *dev)
2817 {
2818         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2819         uint32_t tsync_ctl;
2820
2821         /* Stop the timesync system time. */
2822         wr32(hw, NGBE_TSTIMEINC, 0x0);
2823         /* Reset the timesync system time value. */
2824         wr32(hw, NGBE_TSTIMEL, 0x0);
2825         wr32(hw, NGBE_TSTIMEH, 0x0);
2826
2827         ngbe_start_timecounters(dev);
2828
2829         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
2830         wr32(hw, NGBE_ETFLT(NGBE_ETF_ID_1588),
2831                 RTE_ETHER_TYPE_1588 | NGBE_ETFLT_ENA | NGBE_ETFLT_1588);
2832
2833         /* Enable timestamping of received PTP packets. */
2834         tsync_ctl = rd32(hw, NGBE_TSRXCTL);
2835         tsync_ctl |= NGBE_TSRXCTL_ENA;
2836         wr32(hw, NGBE_TSRXCTL, tsync_ctl);
2837
2838         /* Enable timestamping of transmitted PTP packets. */
2839         tsync_ctl = rd32(hw, NGBE_TSTXCTL);
2840         tsync_ctl |= NGBE_TSTXCTL_ENA;
2841         wr32(hw, NGBE_TSTXCTL, tsync_ctl);
2842
2843         ngbe_flush(hw);
2844
2845         return 0;
2846 }
2847
2848 static int
2849 ngbe_timesync_disable(struct rte_eth_dev *dev)
2850 {
2851         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2852         uint32_t tsync_ctl;
2853
2854         /* Disable timestamping of transmitted PTP packets. */
2855         tsync_ctl = rd32(hw, NGBE_TSTXCTL);
2856         tsync_ctl &= ~NGBE_TSTXCTL_ENA;
2857         wr32(hw, NGBE_TSTXCTL, tsync_ctl);
2858
2859         /* Disable timestamping of received PTP packets. */
2860         tsync_ctl = rd32(hw, NGBE_TSRXCTL);
2861         tsync_ctl &= ~NGBE_TSRXCTL_ENA;
2862         wr32(hw, NGBE_TSRXCTL, tsync_ctl);
2863
2864         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
2865         wr32(hw, NGBE_ETFLT(NGBE_ETF_ID_1588), 0);
2866
2867         /* Stop incrementing the System Time registers. */
2868         wr32(hw, NGBE_TSTIMEINC, 0);
2869
2870         return 0;
2871 }
2872
2873 static int
2874 ngbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
2875                                  struct timespec *timestamp,
2876                                  uint32_t flags __rte_unused)
2877 {
2878         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2879         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2880         uint32_t tsync_rxctl;
2881         uint64_t rx_tstamp_cycles;
2882         uint64_t ns;
2883
2884         tsync_rxctl = rd32(hw, NGBE_TSRXCTL);
2885         if ((tsync_rxctl & NGBE_TSRXCTL_VLD) == 0)
2886                 return -EINVAL;
2887
2888         rx_tstamp_cycles = ngbe_read_rx_tstamp_cyclecounter(dev);
2889         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
2890         *timestamp = rte_ns_to_timespec(ns);
2891
2892         return  0;
2893 }
2894
2895 static int
2896 ngbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
2897                                  struct timespec *timestamp)
2898 {
2899         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2900         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2901         uint32_t tsync_txctl;
2902         uint64_t tx_tstamp_cycles;
2903         uint64_t ns;
2904
2905         tsync_txctl = rd32(hw, NGBE_TSTXCTL);
2906         if ((tsync_txctl & NGBE_TSTXCTL_VLD) == 0)
2907                 return -EINVAL;
2908
2909         tx_tstamp_cycles = ngbe_read_tx_tstamp_cyclecounter(dev);
2910         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
2911         *timestamp = rte_ns_to_timespec(ns);
2912
2913         return 0;
2914 }
2915
2916 static int
2917 ngbe_get_reg_length(struct rte_eth_dev *dev __rte_unused)
2918 {
2919         int count = 0;
2920         int g_ind = 0;
2921         const struct reg_info *reg_group;
2922         const struct reg_info **reg_set = ngbe_regs_others;
2923
2924         while ((reg_group = reg_set[g_ind++]))
2925                 count += ngbe_regs_group_count(reg_group);
2926
2927         return count;
2928 }
2929
2930 static int
2931 ngbe_get_regs(struct rte_eth_dev *dev,
2932               struct rte_dev_reg_info *regs)
2933 {
2934         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2935         uint32_t *data = regs->data;
2936         int g_ind = 0;
2937         int count = 0;
2938         const struct reg_info *reg_group;
2939         const struct reg_info **reg_set = ngbe_regs_others;
2940
2941         if (data == NULL) {
2942                 regs->length = ngbe_get_reg_length(dev);
2943                 regs->width = sizeof(uint32_t);
2944                 return 0;
2945         }
2946
2947         /* Support only full register dump */
2948         if (regs->length == 0 ||
2949             regs->length == (uint32_t)ngbe_get_reg_length(dev)) {
2950                 regs->version = hw->mac.type << 24 |
2951                                 hw->revision_id << 16 |
2952                                 hw->device_id;
2953                 while ((reg_group = reg_set[g_ind++]))
2954                         count += ngbe_read_regs_group(dev, &data[count],
2955                                                       reg_group);
2956                 return 0;
2957         }
2958
2959         return -ENOTSUP;
2960 }
2961
2962 static int
2963 ngbe_get_eeprom_length(struct rte_eth_dev *dev)
2964 {
2965         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2966
2967         /* Return unit is byte count */
2968         return hw->rom.word_size * 2;
2969 }
2970
2971 static int
2972 ngbe_get_eeprom(struct rte_eth_dev *dev,
2973                 struct rte_dev_eeprom_info *in_eeprom)
2974 {
2975         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2976         struct ngbe_rom_info *eeprom = &hw->rom;
2977         uint16_t *data = in_eeprom->data;
2978         int first, length;
2979
2980         first = in_eeprom->offset >> 1;
2981         length = in_eeprom->length >> 1;
2982         if (first > hw->rom.word_size ||
2983             ((first + length) > hw->rom.word_size))
2984                 return -EINVAL;
2985
2986         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
2987
2988         return eeprom->readw_buffer(hw, first, length, data);
2989 }
2990
2991 static int
2992 ngbe_set_eeprom(struct rte_eth_dev *dev,
2993                 struct rte_dev_eeprom_info *in_eeprom)
2994 {
2995         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2996         struct ngbe_rom_info *eeprom = &hw->rom;
2997         uint16_t *data = in_eeprom->data;
2998         int first, length;
2999
3000         first = in_eeprom->offset >> 1;
3001         length = in_eeprom->length >> 1;
3002         if (first > hw->rom.word_size ||
3003             ((first + length) > hw->rom.word_size))
3004                 return -EINVAL;
3005
3006         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
3007
3008         return eeprom->writew_buffer(hw,  first, length, data);
3009 }
3010
3011 static const struct eth_dev_ops ngbe_eth_dev_ops = {
3012         .dev_configure              = ngbe_dev_configure,
3013         .dev_infos_get              = ngbe_dev_info_get,
3014         .dev_start                  = ngbe_dev_start,
3015         .dev_stop                   = ngbe_dev_stop,
3016         .dev_close                  = ngbe_dev_close,
3017         .dev_reset                  = ngbe_dev_reset,
3018         .promiscuous_enable         = ngbe_dev_promiscuous_enable,
3019         .promiscuous_disable        = ngbe_dev_promiscuous_disable,
3020         .allmulticast_enable        = ngbe_dev_allmulticast_enable,
3021         .allmulticast_disable       = ngbe_dev_allmulticast_disable,
3022         .link_update                = ngbe_dev_link_update,
3023         .stats_get                  = ngbe_dev_stats_get,
3024         .xstats_get                 = ngbe_dev_xstats_get,
3025         .xstats_get_by_id           = ngbe_dev_xstats_get_by_id,
3026         .stats_reset                = ngbe_dev_stats_reset,
3027         .xstats_reset               = ngbe_dev_xstats_reset,
3028         .xstats_get_names           = ngbe_dev_xstats_get_names,
3029         .xstats_get_names_by_id     = ngbe_dev_xstats_get_names_by_id,
3030         .fw_version_get             = ngbe_fw_version_get,
3031         .dev_supported_ptypes_get   = ngbe_dev_supported_ptypes_get,
3032         .mtu_set                    = ngbe_dev_mtu_set,
3033         .vlan_filter_set            = ngbe_vlan_filter_set,
3034         .vlan_tpid_set              = ngbe_vlan_tpid_set,
3035         .vlan_offload_set           = ngbe_vlan_offload_set,
3036         .vlan_strip_queue_set       = ngbe_vlan_strip_queue_set,
3037         .rx_queue_start             = ngbe_dev_rx_queue_start,
3038         .rx_queue_stop              = ngbe_dev_rx_queue_stop,
3039         .tx_queue_start             = ngbe_dev_tx_queue_start,
3040         .tx_queue_stop              = ngbe_dev_tx_queue_stop,
3041         .rx_queue_setup             = ngbe_dev_rx_queue_setup,
3042         .rx_queue_release           = ngbe_dev_rx_queue_release,
3043         .tx_queue_setup             = ngbe_dev_tx_queue_setup,
3044         .tx_queue_release           = ngbe_dev_tx_queue_release,
3045         .dev_led_on                 = ngbe_dev_led_on,
3046         .dev_led_off                = ngbe_dev_led_off,
3047         .flow_ctrl_get              = ngbe_flow_ctrl_get,
3048         .flow_ctrl_set              = ngbe_flow_ctrl_set,
3049         .mac_addr_add               = ngbe_add_rar,
3050         .mac_addr_remove            = ngbe_remove_rar,
3051         .mac_addr_set               = ngbe_set_default_mac_addr,
3052         .uc_hash_table_set          = ngbe_uc_hash_table_set,
3053         .uc_all_hash_table_set      = ngbe_uc_all_hash_table_set,
3054         .reta_update                = ngbe_dev_rss_reta_update,
3055         .reta_query                 = ngbe_dev_rss_reta_query,
3056         .rss_hash_update            = ngbe_dev_rss_hash_update,
3057         .rss_hash_conf_get          = ngbe_dev_rss_hash_conf_get,
3058         .set_mc_addr_list           = ngbe_dev_set_mc_addr_list,
3059         .rxq_info_get               = ngbe_rxq_info_get,
3060         .txq_info_get               = ngbe_txq_info_get,
3061         .rx_burst_mode_get          = ngbe_rx_burst_mode_get,
3062         .tx_burst_mode_get          = ngbe_tx_burst_mode_get,
3063         .timesync_enable            = ngbe_timesync_enable,
3064         .timesync_disable           = ngbe_timesync_disable,
3065         .timesync_read_rx_timestamp = ngbe_timesync_read_rx_timestamp,
3066         .timesync_read_tx_timestamp = ngbe_timesync_read_tx_timestamp,
3067         .get_reg                    = ngbe_get_regs,
3068         .get_eeprom_length          = ngbe_get_eeprom_length,
3069         .get_eeprom                 = ngbe_get_eeprom,
3070         .set_eeprom                 = ngbe_set_eeprom,
3071         .timesync_adjust_time       = ngbe_timesync_adjust_time,
3072         .timesync_read_time         = ngbe_timesync_read_time,
3073         .timesync_write_time        = ngbe_timesync_write_time,
3074         .tx_done_cleanup            = ngbe_dev_tx_done_cleanup,
3075 };
3076
3077 RTE_PMD_REGISTER_PCI(net_ngbe, rte_ngbe_pmd);
3078 RTE_PMD_REGISTER_PCI_TABLE(net_ngbe, pci_id_ngbe_map);
3079 RTE_PMD_REGISTER_KMOD_DEP(net_ngbe, "* igb_uio | uio_pci_generic | vfio-pci");
3080
3081 RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_init, init, NOTICE);
3082 RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_driver, driver, NOTICE);
3083
3084 #ifdef RTE_ETHDEV_DEBUG_RX
3085         RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_rx, rx, DEBUG);
3086 #endif
3087 #ifdef RTE_ETHDEV_DEBUG_TX
3088         RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_tx, tx, DEBUG);
3089 #endif