drivers/net/ngbe/ngbe_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd.
   3  * Copyright(c) 2010-2017 Intel Corporation
   4  */
   5
   6 #include <errno.h>
   7 #include <rte_common.h>
   8 #include <ethdev_pci.h>
   9
  10 #include <rte_alarm.h>
  11
  12 #include "ngbe_logs.h"
  13 #include "ngbe.h"
  14 #include "ngbe_ethdev.h"
  15 #include "ngbe_rxtx.h"
  16 #include "ngbe_regs_group.h"
  17
  18 static const struct reg_info ngbe_regs_general[] = {
  19         {NGBE_RST, 1, 1, "NGBE_RST"},
  20         {NGBE_STAT, 1, 1, "NGBE_STAT"},
  21         {NGBE_PORTCTL, 1, 1, "NGBE_PORTCTL"},
  22         {NGBE_GPIODATA, 1, 1, "NGBE_GPIODATA"},
  23         {NGBE_GPIOCTL, 1, 1, "NGBE_GPIOCTL"},
  24         {NGBE_LEDCTL, 1, 1, "NGBE_LEDCTL"},
  25         {0, 0, 0, ""}
  26 };
  27
  28 static const struct reg_info ngbe_regs_nvm[] = {
  29         {0, 0, 0, ""}
  30 };
  31
  32 static const struct reg_info ngbe_regs_interrupt[] = {
  33         {0, 0, 0, ""}
  34 };
  35
  36 static const struct reg_info ngbe_regs_fctl_others[] = {
  37         {0, 0, 0, ""}
  38 };
  39
  40 static const struct reg_info ngbe_regs_rxdma[] = {
  41         {0, 0, 0, ""}
  42 };
  43
  44 static const struct reg_info ngbe_regs_rx[] = {
  45         {0, 0, 0, ""}
  46 };
  47
  48 static struct reg_info ngbe_regs_tx[] = {
  49         {0, 0, 0, ""}
  50 };
  51
  52 static const struct reg_info ngbe_regs_wakeup[] = {
  53         {0, 0, 0, ""}
  54 };
  55
  56 static const struct reg_info ngbe_regs_mac[] = {
  57         {0, 0, 0, ""}
  58 };
  59
  60 static const struct reg_info ngbe_regs_diagnostic[] = {
  61         {0, 0, 0, ""},
  62 };
  63
  64 /* PF registers */
  65 static const struct reg_info *ngbe_regs_others[] = {
  66                                 ngbe_regs_general,
  67                                 ngbe_regs_nvm,
  68                                 ngbe_regs_interrupt,
  69                                 ngbe_regs_fctl_others,
  70                                 ngbe_regs_rxdma,
  71                                 ngbe_regs_rx,
  72                                 ngbe_regs_tx,
  73                                 ngbe_regs_wakeup,
  74                                 ngbe_regs_mac,
  75                                 ngbe_regs_diagnostic,
  76                                 NULL};
  77
  78 static int ngbe_dev_close(struct rte_eth_dev *dev);
  79 static int ngbe_dev_link_update(struct rte_eth_dev *dev,
  80                                 int wait_to_complete);
  81 static int ngbe_dev_stats_reset(struct rte_eth_dev *dev);
  82 static void ngbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
  83 static void ngbe_vlan_hw_strip_disable(struct rte_eth_dev *dev,
  84                                         uint16_t queue);
  85
  86 static void ngbe_dev_link_status_print(struct rte_eth_dev *dev);
  87 static int ngbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
  88 static int ngbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev);
  89 static int ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev);
  90 static int ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
  91 static void ngbe_dev_interrupt_handler(void *param);
  92 static void ngbe_configure_msix(struct rte_eth_dev *dev);
  93
  94 #define NGBE_SET_HWSTRIP(h, q) do {\
  95                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
  96                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
  97                 (h)->bitmap[idx] |= 1 << bit;\
  98         } while (0)
  99
 100 #define NGBE_CLEAR_HWSTRIP(h, q) do {\
 101                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 102                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 103                 (h)->bitmap[idx] &= ~(1 << bit);\
 104         } while (0)
 105
 106 #define NGBE_GET_HWSTRIP(h, q, r) do {\
 107                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 108                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 109                 (r) = (h)->bitmap[idx] >> bit & 1;\
 110         } while (0)
 111
 112 /*
 113  * The set of PCI devices this driver supports
 114  */
 115 static const struct rte_pci_id pci_id_ngbe_map[] = {
 116         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A2) },
 117         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A2S) },
 118         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A4) },
 119         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A4S) },
 120         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL2) },
 121         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S) },
 122         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL4) },
 123         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S) },
 124         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860NCSI) },
 125         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A1) },
 126         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A1L) },
 127         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W) },
 128         { .vendor_id = 0, /* sentinel */ },
 129 };
 130
 131 static const struct rte_eth_desc_lim rx_desc_lim = {
 132         .nb_max = NGBE_RING_DESC_MAX,
 133         .nb_min = NGBE_RING_DESC_MIN,
 134         .nb_align = NGBE_RXD_ALIGN,
 135 };
 136
 137 static const struct rte_eth_desc_lim tx_desc_lim = {
 138         .nb_max = NGBE_RING_DESC_MAX,
 139         .nb_min = NGBE_RING_DESC_MIN,
 140         .nb_align = NGBE_TXD_ALIGN,
 141         .nb_seg_max = NGBE_TX_MAX_SEG,
 142         .nb_mtu_seg_max = NGBE_TX_MAX_SEG,
 143 };
 144
 145 static const struct eth_dev_ops ngbe_eth_dev_ops;
 146
 147 #define HW_XSTAT(m) {#m, offsetof(struct ngbe_hw_stats, m)}
 148 #define HW_XSTAT_NAME(m, n) {n, offsetof(struct ngbe_hw_stats, m)}
 149 static const struct rte_ngbe_xstats_name_off rte_ngbe_stats_strings[] = {
 150         /* MNG RxTx */
 151         HW_XSTAT(mng_bmc2host_packets),
 152         HW_XSTAT(mng_host2bmc_packets),
 153         /* Basic RxTx */
 154         HW_XSTAT(rx_packets),
 155         HW_XSTAT(tx_packets),
 156         HW_XSTAT(rx_bytes),
 157         HW_XSTAT(tx_bytes),
 158         HW_XSTAT(rx_total_bytes),
 159         HW_XSTAT(rx_total_packets),
 160         HW_XSTAT(tx_total_packets),
 161         HW_XSTAT(rx_total_missed_packets),
 162         HW_XSTAT(rx_broadcast_packets),
 163         HW_XSTAT(rx_multicast_packets),
 164         HW_XSTAT(rx_management_packets),
 165         HW_XSTAT(tx_management_packets),
 166         HW_XSTAT(rx_management_dropped),
 167
 168         /* Basic Error */
 169         HW_XSTAT(rx_crc_errors),
 170         HW_XSTAT(rx_illegal_byte_errors),
 171         HW_XSTAT(rx_error_bytes),
 172         HW_XSTAT(rx_mac_short_packet_dropped),
 173         HW_XSTAT(rx_length_errors),
 174         HW_XSTAT(rx_undersize_errors),
 175         HW_XSTAT(rx_fragment_errors),
 176         HW_XSTAT(rx_oversize_errors),
 177         HW_XSTAT(rx_jabber_errors),
 178         HW_XSTAT(rx_l3_l4_xsum_error),
 179         HW_XSTAT(mac_local_errors),
 180         HW_XSTAT(mac_remote_errors),
 181
 182         /* MACSEC */
 183         HW_XSTAT(tx_macsec_pkts_untagged),
 184         HW_XSTAT(tx_macsec_pkts_encrypted),
 185         HW_XSTAT(tx_macsec_pkts_protected),
 186         HW_XSTAT(tx_macsec_octets_encrypted),
 187         HW_XSTAT(tx_macsec_octets_protected),
 188         HW_XSTAT(rx_macsec_pkts_untagged),
 189         HW_XSTAT(rx_macsec_pkts_badtag),
 190         HW_XSTAT(rx_macsec_pkts_nosci),
 191         HW_XSTAT(rx_macsec_pkts_unknownsci),
 192         HW_XSTAT(rx_macsec_octets_decrypted),
 193         HW_XSTAT(rx_macsec_octets_validated),
 194         HW_XSTAT(rx_macsec_sc_pkts_unchecked),
 195         HW_XSTAT(rx_macsec_sc_pkts_delayed),
 196         HW_XSTAT(rx_macsec_sc_pkts_late),
 197         HW_XSTAT(rx_macsec_sa_pkts_ok),
 198         HW_XSTAT(rx_macsec_sa_pkts_invalid),
 199         HW_XSTAT(rx_macsec_sa_pkts_notvalid),
 200         HW_XSTAT(rx_macsec_sa_pkts_unusedsa),
 201         HW_XSTAT(rx_macsec_sa_pkts_notusingsa),
 202
 203         /* MAC RxTx */
 204         HW_XSTAT(rx_size_64_packets),
 205         HW_XSTAT(rx_size_65_to_127_packets),
 206         HW_XSTAT(rx_size_128_to_255_packets),
 207         HW_XSTAT(rx_size_256_to_511_packets),
 208         HW_XSTAT(rx_size_512_to_1023_packets),
 209         HW_XSTAT(rx_size_1024_to_max_packets),
 210         HW_XSTAT(tx_size_64_packets),
 211         HW_XSTAT(tx_size_65_to_127_packets),
 212         HW_XSTAT(tx_size_128_to_255_packets),
 213         HW_XSTAT(tx_size_256_to_511_packets),
 214         HW_XSTAT(tx_size_512_to_1023_packets),
 215         HW_XSTAT(tx_size_1024_to_max_packets),
 216
 217         /* Flow Control */
 218         HW_XSTAT(tx_xon_packets),
 219         HW_XSTAT(rx_xon_packets),
 220         HW_XSTAT(tx_xoff_packets),
 221         HW_XSTAT(rx_xoff_packets),
 222
 223         HW_XSTAT_NAME(tx_xon_packets, "tx_flow_control_xon_packets"),
 224         HW_XSTAT_NAME(rx_xon_packets, "rx_flow_control_xon_packets"),
 225         HW_XSTAT_NAME(tx_xoff_packets, "tx_flow_control_xoff_packets"),
 226         HW_XSTAT_NAME(rx_xoff_packets, "rx_flow_control_xoff_packets"),
 227 };
 228
 229 #define NGBE_NB_HW_STATS (sizeof(rte_ngbe_stats_strings) / \
 230                            sizeof(rte_ngbe_stats_strings[0]))
 231
 232 /* Per-queue statistics */
 233 #define QP_XSTAT(m) {#m, offsetof(struct ngbe_hw_stats, qp[0].m)}
 234 static const struct rte_ngbe_xstats_name_off rte_ngbe_qp_strings[] = {
 235         QP_XSTAT(rx_qp_packets),
 236         QP_XSTAT(tx_qp_packets),
 237         QP_XSTAT(rx_qp_bytes),
 238         QP_XSTAT(tx_qp_bytes),
 239         QP_XSTAT(rx_qp_mc_packets),
 240 };
 241
 242 #define NGBE_NB_QP_STATS (sizeof(rte_ngbe_qp_strings) / \
 243                            sizeof(rte_ngbe_qp_strings[0]))
 244
 245 static inline int32_t
 246 ngbe_pf_reset_hw(struct ngbe_hw *hw)
 247 {
 248         uint32_t ctrl_ext;
 249         int32_t status;
 250
 251         status = hw->mac.reset_hw(hw);
 252
 253         ctrl_ext = rd32(hw, NGBE_PORTCTL);
 254         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 255         ctrl_ext |= NGBE_PORTCTL_RSTDONE;
 256         wr32(hw, NGBE_PORTCTL, ctrl_ext);
 257         ngbe_flush(hw);
 258
 259         if (status == NGBE_ERR_SFP_NOT_PRESENT)
 260                 status = 0;
 261         return status;
 262 }
 263
 264 static inline void
 265 ngbe_enable_intr(struct rte_eth_dev *dev)
 266 {
 267         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 268         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 269
 270         wr32(hw, NGBE_IENMISC, intr->mask_misc);
 271         wr32(hw, NGBE_IMC(0), intr->mask & BIT_MASK32);
 272         ngbe_flush(hw);
 273 }
 274
 275 static void
 276 ngbe_disable_intr(struct ngbe_hw *hw)
 277 {
 278         PMD_INIT_FUNC_TRACE();
 279
 280         wr32(hw, NGBE_IMS(0), NGBE_IMS_MASK);
 281         ngbe_flush(hw);
 282 }
 283
 284 /*
 285  * Ensure that all locks are released before first NVM or PHY access
 286  */
 287 static void
 288 ngbe_swfw_lock_reset(struct ngbe_hw *hw)
 289 {
 290         uint16_t mask;
 291
 292         /*
 293          * These ones are more tricky since they are common to all ports; but
 294          * swfw_sync retries last long enough (1s) to be almost sure that if
 295          * lock can not be taken it is due to an improper lock of the
 296          * semaphore.
 297          */
 298         mask = NGBE_MNGSEM_SWPHY |
 299                NGBE_MNGSEM_SWMBX |
 300                NGBE_MNGSEM_SWFLASH;
 301         if (hw->mac.acquire_swfw_sync(hw, mask) < 0)
 302                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 303
 304         hw->mac.release_swfw_sync(hw, mask);
 305 }
 306
 307 static int
 308 eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 309 {
 310         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 311         struct ngbe_hw *hw = ngbe_dev_hw(eth_dev);
 312         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(eth_dev);
 313         struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(eth_dev);
 314         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 315         const struct rte_memzone *mz;
 316         uint32_t ctrl_ext;
 317         int err, ret;
 318
 319         PMD_INIT_FUNC_TRACE();
 320
 321         eth_dev->dev_ops = &ngbe_eth_dev_ops;
 322         eth_dev->rx_queue_count       = ngbe_dev_rx_queue_count;
 323         eth_dev->rx_descriptor_status = ngbe_dev_rx_descriptor_status;
 324         eth_dev->tx_descriptor_status = ngbe_dev_tx_descriptor_status;
 325         eth_dev->rx_pkt_burst = &ngbe_recv_pkts;
 326         eth_dev->tx_pkt_burst = &ngbe_xmit_pkts;
 327         eth_dev->tx_pkt_prepare = &ngbe_prep_pkts;
 328
 329         /*
 330          * For secondary processes, we don't initialise any further as primary
 331          * has already done this work. Only check we don't need a different
 332          * Rx and Tx function.
 333          */
 334         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 335                 struct ngbe_tx_queue *txq;
 336                 /* Tx queue function in primary, set by last queue initialized
 337                  * Tx queue may not initialized by primary process
 338                  */
 339                 if (eth_dev->data->tx_queues) {
 340                         uint16_t nb_tx_queues = eth_dev->data->nb_tx_queues;
 341                         txq = eth_dev->data->tx_queues[nb_tx_queues - 1];
 342                         ngbe_set_tx_function(eth_dev, txq);
 343                 } else {
 344                         /* Use default Tx function if we get here */
 345                         PMD_INIT_LOG(NOTICE,
 346                                 "No Tx queues configured yet. Using default Tx function.");
 347                 }
 348
 349                 ngbe_set_rx_function(eth_dev);
 350
 351                 return 0;
 352         }
 353
 354         rte_eth_copy_pci_info(eth_dev, pci_dev);
 355         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 356
 357         /* Vendor and Device ID need to be set before init of shared code */
 358         hw->device_id = pci_dev->id.device_id;
 359         hw->vendor_id = pci_dev->id.vendor_id;
 360         hw->sub_system_id = pci_dev->id.subsystem_device_id;
 361         ngbe_map_device_id(hw);
 362         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 363
 364         /* Reserve memory for interrupt status block */
 365         mz = rte_eth_dma_zone_reserve(eth_dev, "ngbe_driver", -1,
 366                 NGBE_ISB_SIZE, NGBE_ALIGN, SOCKET_ID_ANY);
 367         if (mz == NULL)
 368                 return -ENOMEM;
 369
 370         hw->isb_dma = TMZ_PADDR(mz);
 371         hw->isb_mem = TMZ_VADDR(mz);
 372
 373         /* Initialize the shared code (base driver) */
 374         err = ngbe_init_shared_code(hw);
 375         if (err != 0) {
 376                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", err);
 377                 return -EIO;
 378         }
 379
 380         /* Unlock any pending hardware semaphore */
 381         ngbe_swfw_lock_reset(hw);
 382
 383         /* Get Hardware Flow Control setting */
 384         hw->fc.requested_mode = ngbe_fc_full;
 385         hw->fc.current_mode = ngbe_fc_full;
 386         hw->fc.pause_time = NGBE_FC_PAUSE_TIME;
 387         hw->fc.low_water = NGBE_FC_XON_LOTH;
 388         hw->fc.high_water = NGBE_FC_XOFF_HITH;
 389         hw->fc.send_xon = 1;
 390
 391         err = hw->rom.init_params(hw);
 392         if (err != 0) {
 393                 PMD_INIT_LOG(ERR, "The EEPROM init failed: %d", err);
 394                 return -EIO;
 395         }
 396
 397         /* Make sure we have a good EEPROM before we read from it */
 398         err = hw->rom.validate_checksum(hw, NULL);
 399         if (err != 0) {
 400                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", err);
 401                 return -EIO;
 402         }
 403
 404         err = hw->mac.init_hw(hw);
 405         if (err != 0) {
 406                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", err);
 407                 return -EIO;
 408         }
 409
 410         /* Reset the hw statistics */
 411         ngbe_dev_stats_reset(eth_dev);
 412
 413         /* disable interrupt */
 414         ngbe_disable_intr(hw);
 415
 416         /* Allocate memory for storing MAC addresses */
 417         eth_dev->data->mac_addrs = rte_zmalloc("ngbe", RTE_ETHER_ADDR_LEN *
 418                                                hw->mac.num_rar_entries, 0);
 419         if (eth_dev->data->mac_addrs == NULL) {
 420                 PMD_INIT_LOG(ERR,
 421                              "Failed to allocate %u bytes needed to store MAC addresses",
 422                              RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 423                 return -ENOMEM;
 424         }
 425
 426         /* Copy the permanent MAC address */
 427         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr,
 428                         &eth_dev->data->mac_addrs[0]);
 429
 430         /* Allocate memory for storing hash filter MAC addresses */
 431         eth_dev->data->hash_mac_addrs = rte_zmalloc("ngbe",
 432                         RTE_ETHER_ADDR_LEN * NGBE_VMDQ_NUM_UC_MAC, 0);
 433         if (eth_dev->data->hash_mac_addrs == NULL) {
 434                 PMD_INIT_LOG(ERR,
 435                              "Failed to allocate %d bytes needed to store MAC addresses",
 436                              RTE_ETHER_ADDR_LEN * NGBE_VMDQ_NUM_UC_MAC);
 437                 rte_free(eth_dev->data->mac_addrs);
 438                 eth_dev->data->mac_addrs = NULL;
 439                 return -ENOMEM;
 440         }
 441
 442         /* initialize the vfta */
 443         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 444
 445         /* initialize the hw strip bitmap*/
 446         memset(hwstrip, 0, sizeof(*hwstrip));
 447
 448         /* initialize PF if max_vfs not zero */
 449         ret = ngbe_pf_host_init(eth_dev);
 450         if (ret) {
 451                 rte_free(eth_dev->data->mac_addrs);
 452                 eth_dev->data->mac_addrs = NULL;
 453                 rte_free(eth_dev->data->hash_mac_addrs);
 454                 eth_dev->data->hash_mac_addrs = NULL;
 455                 return ret;
 456         }
 457
 458         ctrl_ext = rd32(hw, NGBE_PORTCTL);
 459         /* let hardware know driver is loaded */
 460         ctrl_ext |= NGBE_PORTCTL_DRVLOAD;
 461         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 462         ctrl_ext |= NGBE_PORTCTL_RSTDONE;
 463         wr32(hw, NGBE_PORTCTL, ctrl_ext);
 464         ngbe_flush(hw);
 465
 466         PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 467                         (int)hw->mac.type, (int)hw->phy.type);
 468
 469         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 470                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 471                      pci_dev->id.device_id);
 472
 473         rte_intr_callback_register(intr_handle,
 474                                    ngbe_dev_interrupt_handler, eth_dev);
 475
 476         /* enable uio/vfio intr/eventfd mapping */
 477         rte_intr_enable(intr_handle);
 478
 479         /* enable support intr */
 480         ngbe_enable_intr(eth_dev);
 481
 482         return 0;
 483 }
 484
 485 static int
 486 eth_ngbe_dev_uninit(struct rte_eth_dev *eth_dev)
 487 {
 488         PMD_INIT_FUNC_TRACE();
 489
 490         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 491                 return 0;
 492
 493         ngbe_dev_close(eth_dev);
 494
 495         return 0;
 496 }
 497
 498 static int
 499 eth_ngbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 500                 struct rte_pci_device *pci_dev)
 501 {
 502         return rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
 503                         sizeof(struct ngbe_adapter),
 504                         eth_dev_pci_specific_init, pci_dev,
 505                         eth_ngbe_dev_init, NULL);
 506 }
 507
 508 static int eth_ngbe_pci_remove(struct rte_pci_device *pci_dev)
 509 {
 510         struct rte_eth_dev *ethdev;
 511
 512         ethdev = rte_eth_dev_allocated(pci_dev->device.name);
 513         if (ethdev == NULL)
 514                 return 0;
 515
 516         return rte_eth_dev_destroy(ethdev, eth_ngbe_dev_uninit);
 517 }
 518
 519 static struct rte_pci_driver rte_ngbe_pmd = {
 520         .id_table = pci_id_ngbe_map,
 521         .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
 522                      RTE_PCI_DRV_INTR_LSC,
 523         .probe = eth_ngbe_pci_probe,
 524         .remove = eth_ngbe_pci_remove,
 525 };
 526
 527 static int
 528 ngbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 529 {
 530         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 531         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(dev);
 532         uint32_t vfta;
 533         uint32_t vid_idx;
 534         uint32_t vid_bit;
 535
 536         vid_idx = (uint32_t)((vlan_id >> 5) & 0x7F);
 537         vid_bit = (uint32_t)(1 << (vlan_id & 0x1F));
 538         vfta = rd32(hw, NGBE_VLANTBL(vid_idx));
 539         if (on)
 540                 vfta |= vid_bit;
 541         else
 542                 vfta &= ~vid_bit;
 543         wr32(hw, NGBE_VLANTBL(vid_idx), vfta);
 544
 545         /* update local VFTA copy */
 546         shadow_vfta->vfta[vid_idx] = vfta;
 547
 548         return 0;
 549 }
 550
 551 static void
 552 ngbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 553 {
 554         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 555         struct ngbe_rx_queue *rxq;
 556         bool restart;
 557         uint32_t rxcfg, rxbal, rxbah;
 558
 559         if (on)
 560                 ngbe_vlan_hw_strip_enable(dev, queue);
 561         else
 562                 ngbe_vlan_hw_strip_disable(dev, queue);
 563
 564         rxq = dev->data->rx_queues[queue];
 565         rxbal = rd32(hw, NGBE_RXBAL(rxq->reg_idx));
 566         rxbah = rd32(hw, NGBE_RXBAH(rxq->reg_idx));
 567         rxcfg = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
 568         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) {
 569                 restart = (rxcfg & NGBE_RXCFG_ENA) &&
 570                         !(rxcfg & NGBE_RXCFG_VLAN);
 571                 rxcfg |= NGBE_RXCFG_VLAN;
 572         } else {
 573                 restart = (rxcfg & NGBE_RXCFG_ENA) &&
 574                         (rxcfg & NGBE_RXCFG_VLAN);
 575                 rxcfg &= ~NGBE_RXCFG_VLAN;
 576         }
 577         rxcfg &= ~NGBE_RXCFG_ENA;
 578
 579         if (restart) {
 580                 /* set vlan strip for ring */
 581                 ngbe_dev_rx_queue_stop(dev, queue);
 582                 wr32(hw, NGBE_RXBAL(rxq->reg_idx), rxbal);
 583                 wr32(hw, NGBE_RXBAH(rxq->reg_idx), rxbah);
 584                 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxcfg);
 585                 ngbe_dev_rx_queue_start(dev, queue);
 586         }
 587 }
 588
 589 static int
 590 ngbe_vlan_tpid_set(struct rte_eth_dev *dev,
 591                     enum rte_vlan_type vlan_type,
 592                     uint16_t tpid)
 593 {
 594         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 595         int ret = 0;
 596         uint32_t portctrl, vlan_ext, qinq;
 597
 598         portctrl = rd32(hw, NGBE_PORTCTL);
 599
 600         vlan_ext = (portctrl & NGBE_PORTCTL_VLANEXT);
 601         qinq = vlan_ext && (portctrl & NGBE_PORTCTL_QINQ);
 602         switch (vlan_type) {
 603         case RTE_ETH_VLAN_TYPE_INNER:
 604                 if (vlan_ext) {
 605                         wr32m(hw, NGBE_VLANCTL,
 606                                 NGBE_VLANCTL_TPID_MASK,
 607                                 NGBE_VLANCTL_TPID(tpid));
 608                         wr32m(hw, NGBE_DMATXCTRL,
 609                                 NGBE_DMATXCTRL_TPID_MASK,
 610                                 NGBE_DMATXCTRL_TPID(tpid));
 611                 } else {
 612                         ret = -ENOTSUP;
 613                         PMD_DRV_LOG(ERR,
 614                                 "Inner type is not supported by single VLAN");
 615                 }
 616
 617                 if (qinq) {
 618                         wr32m(hw, NGBE_TAGTPID(0),
 619                                 NGBE_TAGTPID_LSB_MASK,
 620                                 NGBE_TAGTPID_LSB(tpid));
 621                 }
 622                 break;
 623         case RTE_ETH_VLAN_TYPE_OUTER:
 624                 if (vlan_ext) {
 625                         /* Only the high 16-bits is valid */
 626                         wr32m(hw, NGBE_EXTAG,
 627                                 NGBE_EXTAG_VLAN_MASK,
 628                                 NGBE_EXTAG_VLAN(tpid));
 629                 } else {
 630                         wr32m(hw, NGBE_VLANCTL,
 631                                 NGBE_VLANCTL_TPID_MASK,
 632                                 NGBE_VLANCTL_TPID(tpid));
 633                         wr32m(hw, NGBE_DMATXCTRL,
 634                                 NGBE_DMATXCTRL_TPID_MASK,
 635                                 NGBE_DMATXCTRL_TPID(tpid));
 636                 }
 637
 638                 if (qinq) {
 639                         wr32m(hw, NGBE_TAGTPID(0),
 640                                 NGBE_TAGTPID_MSB_MASK,
 641                                 NGBE_TAGTPID_MSB(tpid));
 642                 }
 643                 break;
 644         default:
 645                 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type);
 646                 return -EINVAL;
 647         }
 648
 649         return ret;
 650 }
 651
 652 void
 653 ngbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 654 {
 655         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 656         uint32_t vlnctrl;
 657
 658         PMD_INIT_FUNC_TRACE();
 659
 660         /* Filter Table Disable */
 661         vlnctrl = rd32(hw, NGBE_VLANCTL);
 662         vlnctrl &= ~NGBE_VLANCTL_VFE;
 663         wr32(hw, NGBE_VLANCTL, vlnctrl);
 664 }
 665
 666 void
 667 ngbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 668 {
 669         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 670         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(dev);
 671         uint32_t vlnctrl;
 672         uint16_t i;
 673
 674         PMD_INIT_FUNC_TRACE();
 675
 676         /* Filter Table Enable */
 677         vlnctrl = rd32(hw, NGBE_VLANCTL);
 678         vlnctrl &= ~NGBE_VLANCTL_CFIENA;
 679         vlnctrl |= NGBE_VLANCTL_VFE;
 680         wr32(hw, NGBE_VLANCTL, vlnctrl);
 681
 682         /* write whatever is in local vfta copy */
 683         for (i = 0; i < NGBE_VFTA_SIZE; i++)
 684                 wr32(hw, NGBE_VLANTBL(i), shadow_vfta->vfta[i]);
 685 }
 686
 687 void
 688 ngbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 689 {
 690         struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(dev);
 691         struct ngbe_rx_queue *rxq;
 692
 693         if (queue >= NGBE_MAX_RX_QUEUE_NUM)
 694                 return;
 695
 696         if (on)
 697                 NGBE_SET_HWSTRIP(hwstrip, queue);
 698         else
 699                 NGBE_CLEAR_HWSTRIP(hwstrip, queue);
 700
 701         if (queue >= dev->data->nb_rx_queues)
 702                 return;
 703
 704         rxq = dev->data->rx_queues[queue];
 705
 706         if (on) {
 707                 rxq->vlan_flags = RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
 708                 rxq->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 709         } else {
 710                 rxq->vlan_flags = RTE_MBUF_F_RX_VLAN;
 711                 rxq->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 712         }
 713 }
 714
 715 static void
 716 ngbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 717 {
 718         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 719         uint32_t ctrl;
 720
 721         PMD_INIT_FUNC_TRACE();
 722
 723         ctrl = rd32(hw, NGBE_RXCFG(queue));
 724         ctrl &= ~NGBE_RXCFG_VLAN;
 725         wr32(hw, NGBE_RXCFG(queue), ctrl);
 726
 727         /* record those setting for HW strip per queue */
 728         ngbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 729 }
 730
 731 static void
 732 ngbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 733 {
 734         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 735         uint32_t ctrl;
 736
 737         PMD_INIT_FUNC_TRACE();
 738
 739         ctrl = rd32(hw, NGBE_RXCFG(queue));
 740         ctrl |= NGBE_RXCFG_VLAN;
 741         wr32(hw, NGBE_RXCFG(queue), ctrl);
 742
 743         /* record those setting for HW strip per queue */
 744         ngbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 745 }
 746
 747 static void
 748 ngbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
 749 {
 750         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 751         uint32_t ctrl;
 752
 753         PMD_INIT_FUNC_TRACE();
 754
 755         ctrl = rd32(hw, NGBE_PORTCTL);
 756         ctrl &= ~NGBE_PORTCTL_VLANEXT;
 757         ctrl &= ~NGBE_PORTCTL_QINQ;
 758         wr32(hw, NGBE_PORTCTL, ctrl);
 759 }
 760
 761 static void
 762 ngbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
 763 {
 764         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 765         uint32_t ctrl;
 766
 767         PMD_INIT_FUNC_TRACE();
 768
 769         ctrl  = rd32(hw, NGBE_PORTCTL);
 770         ctrl |= NGBE_PORTCTL_VLANEXT | NGBE_PORTCTL_QINQ;
 771         wr32(hw, NGBE_PORTCTL, ctrl);
 772 }
 773
 774 static void
 775 ngbe_qinq_hw_strip_disable(struct rte_eth_dev *dev)
 776 {
 777         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 778         uint32_t ctrl;
 779
 780         PMD_INIT_FUNC_TRACE();
 781
 782         ctrl = rd32(hw, NGBE_PORTCTL);
 783         ctrl &= ~NGBE_PORTCTL_QINQ;
 784         wr32(hw, NGBE_PORTCTL, ctrl);
 785 }
 786
 787 static void
 788 ngbe_qinq_hw_strip_enable(struct rte_eth_dev *dev)
 789 {
 790         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 791         uint32_t ctrl;
 792
 793         PMD_INIT_FUNC_TRACE();
 794
 795         ctrl  = rd32(hw, NGBE_PORTCTL);
 796         ctrl |= NGBE_PORTCTL_QINQ | NGBE_PORTCTL_VLANEXT;
 797         wr32(hw, NGBE_PORTCTL, ctrl);
 798 }
 799
 800 void
 801 ngbe_vlan_hw_strip_config(struct rte_eth_dev *dev)
 802 {
 803         struct ngbe_rx_queue *rxq;
 804         uint16_t i;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 809                 rxq = dev->data->rx_queues[i];
 810
 811                 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 812                         ngbe_vlan_hw_strip_enable(dev, i);
 813                 else
 814                         ngbe_vlan_hw_strip_disable(dev, i);
 815         }
 816 }
 817
 818 void
 819 ngbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, int mask)
 820 {
 821         uint16_t i;
 822         struct rte_eth_rxmode *rxmode;
 823         struct ngbe_rx_queue *rxq;
 824
 825         if (mask & RTE_ETH_VLAN_STRIP_MASK) {
 826                 rxmode = &dev->data->dev_conf.rxmode;
 827                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 828                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 829                                 rxq = dev->data->rx_queues[i];
 830                                 rxq->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 831                         }
 832                 else
 833                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 834                                 rxq = dev->data->rx_queues[i];
 835                                 rxq->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 836                         }
 837         }
 838 }
 839
 840 static int
 841 ngbe_vlan_offload_config(struct rte_eth_dev *dev, int mask)
 842 {
 843         struct rte_eth_rxmode *rxmode;
 844         rxmode = &dev->data->dev_conf.rxmode;
 845
 846         if (mask & RTE_ETH_VLAN_STRIP_MASK)
 847                 ngbe_vlan_hw_strip_config(dev);
 848
 849         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
 850                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER)
 851                         ngbe_vlan_hw_filter_enable(dev);
 852                 else
 853                         ngbe_vlan_hw_filter_disable(dev);
 854         }
 855
 856         if (mask & RTE_ETH_VLAN_EXTEND_MASK) {
 857                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_EXTEND)
 858                         ngbe_vlan_hw_extend_enable(dev);
 859                 else
 860                         ngbe_vlan_hw_extend_disable(dev);
 861         }
 862
 863         if (mask & RTE_ETH_QINQ_STRIP_MASK) {
 864                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP)
 865                         ngbe_qinq_hw_strip_enable(dev);
 866                 else
 867                         ngbe_qinq_hw_strip_disable(dev);
 868         }
 869
 870         return 0;
 871 }
 872
 873 static int
 874 ngbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
 875 {
 876         ngbe_config_vlan_strip_on_all_queues(dev, mask);
 877
 878         ngbe_vlan_offload_config(dev, mask);
 879
 880         return 0;
 881 }
 882
 883 static int
 884 ngbe_dev_configure(struct rte_eth_dev *dev)
 885 {
 886         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 887         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
 888
 889         PMD_INIT_FUNC_TRACE();
 890
 891         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
 892                 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
 893
 894         /* set flag to update link status after init */
 895         intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
 896
 897         /*
 898          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
 899          * allocation Rx preconditions we will reset it.
 900          */
 901         adapter->rx_bulk_alloc_allowed = true;
 902
 903         return 0;
 904 }
 905
 906 static void
 907 ngbe_dev_phy_intr_setup(struct rte_eth_dev *dev)
 908 {
 909         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 910         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 911
 912         wr32(hw, NGBE_GPIODIR, NGBE_GPIODIR_DDR(1));
 913         wr32(hw, NGBE_GPIOINTEN, NGBE_GPIOINTEN_INT(3));
 914         wr32(hw, NGBE_GPIOINTTYPE, NGBE_GPIOINTTYPE_LEVEL(0));
 915         if (hw->phy.type == ngbe_phy_yt8521s_sfi)
 916                 wr32(hw, NGBE_GPIOINTPOL, NGBE_GPIOINTPOL_ACT(0));
 917         else
 918                 wr32(hw, NGBE_GPIOINTPOL, NGBE_GPIOINTPOL_ACT(3));
 919
 920         intr->mask_misc |= NGBE_ICRMISC_GPIO;
 921 }
 922
 923 /*
 924  * Configure device link speed and setup link.
 925  * It returns 0 on success.
 926  */
 927 static int
 928 ngbe_dev_start(struct rte_eth_dev *dev)
 929 {
 930         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 931         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
 932         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 933         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 934         uint32_t intr_vector = 0;
 935         int err;
 936         bool link_up = false, negotiate = false;
 937         uint32_t speed = 0;
 938         uint32_t allowed_speeds = 0;
 939         int mask = 0;
 940         int status;
 941         uint32_t *link_speeds;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Stop the link setup handler before resetting the HW. */
 946         rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev);
 947
 948         /* disable uio/vfio intr/eventfd mapping */
 949         rte_intr_disable(intr_handle);
 950
 951         /* stop adapter */
 952         hw->adapter_stopped = 0;
 953
 954         /* reinitialize adapter, this calls reset and start */
 955         hw->nb_rx_queues = dev->data->nb_rx_queues;
 956         hw->nb_tx_queues = dev->data->nb_tx_queues;
 957         status = ngbe_pf_reset_hw(hw);
 958         if (status != 0)
 959                 return -1;
 960         hw->mac.start_hw(hw);
 961         hw->mac.get_link_status = true;
 962
 963         ngbe_set_pcie_master(hw, true);
 964
 965         /* configure PF module if SRIOV enabled */
 966         ngbe_pf_host_configure(dev);
 967
 968         ngbe_dev_phy_intr_setup(dev);
 969
 970         /* check and configure queue intr-vector mapping */
 971         if ((rte_intr_cap_multiple(intr_handle) ||
 972              !RTE_ETH_DEV_SRIOV(dev).active) &&
 973             dev->data->dev_conf.intr_conf.rxq != 0) {
 974                 intr_vector = dev->data->nb_rx_queues;
 975                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 976                         return -1;
 977         }
 978
 979         if (rte_intr_dp_is_en(intr_handle)) {
 980                 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec",
 981                                                    dev->data->nb_rx_queues)) {
 982                         PMD_INIT_LOG(ERR,
 983                                      "Failed to allocate %d rx_queues intr_vec",
 984                                      dev->data->nb_rx_queues);
 985                         return -ENOMEM;
 986                 }
 987         }
 988
 989         /* configure MSI-X for sleep until Rx interrupt */
 990         ngbe_configure_msix(dev);
 991
 992         /* initialize transmission unit */
 993         ngbe_dev_tx_init(dev);
 994
 995         /* This can fail when allocating mbufs for descriptor rings */
 996         err = ngbe_dev_rx_init(dev);
 997         if (err != 0) {
 998                 PMD_INIT_LOG(ERR, "Unable to initialize Rx hardware");
 999                 goto error;
1000         }
1001
1002         mask = RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK |
1003                 RTE_ETH_VLAN_EXTEND_MASK;
1004         err = ngbe_vlan_offload_config(dev, mask);
1005         if (err != 0) {
1006                 PMD_INIT_LOG(ERR, "Unable to set VLAN offload");
1007                 goto error;
1008         }
1009
1010         hw->mac.setup_pba(hw);
1011         ngbe_configure_port(dev);
1012
1013         err = ngbe_dev_rxtx_start(dev);
1014         if (err < 0) {
1015                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1016                 goto error;
1017         }
1018
1019         /* Skip link setup if loopback mode is enabled. */
1020         if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
1021                 goto skip_link_setup;
1022
1023         err = hw->mac.check_link(hw, &speed, &link_up, 0);
1024         if (err != 0)
1025                 goto error;
1026         dev->data->dev_link.link_status = link_up;
1027
1028         link_speeds = &dev->data->dev_conf.link_speeds;
1029         if (*link_speeds == RTE_ETH_LINK_SPEED_AUTONEG)
1030                 negotiate = true;
1031
1032         err = hw->mac.get_link_capabilities(hw, &speed, &negotiate);
1033         if (err != 0)
1034                 goto error;
1035
1036         allowed_speeds = 0;
1037         if (hw->mac.default_speeds & NGBE_LINK_SPEED_1GB_FULL)
1038                 allowed_speeds |= RTE_ETH_LINK_SPEED_1G;
1039         if (hw->mac.default_speeds & NGBE_LINK_SPEED_100M_FULL)
1040                 allowed_speeds |= RTE_ETH_LINK_SPEED_100M;
1041         if (hw->mac.default_speeds & NGBE_LINK_SPEED_10M_FULL)
1042                 allowed_speeds |= RTE_ETH_LINK_SPEED_10M;
1043
1044         if (*link_speeds & ~allowed_speeds) {
1045                 PMD_INIT_LOG(ERR, "Invalid link setting");
1046                 goto error;
1047         }
1048
1049         speed = 0x0;
1050         if (*link_speeds == RTE_ETH_LINK_SPEED_AUTONEG) {
1051                 speed = hw->mac.default_speeds;
1052         } else {
1053                 if (*link_speeds & RTE_ETH_LINK_SPEED_1G)
1054                         speed |= NGBE_LINK_SPEED_1GB_FULL;
1055                 if (*link_speeds & RTE_ETH_LINK_SPEED_100M)
1056                         speed |= NGBE_LINK_SPEED_100M_FULL;
1057                 if (*link_speeds & RTE_ETH_LINK_SPEED_10M)
1058                         speed |= NGBE_LINK_SPEED_10M_FULL;
1059         }
1060
1061         err = hw->phy.init_hw(hw);
1062         if (err != 0) {
1063                 PMD_INIT_LOG(ERR, "PHY init failed");
1064                 goto error;
1065         }
1066         err = hw->mac.setup_link(hw, speed, link_up);
1067         if (err != 0)
1068                 goto error;
1069
1070 skip_link_setup:
1071
1072         if (rte_intr_allow_others(intr_handle)) {
1073                 ngbe_dev_misc_interrupt_setup(dev);
1074                 /* check if lsc interrupt is enabled */
1075                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1076                         ngbe_dev_lsc_interrupt_setup(dev, TRUE);
1077                 else
1078                         ngbe_dev_lsc_interrupt_setup(dev, FALSE);
1079                 ngbe_dev_macsec_interrupt_setup(dev);
1080                 ngbe_set_ivar_map(hw, -1, 1, NGBE_MISC_VEC_ID);
1081         } else {
1082                 rte_intr_callback_unregister(intr_handle,
1083                                              ngbe_dev_interrupt_handler, dev);
1084                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1085                         PMD_INIT_LOG(INFO,
1086                                      "LSC won't enable because of no intr multiplex");
1087         }
1088
1089         /* check if rxq interrupt is enabled */
1090         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1091             rte_intr_dp_is_en(intr_handle))
1092                 ngbe_dev_rxq_interrupt_setup(dev);
1093
1094         /* enable UIO/VFIO intr/eventfd mapping */
1095         rte_intr_enable(intr_handle);
1096
1097         /* resume enabled intr since HW reset */
1098         ngbe_enable_intr(dev);
1099
1100         if (hw->gpio_ctl) {
1101                 /* gpio0 is used to power on/off control*/
1102                 wr32(hw, NGBE_GPIODATA, 0);
1103         }
1104
1105         /*
1106          * Update link status right before return, because it may
1107          * start link configuration process in a separate thread.
1108          */
1109         ngbe_dev_link_update(dev, 0);
1110
1111         ngbe_read_stats_registers(hw, hw_stats);
1112         hw->offset_loaded = 1;
1113
1114         return 0;
1115
1116 error:
1117         PMD_INIT_LOG(ERR, "failure in dev start: %d", err);
1118         ngbe_dev_clear_queues(dev);
1119         return -EIO;
1120 }
1121
1122 /*
1123  * Stop device: disable rx and tx functions to allow for reconfiguring.
1124  */
1125 static int
1126 ngbe_dev_stop(struct rte_eth_dev *dev)
1127 {
1128         struct rte_eth_link link;
1129         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
1130         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1131         struct ngbe_vf_info *vfinfo = *NGBE_DEV_VFDATA(dev);
1132         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1133         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1134         int vf;
1135
1136         if (hw->adapter_stopped)
1137                 return 0;
1138
1139         PMD_INIT_FUNC_TRACE();
1140
1141         rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev);
1142
1143         if (hw->gpio_ctl) {
1144                 /* gpio0 is used to power on/off control*/
1145                 wr32(hw, NGBE_GPIODATA, NGBE_GPIOBIT_0);
1146         }
1147
1148         /* disable interrupts */
1149         ngbe_disable_intr(hw);
1150
1151         /* reset the NIC */
1152         ngbe_pf_reset_hw(hw);
1153         hw->adapter_stopped = 0;
1154
1155         /* stop adapter */
1156         ngbe_stop_hw(hw);
1157
1158         for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++)
1159                 vfinfo[vf].clear_to_send = false;
1160
1161         ngbe_dev_clear_queues(dev);
1162
1163         /* Clear stored conf */
1164         dev->data->scattered_rx = 0;
1165
1166         /* Clear recorded link status */
1167         memset(&link, 0, sizeof(link));
1168         rte_eth_linkstatus_set(dev, &link);
1169
1170         if (!rte_intr_allow_others(intr_handle))
1171                 /* resume to the default handler */
1172                 rte_intr_callback_register(intr_handle,
1173                                            ngbe_dev_interrupt_handler,
1174                                            (void *)dev);
1175
1176         /* Clean datapath event and queue/vec mapping */
1177         rte_intr_efd_disable(intr_handle);
1178         rte_intr_vec_list_free(intr_handle);
1179
1180         ngbe_set_pcie_master(hw, true);
1181
1182         adapter->rss_reta_updated = 0;
1183
1184         hw->adapter_stopped = true;
1185         dev->data->dev_started = 0;
1186
1187         return 0;
1188 }
1189
1190 /*
1191  * Reset and stop device.
1192  */
1193 static int
1194 ngbe_dev_close(struct rte_eth_dev *dev)
1195 {
1196         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1197         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1198         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1199         int retries = 0;
1200         int ret;
1201
1202         PMD_INIT_FUNC_TRACE();
1203
1204         ngbe_pf_reset_hw(hw);
1205
1206         ngbe_dev_stop(dev);
1207
1208         ngbe_dev_free_queues(dev);
1209
1210         ngbe_set_pcie_master(hw, false);
1211
1212         /* reprogram the RAR[0] in case user changed it. */
1213         ngbe_set_rar(hw, 0, hw->mac.addr, 0, true);
1214
1215         /* Unlock any pending hardware semaphore */
1216         ngbe_swfw_lock_reset(hw);
1217
1218         /* disable uio intr before callback unregister */
1219         rte_intr_disable(intr_handle);
1220
1221         do {
1222                 ret = rte_intr_callback_unregister(intr_handle,
1223                                 ngbe_dev_interrupt_handler, dev);
1224                 if (ret >= 0 || ret == -ENOENT) {
1225                         break;
1226                 } else if (ret != -EAGAIN) {
1227                         PMD_INIT_LOG(ERR,
1228                                 "intr callback unregister failed: %d",
1229                                 ret);
1230                 }
1231                 rte_delay_ms(100);
1232         } while (retries++ < (10 + NGBE_LINK_UP_TIME));
1233
1234         /* uninitialize PF if max_vfs not zero */
1235         ngbe_pf_host_uninit(dev);
1236
1237         rte_free(dev->data->mac_addrs);
1238         dev->data->mac_addrs = NULL;
1239
1240         rte_free(dev->data->hash_mac_addrs);
1241         dev->data->hash_mac_addrs = NULL;
1242
1243         return ret;
1244 }
1245
1246 /*
1247  * Reset PF device.
1248  */
1249 static int
1250 ngbe_dev_reset(struct rte_eth_dev *dev)
1251 {
1252         int ret;
1253
1254         /* When a DPDK PMD PF begin to reset PF port, it should notify all
1255          * its VF to make them align with it. The detailed notification
1256          * mechanism is PMD specific. As to ngbe PF, it is rather complex.
1257          * To avoid unexpected behavior in VF, currently reset of PF with
1258          * SR-IOV activation is not supported. It might be supported later.
1259          */
1260         if (dev->data->sriov.active)
1261                 return -ENOTSUP;
1262
1263         ret = eth_ngbe_dev_uninit(dev);
1264         if (ret != 0)
1265                 return ret;
1266
1267         ret = eth_ngbe_dev_init(dev, NULL);
1268
1269         return ret;
1270 }
1271
1272 #define UPDATE_QP_COUNTER_32bit(reg, last_counter, counter)     \
1273         {                                                       \
1274                 uint32_t current_counter = rd32(hw, reg);       \
1275                 if (current_counter < last_counter)             \
1276                         current_counter += 0x100000000LL;       \
1277                 if (!hw->offset_loaded)                         \
1278                         last_counter = current_counter;         \
1279                 counter = current_counter - last_counter;       \
1280                 counter &= 0xFFFFFFFFLL;                        \
1281         }
1282
1283 #define UPDATE_QP_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
1284         {                                                                \
1285                 uint64_t current_counter_lsb = rd32(hw, reg_lsb);        \
1286                 uint64_t current_counter_msb = rd32(hw, reg_msb);        \
1287                 uint64_t current_counter = (current_counter_msb << 32) | \
1288                         current_counter_lsb;                             \
1289                 if (current_counter < last_counter)                      \
1290                         current_counter += 0x1000000000LL;               \
1291                 if (!hw->offset_loaded)                                  \
1292                         last_counter = current_counter;                  \
1293                 counter = current_counter - last_counter;                \
1294                 counter &= 0xFFFFFFFFFLL;                                \
1295         }
1296
1297 void
1298 ngbe_read_stats_registers(struct ngbe_hw *hw,
1299                            struct ngbe_hw_stats *hw_stats)
1300 {
1301         unsigned int i;
1302
1303         /* QP Stats */
1304         for (i = 0; i < hw->nb_rx_queues; i++) {
1305                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXPKT(i),
1306                         hw->qp_last[i].rx_qp_packets,
1307                         hw_stats->qp[i].rx_qp_packets);
1308                 UPDATE_QP_COUNTER_36bit(NGBE_QPRXOCTL(i), NGBE_QPRXOCTH(i),
1309                         hw->qp_last[i].rx_qp_bytes,
1310                         hw_stats->qp[i].rx_qp_bytes);
1311                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXMPKT(i),
1312                         hw->qp_last[i].rx_qp_mc_packets,
1313                         hw_stats->qp[i].rx_qp_mc_packets);
1314                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXBPKT(i),
1315                         hw->qp_last[i].rx_qp_bc_packets,
1316                         hw_stats->qp[i].rx_qp_bc_packets);
1317         }
1318
1319         for (i = 0; i < hw->nb_tx_queues; i++) {
1320                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXPKT(i),
1321                         hw->qp_last[i].tx_qp_packets,
1322                         hw_stats->qp[i].tx_qp_packets);
1323                 UPDATE_QP_COUNTER_36bit(NGBE_QPTXOCTL(i), NGBE_QPTXOCTH(i),
1324                         hw->qp_last[i].tx_qp_bytes,
1325                         hw_stats->qp[i].tx_qp_bytes);
1326                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXMPKT(i),
1327                         hw->qp_last[i].tx_qp_mc_packets,
1328                         hw_stats->qp[i].tx_qp_mc_packets);
1329                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXBPKT(i),
1330                         hw->qp_last[i].tx_qp_bc_packets,
1331                         hw_stats->qp[i].tx_qp_bc_packets);
1332         }
1333
1334         /* PB Stats */
1335         hw_stats->rx_up_dropped += rd32(hw, NGBE_PBRXMISS);
1336         hw_stats->rdb_pkt_cnt += rd32(hw, NGBE_PBRXPKT);
1337         hw_stats->rdb_repli_cnt += rd32(hw, NGBE_PBRXREP);
1338         hw_stats->rdb_drp_cnt += rd32(hw, NGBE_PBRXDROP);
1339         hw_stats->tx_xoff_packets += rd32(hw, NGBE_PBTXLNKXOFF);
1340         hw_stats->tx_xon_packets += rd32(hw, NGBE_PBTXLNKXON);
1341
1342         hw_stats->rx_xon_packets += rd32(hw, NGBE_PBRXLNKXON);
1343         hw_stats->rx_xoff_packets += rd32(hw, NGBE_PBRXLNKXOFF);
1344
1345         /* DMA Stats */
1346         hw_stats->rx_drop_packets += rd32(hw, NGBE_DMARXDROP);
1347         hw_stats->tx_drop_packets += rd32(hw, NGBE_DMATXDROP);
1348         hw_stats->rx_dma_drop += rd32(hw, NGBE_DMARXDROP);
1349         hw_stats->tx_secdrp_packets += rd32(hw, NGBE_DMATXSECDROP);
1350         hw_stats->rx_packets += rd32(hw, NGBE_DMARXPKT);
1351         hw_stats->tx_packets += rd32(hw, NGBE_DMATXPKT);
1352         hw_stats->rx_bytes += rd64(hw, NGBE_DMARXOCTL);
1353         hw_stats->tx_bytes += rd64(hw, NGBE_DMATXOCTL);
1354
1355         /* MAC Stats */
1356         hw_stats->rx_crc_errors += rd64(hw, NGBE_MACRXERRCRCL);
1357         hw_stats->rx_multicast_packets += rd64(hw, NGBE_MACRXMPKTL);
1358         hw_stats->tx_multicast_packets += rd64(hw, NGBE_MACTXMPKTL);
1359
1360         hw_stats->rx_total_packets += rd64(hw, NGBE_MACRXPKTL);
1361         hw_stats->tx_total_packets += rd64(hw, NGBE_MACTXPKTL);
1362         hw_stats->rx_total_bytes += rd64(hw, NGBE_MACRXGBOCTL);
1363
1364         hw_stats->rx_broadcast_packets += rd64(hw, NGBE_MACRXOCTL);
1365         hw_stats->tx_broadcast_packets += rd32(hw, NGBE_MACTXOCTL);
1366
1367         hw_stats->rx_size_64_packets += rd64(hw, NGBE_MACRX1TO64L);
1368         hw_stats->rx_size_65_to_127_packets += rd64(hw, NGBE_MACRX65TO127L);
1369         hw_stats->rx_size_128_to_255_packets += rd64(hw, NGBE_MACRX128TO255L);
1370         hw_stats->rx_size_256_to_511_packets += rd64(hw, NGBE_MACRX256TO511L);
1371         hw_stats->rx_size_512_to_1023_packets +=
1372                         rd64(hw, NGBE_MACRX512TO1023L);
1373         hw_stats->rx_size_1024_to_max_packets +=
1374                         rd64(hw, NGBE_MACRX1024TOMAXL);
1375         hw_stats->tx_size_64_packets += rd64(hw, NGBE_MACTX1TO64L);
1376         hw_stats->tx_size_65_to_127_packets += rd64(hw, NGBE_MACTX65TO127L);
1377         hw_stats->tx_size_128_to_255_packets += rd64(hw, NGBE_MACTX128TO255L);
1378         hw_stats->tx_size_256_to_511_packets += rd64(hw, NGBE_MACTX256TO511L);
1379         hw_stats->tx_size_512_to_1023_packets +=
1380                         rd64(hw, NGBE_MACTX512TO1023L);
1381         hw_stats->tx_size_1024_to_max_packets +=
1382                         rd64(hw, NGBE_MACTX1024TOMAXL);
1383
1384         hw_stats->rx_undersize_errors += rd64(hw, NGBE_MACRXERRLENL);
1385         hw_stats->rx_oversize_errors += rd32(hw, NGBE_MACRXOVERSIZE);
1386         hw_stats->rx_jabber_errors += rd32(hw, NGBE_MACRXJABBER);
1387
1388         /* MNG Stats */
1389         hw_stats->mng_bmc2host_packets = rd32(hw, NGBE_MNGBMC2OS);
1390         hw_stats->mng_host2bmc_packets = rd32(hw, NGBE_MNGOS2BMC);
1391         hw_stats->rx_management_packets = rd32(hw, NGBE_DMARXMNG);
1392         hw_stats->tx_management_packets = rd32(hw, NGBE_DMATXMNG);
1393
1394         /* MACsec Stats */
1395         hw_stats->tx_macsec_pkts_untagged += rd32(hw, NGBE_LSECTX_UTPKT);
1396         hw_stats->tx_macsec_pkts_encrypted +=
1397                         rd32(hw, NGBE_LSECTX_ENCPKT);
1398         hw_stats->tx_macsec_pkts_protected +=
1399                         rd32(hw, NGBE_LSECTX_PROTPKT);
1400         hw_stats->tx_macsec_octets_encrypted +=
1401                         rd32(hw, NGBE_LSECTX_ENCOCT);
1402         hw_stats->tx_macsec_octets_protected +=
1403                         rd32(hw, NGBE_LSECTX_PROTOCT);
1404         hw_stats->rx_macsec_pkts_untagged += rd32(hw, NGBE_LSECRX_UTPKT);
1405         hw_stats->rx_macsec_pkts_badtag += rd32(hw, NGBE_LSECRX_BTPKT);
1406         hw_stats->rx_macsec_pkts_nosci += rd32(hw, NGBE_LSECRX_NOSCIPKT);
1407         hw_stats->rx_macsec_pkts_unknownsci += rd32(hw, NGBE_LSECRX_UNSCIPKT);
1408         hw_stats->rx_macsec_octets_decrypted += rd32(hw, NGBE_LSECRX_DECOCT);
1409         hw_stats->rx_macsec_octets_validated += rd32(hw, NGBE_LSECRX_VLDOCT);
1410         hw_stats->rx_macsec_sc_pkts_unchecked +=
1411                         rd32(hw, NGBE_LSECRX_UNCHKPKT);
1412         hw_stats->rx_macsec_sc_pkts_delayed += rd32(hw, NGBE_LSECRX_DLYPKT);
1413         hw_stats->rx_macsec_sc_pkts_late += rd32(hw, NGBE_LSECRX_LATEPKT);
1414         for (i = 0; i < 2; i++) {
1415                 hw_stats->rx_macsec_sa_pkts_ok +=
1416                         rd32(hw, NGBE_LSECRX_OKPKT(i));
1417                 hw_stats->rx_macsec_sa_pkts_invalid +=
1418                         rd32(hw, NGBE_LSECRX_INVPKT(i));
1419                 hw_stats->rx_macsec_sa_pkts_notvalid +=
1420                         rd32(hw, NGBE_LSECRX_BADPKT(i));
1421         }
1422         for (i = 0; i < 4; i++) {
1423                 hw_stats->rx_macsec_sa_pkts_unusedsa +=
1424                         rd32(hw, NGBE_LSECRX_INVSAPKT(i));
1425                 hw_stats->rx_macsec_sa_pkts_notusingsa +=
1426                         rd32(hw, NGBE_LSECRX_BADSAPKT(i));
1427         }
1428         hw_stats->rx_total_missed_packets =
1429                         hw_stats->rx_up_dropped;
1430 }
1431
1432 static int
1433 ngbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1434 {
1435         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1436         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1437         struct ngbe_stat_mappings *stat_mappings =
1438                         NGBE_DEV_STAT_MAPPINGS(dev);
1439         uint32_t i, j;
1440
1441         ngbe_read_stats_registers(hw, hw_stats);
1442
1443         if (stats == NULL)
1444                 return -EINVAL;
1445
1446         /* Fill out the rte_eth_stats statistics structure */
1447         stats->ipackets = hw_stats->rx_packets;
1448         stats->ibytes = hw_stats->rx_bytes;
1449         stats->opackets = hw_stats->tx_packets;
1450         stats->obytes = hw_stats->tx_bytes;
1451
1452         memset(&stats->q_ipackets, 0, sizeof(stats->q_ipackets));
1453         memset(&stats->q_opackets, 0, sizeof(stats->q_opackets));
1454         memset(&stats->q_ibytes, 0, sizeof(stats->q_ibytes));
1455         memset(&stats->q_obytes, 0, sizeof(stats->q_obytes));
1456         memset(&stats->q_errors, 0, sizeof(stats->q_errors));
1457         for (i = 0; i < NGBE_MAX_QP; i++) {
1458                 uint32_t n = i / NB_QMAP_FIELDS_PER_QSM_REG;
1459                 uint32_t offset = (i % NB_QMAP_FIELDS_PER_QSM_REG) * 8;
1460                 uint32_t q_map;
1461
1462                 q_map = (stat_mappings->rqsm[n] >> offset)
1463                                 & QMAP_FIELD_RESERVED_BITS_MASK;
1464                 j = (q_map < RTE_ETHDEV_QUEUE_STAT_CNTRS
1465                      ? q_map : q_map % RTE_ETHDEV_QUEUE_STAT_CNTRS);
1466                 stats->q_ipackets[j] += hw_stats->qp[i].rx_qp_packets;
1467                 stats->q_ibytes[j] += hw_stats->qp[i].rx_qp_bytes;
1468
1469                 q_map = (stat_mappings->tqsm[n] >> offset)
1470                                 & QMAP_FIELD_RESERVED_BITS_MASK;
1471                 j = (q_map < RTE_ETHDEV_QUEUE_STAT_CNTRS
1472                      ? q_map : q_map % RTE_ETHDEV_QUEUE_STAT_CNTRS);
1473                 stats->q_opackets[j] += hw_stats->qp[i].tx_qp_packets;
1474                 stats->q_obytes[j] += hw_stats->qp[i].tx_qp_bytes;
1475         }
1476
1477         /* Rx Errors */
1478         stats->imissed  = hw_stats->rx_total_missed_packets +
1479                           hw_stats->rx_dma_drop;
1480         stats->ierrors  = hw_stats->rx_crc_errors +
1481                           hw_stats->rx_mac_short_packet_dropped +
1482                           hw_stats->rx_length_errors +
1483                           hw_stats->rx_undersize_errors +
1484                           hw_stats->rx_oversize_errors +
1485                           hw_stats->rx_illegal_byte_errors +
1486                           hw_stats->rx_error_bytes +
1487                           hw_stats->rx_fragment_errors;
1488
1489         /* Tx Errors */
1490         stats->oerrors  = 0;
1491         return 0;
1492 }
1493
1494 static int
1495 ngbe_dev_stats_reset(struct rte_eth_dev *dev)
1496 {
1497         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1498         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1499
1500         /* HW registers are cleared on read */
1501         hw->offset_loaded = 0;
1502         ngbe_dev_stats_get(dev, NULL);
1503         hw->offset_loaded = 1;
1504
1505         /* Reset software totals */
1506         memset(hw_stats, 0, sizeof(*hw_stats));
1507
1508         return 0;
1509 }
1510
1511 /* This function calculates the number of xstats based on the current config */
1512 static unsigned
1513 ngbe_xstats_calc_num(struct rte_eth_dev *dev)
1514 {
1515         int nb_queues = max(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
1516         return NGBE_NB_HW_STATS +
1517                NGBE_NB_QP_STATS * nb_queues;
1518 }
1519
1520 static inline int
1521 ngbe_get_name_by_id(uint32_t id, char *name, uint32_t size)
1522 {
1523         int nb, st;
1524
1525         /* Extended stats from ngbe_hw_stats */
1526         if (id < NGBE_NB_HW_STATS) {
1527                 snprintf(name, size, "[hw]%s",
1528                         rte_ngbe_stats_strings[id].name);
1529                 return 0;
1530         }
1531         id -= NGBE_NB_HW_STATS;
1532
1533         /* Queue Stats */
1534         if (id < NGBE_NB_QP_STATS * NGBE_MAX_QP) {
1535                 nb = id / NGBE_NB_QP_STATS;
1536                 st = id % NGBE_NB_QP_STATS;
1537                 snprintf(name, size, "[q%u]%s", nb,
1538                         rte_ngbe_qp_strings[st].name);
1539                 return 0;
1540         }
1541         id -= NGBE_NB_QP_STATS * NGBE_MAX_QP;
1542
1543         return -(int)(id + 1);
1544 }
1545
1546 static inline int
1547 ngbe_get_offset_by_id(uint32_t id, uint32_t *offset)
1548 {
1549         int nb, st;
1550
1551         /* Extended stats from ngbe_hw_stats */
1552         if (id < NGBE_NB_HW_STATS) {
1553                 *offset = rte_ngbe_stats_strings[id].offset;
1554                 return 0;
1555         }
1556         id -= NGBE_NB_HW_STATS;
1557
1558         /* Queue Stats */
1559         if (id < NGBE_NB_QP_STATS * NGBE_MAX_QP) {
1560                 nb = id / NGBE_NB_QP_STATS;
1561                 st = id % NGBE_NB_QP_STATS;
1562                 *offset = rte_ngbe_qp_strings[st].offset +
1563                         nb * (NGBE_NB_QP_STATS * sizeof(uint64_t));
1564                 return 0;
1565         }
1566
1567         return -1;
1568 }
1569
1570 static int ngbe_dev_xstats_get_names(struct rte_eth_dev *dev,
1571         struct rte_eth_xstat_name *xstats_names, unsigned int limit)
1572 {
1573         unsigned int i, count;
1574
1575         count = ngbe_xstats_calc_num(dev);
1576         if (xstats_names == NULL)
1577                 return count;
1578
1579         /* Note: limit >= cnt_stats checked upstream
1580          * in rte_eth_xstats_names()
1581          */
1582         limit = min(limit, count);
1583
1584         /* Extended stats from ngbe_hw_stats */
1585         for (i = 0; i < limit; i++) {
1586                 if (ngbe_get_name_by_id(i, xstats_names[i].name,
1587                         sizeof(xstats_names[i].name))) {
1588                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1589                         break;
1590                 }
1591         }
1592
1593         return i;
1594 }
1595
1596 static int ngbe_dev_xstats_get_names_by_id(struct rte_eth_dev *dev,
1597         const uint64_t *ids,
1598         struct rte_eth_xstat_name *xstats_names,
1599         unsigned int limit)
1600 {
1601         unsigned int i;
1602
1603         if (ids == NULL)
1604                 return ngbe_dev_xstats_get_names(dev, xstats_names, limit);
1605
1606         for (i = 0; i < limit; i++) {
1607                 if (ngbe_get_name_by_id(ids[i], xstats_names[i].name,
1608                                 sizeof(xstats_names[i].name))) {
1609                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1610                         return -1;
1611                 }
1612         }
1613
1614         return i;
1615 }
1616
1617 static int
1618 ngbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1619                                          unsigned int limit)
1620 {
1621         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1622         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1623         unsigned int i, count;
1624
1625         ngbe_read_stats_registers(hw, hw_stats);
1626
1627         /* If this is a reset xstats is NULL, and we have cleared the
1628          * registers by reading them.
1629          */
1630         count = ngbe_xstats_calc_num(dev);
1631         if (xstats == NULL)
1632                 return count;
1633
1634         limit = min(limit, ngbe_xstats_calc_num(dev));
1635
1636         /* Extended stats from ngbe_hw_stats */
1637         for (i = 0; i < limit; i++) {
1638                 uint32_t offset = 0;
1639
1640                 if (ngbe_get_offset_by_id(i, &offset)) {
1641                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1642                         break;
1643                 }
1644                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + offset);
1645                 xstats[i].id = i;
1646         }
1647
1648         return i;
1649 }
1650
1651 static int
1652 ngbe_dev_xstats_get_(struct rte_eth_dev *dev, uint64_t *values,
1653                                          unsigned int limit)
1654 {
1655         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1656         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1657         unsigned int i, count;
1658
1659         ngbe_read_stats_registers(hw, hw_stats);
1660
1661         /* If this is a reset xstats is NULL, and we have cleared the
1662          * registers by reading them.
1663          */
1664         count = ngbe_xstats_calc_num(dev);
1665         if (values == NULL)
1666                 return count;
1667
1668         limit = min(limit, ngbe_xstats_calc_num(dev));
1669
1670         /* Extended stats from ngbe_hw_stats */
1671         for (i = 0; i < limit; i++) {
1672                 uint32_t offset;
1673
1674                 if (ngbe_get_offset_by_id(i, &offset)) {
1675                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1676                         break;
1677                 }
1678                 values[i] = *(uint64_t *)(((char *)hw_stats) + offset);
1679         }
1680
1681         return i;
1682 }
1683
1684 static int
1685 ngbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1686                 uint64_t *values, unsigned int limit)
1687 {
1688         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1689         unsigned int i;
1690
1691         if (ids == NULL)
1692                 return ngbe_dev_xstats_get_(dev, values, limit);
1693
1694         for (i = 0; i < limit; i++) {
1695                 uint32_t offset;
1696
1697                 if (ngbe_get_offset_by_id(ids[i], &offset)) {
1698                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1699                         break;
1700                 }
1701                 values[i] = *(uint64_t *)(((char *)hw_stats) + offset);
1702         }
1703
1704         return i;
1705 }
1706
1707 static int
1708 ngbe_dev_xstats_reset(struct rte_eth_dev *dev)
1709 {
1710         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1711         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1712
1713         /* HW registers are cleared on read */
1714         hw->offset_loaded = 0;
1715         ngbe_read_stats_registers(hw, hw_stats);
1716         hw->offset_loaded = 1;
1717
1718         /* Reset software totals */
1719         memset(hw_stats, 0, sizeof(*hw_stats));
1720
1721         return 0;
1722 }
1723
1724 static int
1725 ngbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
1726 {
1727         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1728         int ret;
1729
1730         ret = snprintf(fw_version, fw_size, "0x%08x", hw->eeprom_id);
1731
1732         if (ret < 0)
1733                 return -EINVAL;
1734
1735         ret += 1; /* add the size of '\0' */
1736         if (fw_size < (size_t)ret)
1737                 return ret;
1738
1739         return 0;
1740 }
1741
1742 static int
1743 ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1744 {
1745         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1746         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1747
1748         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1749         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1750         dev_info->min_rx_bufsize = 1024;
1751         dev_info->max_rx_pktlen = 15872;
1752         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1753         dev_info->max_hash_mac_addrs = NGBE_VMDQ_NUM_UC_MAC;
1754         dev_info->max_vfs = pci_dev->max_vfs;
1755         dev_info->rx_queue_offload_capa = ngbe_get_rx_queue_offloads(dev);
1756         dev_info->rx_offload_capa = (ngbe_get_rx_port_offloads(dev) |
1757                                      dev_info->rx_queue_offload_capa);
1758         dev_info->tx_queue_offload_capa = 0;
1759         dev_info->tx_offload_capa = ngbe_get_tx_port_offloads(dev);
1760
1761         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1762                 .rx_thresh = {
1763                         .pthresh = NGBE_DEFAULT_RX_PTHRESH,
1764                         .hthresh = NGBE_DEFAULT_RX_HTHRESH,
1765                         .wthresh = NGBE_DEFAULT_RX_WTHRESH,
1766                 },
1767                 .rx_free_thresh = NGBE_DEFAULT_RX_FREE_THRESH,
1768                 .rx_drop_en = 0,
1769                 .offloads = 0,
1770         };
1771
1772         dev_info->default_txconf = (struct rte_eth_txconf) {
1773                 .tx_thresh = {
1774                         .pthresh = NGBE_DEFAULT_TX_PTHRESH,
1775                         .hthresh = NGBE_DEFAULT_TX_HTHRESH,
1776                         .wthresh = NGBE_DEFAULT_TX_WTHRESH,
1777                 },
1778                 .tx_free_thresh = NGBE_DEFAULT_TX_FREE_THRESH,
1779                 .offloads = 0,
1780         };
1781
1782         dev_info->rx_desc_lim = rx_desc_lim;
1783         dev_info->tx_desc_lim = tx_desc_lim;
1784
1785         dev_info->hash_key_size = NGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
1786         dev_info->reta_size = RTE_ETH_RSS_RETA_SIZE_128;
1787         dev_info->flow_type_rss_offloads = NGBE_RSS_OFFLOAD_ALL;
1788
1789         dev_info->speed_capa = RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_100M |
1790                                 RTE_ETH_LINK_SPEED_10M;
1791
1792         /* Driver-preferred Rx/Tx parameters */
1793         dev_info->default_rxportconf.burst_size = 32;
1794         dev_info->default_txportconf.burst_size = 32;
1795         dev_info->default_rxportconf.nb_queues = 1;
1796         dev_info->default_txportconf.nb_queues = 1;
1797         dev_info->default_rxportconf.ring_size = 256;
1798         dev_info->default_txportconf.ring_size = 256;
1799
1800         return 0;
1801 }
1802
1803 const uint32_t *
1804 ngbe_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1805 {
1806         if (dev->rx_pkt_burst == ngbe_recv_pkts ||
1807             dev->rx_pkt_burst == ngbe_recv_pkts_sc_single_alloc ||
1808             dev->rx_pkt_burst == ngbe_recv_pkts_sc_bulk_alloc ||
1809             dev->rx_pkt_burst == ngbe_recv_pkts_bulk_alloc)
1810                 return ngbe_get_supported_ptypes();
1811
1812         return NULL;
1813 }
1814
1815 void
1816 ngbe_dev_setup_link_alarm_handler(void *param)
1817 {
1818         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1819         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1820         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1821         u32 speed;
1822         bool autoneg = false;
1823
1824         speed = hw->phy.autoneg_advertised;
1825         if (!speed)
1826                 hw->mac.get_link_capabilities(hw, &speed, &autoneg);
1827
1828         hw->mac.setup_link(hw, speed, true);
1829
1830         intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
1831 }
1832
1833 /* return 0 means link status changed, -1 means not changed */
1834 int
1835 ngbe_dev_link_update_share(struct rte_eth_dev *dev,
1836                             int wait_to_complete)
1837 {
1838         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1839         struct rte_eth_link link;
1840         u32 link_speed = NGBE_LINK_SPEED_UNKNOWN;
1841         u32 lan_speed = 0;
1842         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1843         bool link_up;
1844         int err;
1845         int wait = 1;
1846
1847         memset(&link, 0, sizeof(link));
1848         link.link_status = RTE_ETH_LINK_DOWN;
1849         link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1850         link.link_duplex = RTE_ETH_LINK_HALF_DUPLEX;
1851         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
1852                         ~RTE_ETH_LINK_SPEED_AUTONEG);
1853
1854         hw->mac.get_link_status = true;
1855
1856         if (intr->flags & NGBE_FLAG_NEED_LINK_CONFIG)
1857                 return rte_eth_linkstatus_set(dev, &link);
1858
1859         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1860         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1861                 wait = 0;
1862
1863         err = hw->mac.check_link(hw, &link_speed, &link_up, wait);
1864         if (err != 0) {
1865                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1866                 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1867                 return rte_eth_linkstatus_set(dev, &link);
1868         }
1869
1870         if (!link_up) {
1871                 if (hw->phy.media_type == ngbe_media_type_fiber &&
1872                         hw->phy.type != ngbe_phy_mvl_sfi) {
1873                         intr->flags |= NGBE_FLAG_NEED_LINK_CONFIG;
1874                         rte_eal_alarm_set(10,
1875                                 ngbe_dev_setup_link_alarm_handler, dev);
1876                 }
1877
1878                 return rte_eth_linkstatus_set(dev, &link);
1879         }
1880
1881         intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
1882         link.link_status = RTE_ETH_LINK_UP;
1883         link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1884
1885         switch (link_speed) {
1886         default:
1887         case NGBE_LINK_SPEED_UNKNOWN:
1888                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1889                 break;
1890
1891         case NGBE_LINK_SPEED_10M_FULL:
1892                 link.link_speed = RTE_ETH_SPEED_NUM_10M;
1893                 lan_speed = 0;
1894                 break;
1895
1896         case NGBE_LINK_SPEED_100M_FULL:
1897                 link.link_speed = RTE_ETH_SPEED_NUM_100M;
1898                 lan_speed = 1;
1899                 break;
1900
1901         case NGBE_LINK_SPEED_1GB_FULL:
1902                 link.link_speed = RTE_ETH_SPEED_NUM_1G;
1903                 lan_speed = 2;
1904                 break;
1905         }
1906
1907         if (hw->is_pf) {
1908                 wr32m(hw, NGBE_LAN_SPEED, NGBE_LAN_SPEED_MASK, lan_speed);
1909                 if (link_speed & (NGBE_LINK_SPEED_1GB_FULL |
1910                                 NGBE_LINK_SPEED_100M_FULL |
1911                                 NGBE_LINK_SPEED_10M_FULL)) {
1912                         wr32m(hw, NGBE_MACTXCFG, NGBE_MACTXCFG_SPEED_MASK,
1913                                 NGBE_MACTXCFG_SPEED_1G | NGBE_MACTXCFG_TE);
1914                 }
1915         }
1916
1917         return rte_eth_linkstatus_set(dev, &link);
1918 }
1919
1920 static int
1921 ngbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1922 {
1923         return ngbe_dev_link_update_share(dev, wait_to_complete);
1924 }
1925
1926 static int
1927 ngbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1928 {
1929         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1930         uint32_t fctrl;
1931
1932         fctrl = rd32(hw, NGBE_PSRCTL);
1933         fctrl |= (NGBE_PSRCTL_UCP | NGBE_PSRCTL_MCP);
1934         wr32(hw, NGBE_PSRCTL, fctrl);
1935
1936         return 0;
1937 }
1938
1939 static int
1940 ngbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1941 {
1942         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1943         uint32_t fctrl;
1944
1945         fctrl = rd32(hw, NGBE_PSRCTL);
1946         fctrl &= (~NGBE_PSRCTL_UCP);
1947         if (dev->data->all_multicast == 1)
1948                 fctrl |= NGBE_PSRCTL_MCP;
1949         else
1950                 fctrl &= (~NGBE_PSRCTL_MCP);
1951         wr32(hw, NGBE_PSRCTL, fctrl);
1952
1953         return 0;
1954 }
1955
1956 static int
1957 ngbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1958 {
1959         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1960         uint32_t fctrl;
1961
1962         fctrl = rd32(hw, NGBE_PSRCTL);
1963         fctrl |= NGBE_PSRCTL_MCP;
1964         wr32(hw, NGBE_PSRCTL, fctrl);
1965
1966         return 0;
1967 }
1968
1969 static int
1970 ngbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1971 {
1972         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1973         uint32_t fctrl;
1974
1975         if (dev->data->promiscuous == 1)
1976                 return 0; /* must remain in all_multicast mode */
1977
1978         fctrl = rd32(hw, NGBE_PSRCTL);
1979         fctrl &= (~NGBE_PSRCTL_MCP);
1980         wr32(hw, NGBE_PSRCTL, fctrl);
1981
1982         return 0;
1983 }
1984
1985 /**
1986  * It clears the interrupt causes and enables the interrupt.
1987  * It will be called once only during NIC initialized.
1988  *
1989  * @param dev
1990  *  Pointer to struct rte_eth_dev.
1991  * @param on
1992  *  Enable or Disable.
1993  *
1994  * @return
1995  *  - On success, zero.
1996  *  - On failure, a negative value.
1997  */
1998 static int
1999 ngbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
2000 {
2001         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2002
2003         ngbe_dev_link_status_print(dev);
2004         if (on != 0) {
2005                 intr->mask_misc |= NGBE_ICRMISC_PHY;
2006                 intr->mask_misc |= NGBE_ICRMISC_GPIO;
2007         } else {
2008                 intr->mask_misc &= ~NGBE_ICRMISC_PHY;
2009                 intr->mask_misc &= ~NGBE_ICRMISC_GPIO;
2010         }
2011
2012         return 0;
2013 }
2014
2015 /**
2016  * It clears the interrupt causes and enables the interrupt.
2017  * It will be called once only during NIC initialized.
2018  *
2019  * @param dev
2020  *  Pointer to struct rte_eth_dev.
2021  *
2022  * @return
2023  *  - On success, zero.
2024  *  - On failure, a negative value.
2025  */
2026 static int
2027 ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev)
2028 {
2029         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2030         u64 mask;
2031
2032         mask = NGBE_ICR_MASK;
2033         mask &= (1ULL << NGBE_MISC_VEC_ID);
2034         intr->mask |= mask;
2035         intr->mask_misc |= NGBE_ICRMISC_GPIO;
2036
2037         return 0;
2038 }
2039
2040 /**
2041  * It clears the interrupt causes and enables the interrupt.
2042  * It will be called once only during NIC initialized.
2043  *
2044  * @param dev
2045  *  Pointer to struct rte_eth_dev.
2046  *
2047  * @return
2048  *  - On success, zero.
2049  *  - On failure, a negative value.
2050  */
2051 static int
2052 ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2053 {
2054         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2055         u64 mask;
2056
2057         mask = NGBE_ICR_MASK;
2058         mask &= ~((1ULL << NGBE_RX_VEC_START) - 1);
2059         intr->mask |= mask;
2060
2061         return 0;
2062 }
2063
2064 /**
2065  * It clears the interrupt causes and enables the interrupt.
2066  * It will be called once only during NIC initialized.
2067  *
2068  * @param dev
2069  *  Pointer to struct rte_eth_dev.
2070  *
2071  * @return
2072  *  - On success, zero.
2073  *  - On failure, a negative value.
2074  */
2075 static int
2076 ngbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev)
2077 {
2078         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2079
2080         intr->mask_misc |= NGBE_ICRMISC_LNKSEC;
2081
2082         return 0;
2083 }
2084
2085 /*
2086  * It reads ICR and sets flag for the link_update.
2087  *
2088  * @param dev
2089  *  Pointer to struct rte_eth_dev.
2090  *
2091  * @return
2092  *  - On success, zero.
2093  *  - On failure, a negative value.
2094  */
2095 static int
2096 ngbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2097 {
2098         uint32_t eicr;
2099         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2100         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2101
2102         /* read-on-clear nic registers here */
2103         eicr = ((u32 *)hw->isb_mem)[NGBE_ISB_MISC];
2104         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
2105
2106         intr->flags = 0;
2107
2108         /* set flag for async link update */
2109         if (eicr & NGBE_ICRMISC_PHY)
2110                 intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
2111
2112         if (eicr & NGBE_ICRMISC_VFMBX)
2113                 intr->flags |= NGBE_FLAG_MAILBOX;
2114
2115         if (eicr & NGBE_ICRMISC_LNKSEC)
2116                 intr->flags |= NGBE_FLAG_MACSEC;
2117
2118         if (eicr & NGBE_ICRMISC_GPIO)
2119                 intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
2120
2121         ((u32 *)hw->isb_mem)[NGBE_ISB_MISC] = 0;
2122
2123         return 0;
2124 }
2125
2126 /**
2127  * It gets and then prints the link status.
2128  *
2129  * @param dev
2130  *  Pointer to struct rte_eth_dev.
2131  *
2132  * @return
2133  *  - On success, zero.
2134  *  - On failure, a negative value.
2135  */
2136 static void
2137 ngbe_dev_link_status_print(struct rte_eth_dev *dev)
2138 {
2139         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2140         struct rte_eth_link link;
2141
2142         rte_eth_linkstatus_get(dev, &link);
2143
2144         if (link.link_status == RTE_ETH_LINK_UP) {
2145                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2146                                         (int)(dev->data->port_id),
2147                                         (unsigned int)link.link_speed,
2148                         link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX ?
2149                                         "full-duplex" : "half-duplex");
2150         } else {
2151                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2152                                 (int)(dev->data->port_id));
2153         }
2154         PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
2155                                 pci_dev->addr.domain,
2156                                 pci_dev->addr.bus,
2157                                 pci_dev->addr.devid,
2158                                 pci_dev->addr.function);
2159 }
2160
2161 /*
2162  * It executes link_update after knowing an interrupt occurred.
2163  *
2164  * @param dev
2165  *  Pointer to struct rte_eth_dev.
2166  *
2167  * @return
2168  *  - On success, zero.
2169  *  - On failure, a negative value.
2170  */
2171 static int
2172 ngbe_dev_interrupt_action(struct rte_eth_dev *dev)
2173 {
2174         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2175
2176         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2177
2178         if (intr->flags & NGBE_FLAG_MAILBOX) {
2179                 ngbe_pf_mbx_process(dev);
2180                 intr->flags &= ~NGBE_FLAG_MAILBOX;
2181         }
2182
2183         if (intr->flags & NGBE_FLAG_NEED_LINK_UPDATE) {
2184                 struct rte_eth_link link;
2185
2186                 /*get the link status before link update, for predicting later*/
2187                 rte_eth_linkstatus_get(dev, &link);
2188
2189                 ngbe_dev_link_update(dev, 0);
2190                 intr->flags &= ~NGBE_FLAG_NEED_LINK_UPDATE;
2191                 ngbe_dev_link_status_print(dev);
2192                 if (dev->data->dev_link.link_speed != link.link_speed)
2193                         rte_eth_dev_callback_process(dev,
2194                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2195         }
2196
2197         PMD_DRV_LOG(DEBUG, "enable intr immediately");
2198         ngbe_enable_intr(dev);
2199
2200         return 0;
2201 }
2202
2203 /**
2204  * Interrupt handler triggered by NIC  for handling
2205  * specific interrupt.
2206  *
2207  * @param param
2208  *  The address of parameter (struct rte_eth_dev *) registered before.
2209  */
2210 static void
2211 ngbe_dev_interrupt_handler(void *param)
2212 {
2213         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2214
2215         ngbe_dev_interrupt_get_status(dev);
2216         ngbe_dev_interrupt_action(dev);
2217 }
2218
2219 static int
2220 ngbe_dev_led_on(struct rte_eth_dev *dev)
2221 {
2222         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2223         return hw->mac.led_on(hw, 0) == 0 ? 0 : -ENOTSUP;
2224 }
2225
2226 static int
2227 ngbe_dev_led_off(struct rte_eth_dev *dev)
2228 {
2229         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2230         return hw->mac.led_off(hw, 0) == 0 ? 0 : -ENOTSUP;
2231 }
2232
2233 static int
2234 ngbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2235 {
2236         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2237         uint32_t mflcn_reg;
2238         uint32_t fccfg_reg;
2239         int rx_pause;
2240         int tx_pause;
2241
2242         fc_conf->pause_time = hw->fc.pause_time;
2243         fc_conf->high_water = hw->fc.high_water;
2244         fc_conf->low_water = hw->fc.low_water;
2245         fc_conf->send_xon = hw->fc.send_xon;
2246         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2247
2248         /*
2249          * Return rx_pause status according to actual setting of
2250          * RXFCCFG register.
2251          */
2252         mflcn_reg = rd32(hw, NGBE_RXFCCFG);
2253         if (mflcn_reg & NGBE_RXFCCFG_FC)
2254                 rx_pause = 1;
2255         else
2256                 rx_pause = 0;
2257
2258         /*
2259          * Return tx_pause status according to actual setting of
2260          * TXFCCFG register.
2261          */
2262         fccfg_reg = rd32(hw, NGBE_TXFCCFG);
2263         if (fccfg_reg & NGBE_TXFCCFG_FC)
2264                 tx_pause = 1;
2265         else
2266                 tx_pause = 0;
2267
2268         if (rx_pause && tx_pause)
2269                 fc_conf->mode = RTE_ETH_FC_FULL;
2270         else if (rx_pause)
2271                 fc_conf->mode = RTE_ETH_FC_RX_PAUSE;
2272         else if (tx_pause)
2273                 fc_conf->mode = RTE_ETH_FC_TX_PAUSE;
2274         else
2275                 fc_conf->mode = RTE_ETH_FC_NONE;
2276
2277         return 0;
2278 }
2279
2280 static int
2281 ngbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2282 {
2283         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2284         int err;
2285         uint32_t rx_buf_size;
2286         uint32_t max_high_water;
2287         enum ngbe_fc_mode rte_fcmode_2_ngbe_fcmode[] = {
2288                 ngbe_fc_none,
2289                 ngbe_fc_rx_pause,
2290                 ngbe_fc_tx_pause,
2291                 ngbe_fc_full
2292         };
2293
2294         PMD_INIT_FUNC_TRACE();
2295
2296         rx_buf_size = rd32(hw, NGBE_PBRXSIZE);
2297         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2298
2299         /*
2300          * At least reserve one Ethernet frame for watermark
2301          * high_water/low_water in kilo bytes for ngbe
2302          */
2303         max_high_water = (rx_buf_size - RTE_ETHER_MAX_LEN) >> 10;
2304         if (fc_conf->high_water > max_high_water ||
2305             fc_conf->high_water < fc_conf->low_water) {
2306                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2307                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2308                 return -EINVAL;
2309         }
2310
2311         hw->fc.requested_mode = rte_fcmode_2_ngbe_fcmode[fc_conf->mode];
2312         hw->fc.pause_time     = fc_conf->pause_time;
2313         hw->fc.high_water     = fc_conf->high_water;
2314         hw->fc.low_water      = fc_conf->low_water;
2315         hw->fc.send_xon       = fc_conf->send_xon;
2316         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
2317
2318         err = hw->mac.fc_enable(hw);
2319
2320         /* Not negotiated is not an error case */
2321         if (err == 0 || err == NGBE_ERR_FC_NOT_NEGOTIATED) {
2322                 wr32m(hw, NGBE_MACRXFLT, NGBE_MACRXFLT_CTL_MASK,
2323                       (fc_conf->mac_ctrl_frame_fwd
2324                        ? NGBE_MACRXFLT_CTL_NOPS : NGBE_MACRXFLT_CTL_DROP));
2325                 ngbe_flush(hw);
2326
2327                 return 0;
2328         }
2329
2330         PMD_INIT_LOG(ERR, "ngbe_fc_enable = 0x%x", err);
2331         return -EIO;
2332 }
2333
2334 int
2335 ngbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2336                           struct rte_eth_rss_reta_entry64 *reta_conf,
2337                           uint16_t reta_size)
2338 {
2339         uint8_t i, j, mask;
2340         uint32_t reta;
2341         uint16_t idx, shift;
2342         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2343         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2344
2345         PMD_INIT_FUNC_TRACE();
2346
2347         if (!hw->is_pf) {
2348                 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
2349                         "NIC.");
2350                 return -ENOTSUP;
2351         }
2352
2353         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2354                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2355                         "(%d) doesn't match the number hardware can supported "
2356                         "(%d)", reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2357                 return -EINVAL;
2358         }
2359
2360         for (i = 0; i < reta_size; i += 4) {
2361                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2362                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2363                 mask = (uint8_t)RS64(reta_conf[idx].mask, shift, 0xF);
2364                 if (!mask)
2365                         continue;
2366
2367                 reta = rd32a(hw, NGBE_REG_RSSTBL, i >> 2);
2368                 for (j = 0; j < 4; j++) {
2369                         if (RS8(mask, j, 0x1)) {
2370                                 reta  &= ~(MS32(8 * j, 0xFF));
2371                                 reta |= LS32(reta_conf[idx].reta[shift + j],
2372                                                 8 * j, 0xFF);
2373                         }
2374                 }
2375                 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta);
2376         }
2377         adapter->rss_reta_updated = 1;
2378
2379         return 0;
2380 }
2381
2382 int
2383 ngbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2384                          struct rte_eth_rss_reta_entry64 *reta_conf,
2385                          uint16_t reta_size)
2386 {
2387         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2388         uint8_t i, j, mask;
2389         uint32_t reta;
2390         uint16_t idx, shift;
2391
2392         PMD_INIT_FUNC_TRACE();
2393
2394         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2395                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2396                         "(%d) doesn't match the number hardware can supported "
2397                         "(%d)", reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2398                 return -EINVAL;
2399         }
2400
2401         for (i = 0; i < reta_size; i += 4) {
2402                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2403                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2404                 mask = (uint8_t)RS64(reta_conf[idx].mask, shift, 0xF);
2405                 if (!mask)
2406                         continue;
2407
2408                 reta = rd32a(hw, NGBE_REG_RSSTBL, i >> 2);
2409                 for (j = 0; j < 4; j++) {
2410                         if (RS8(mask, j, 0x1))
2411                                 reta_conf[idx].reta[shift + j] =
2412                                         (uint16_t)RS32(reta, 8 * j, 0xFF);
2413                 }
2414         }
2415
2416         return 0;
2417 }
2418
2419 static int
2420 ngbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
2421                                 uint32_t index, uint32_t pool)
2422 {
2423         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2424         uint32_t enable_addr = 1;
2425
2426         return ngbe_set_rar(hw, index, mac_addr->addr_bytes,
2427                              pool, enable_addr);
2428 }
2429
2430 static void
2431 ngbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2432 {
2433         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2434
2435         ngbe_clear_rar(hw, index);
2436 }
2437
2438 static int
2439 ngbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
2440 {
2441         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2442
2443         ngbe_remove_rar(dev, 0);
2444         ngbe_add_rar(dev, addr, 0, pci_dev->max_vfs);
2445
2446         return 0;
2447 }
2448
2449 static int
2450 ngbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
2451 {
2452         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2453         uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 4;
2454         struct rte_eth_dev_data *dev_data = dev->data;
2455
2456         /* If device is started, refuse mtu that requires the support of
2457          * scattered packets when this feature has not been enabled before.
2458          */
2459         if (dev_data->dev_started && !dev_data->scattered_rx &&
2460             (frame_size + 2 * RTE_VLAN_HLEN >
2461              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
2462                 PMD_INIT_LOG(ERR, "Stop port first.");
2463                 return -EINVAL;
2464         }
2465
2466         if (hw->mode)
2467                 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2468                         NGBE_FRAME_SIZE_MAX);
2469         else
2470                 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2471                         NGBE_FRMSZ_MAX(frame_size));
2472
2473         return 0;
2474 }
2475
2476 static uint32_t
2477 ngbe_uta_vector(struct ngbe_hw *hw, struct rte_ether_addr *uc_addr)
2478 {
2479         uint32_t vector = 0;
2480
2481         switch (hw->mac.mc_filter_type) {
2482         case 0:   /* use bits [47:36] of the address */
2483                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2484                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2485                 break;
2486         case 1:   /* use bits [46:35] of the address */
2487                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2488                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2489                 break;
2490         case 2:   /* use bits [45:34] of the address */
2491                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2492                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2493                 break;
2494         case 3:   /* use bits [43:32] of the address */
2495                 vector = ((uc_addr->addr_bytes[4]) |
2496                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2497                 break;
2498         default:  /* Invalid mc_filter_type */
2499                 break;
2500         }
2501
2502         /* vector can only be 12-bits or boundary will be exceeded */
2503         vector &= 0xFFF;
2504         return vector;
2505 }
2506
2507 static int
2508 ngbe_uc_hash_table_set(struct rte_eth_dev *dev,
2509                         struct rte_ether_addr *mac_addr, uint8_t on)
2510 {
2511         uint32_t vector;
2512         uint32_t uta_idx;
2513         uint32_t reg_val;
2514         uint32_t uta_mask;
2515         uint32_t psrctl;
2516
2517         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2518         struct ngbe_uta_info *uta_info = NGBE_DEV_UTA_INFO(dev);
2519
2520         vector = ngbe_uta_vector(hw, mac_addr);
2521         uta_idx = (vector >> 5) & 0x7F;
2522         uta_mask = 0x1UL << (vector & 0x1F);
2523
2524         if (!!on == !!(uta_info->uta_shadow[uta_idx] & uta_mask))
2525                 return 0;
2526
2527         reg_val = rd32(hw, NGBE_UCADDRTBL(uta_idx));
2528         if (on) {
2529                 uta_info->uta_in_use++;
2530                 reg_val |= uta_mask;
2531                 uta_info->uta_shadow[uta_idx] |= uta_mask;
2532         } else {
2533                 uta_info->uta_in_use--;
2534                 reg_val &= ~uta_mask;
2535                 uta_info->uta_shadow[uta_idx] &= ~uta_mask;
2536         }
2537
2538         wr32(hw, NGBE_UCADDRTBL(uta_idx), reg_val);
2539
2540         psrctl = rd32(hw, NGBE_PSRCTL);
2541         if (uta_info->uta_in_use > 0)
2542                 psrctl |= NGBE_PSRCTL_UCHFENA;
2543         else
2544                 psrctl &= ~NGBE_PSRCTL_UCHFENA;
2545
2546         psrctl &= ~NGBE_PSRCTL_ADHF12_MASK;
2547         psrctl |= NGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
2548         wr32(hw, NGBE_PSRCTL, psrctl);
2549
2550         return 0;
2551 }
2552
2553 static int
2554 ngbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2555 {
2556         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2557         struct ngbe_uta_info *uta_info = NGBE_DEV_UTA_INFO(dev);
2558         uint32_t psrctl;
2559         int i;
2560
2561         if (on) {
2562                 for (i = 0; i < RTE_ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2563                         uta_info->uta_shadow[i] = ~0;
2564                         wr32(hw, NGBE_UCADDRTBL(i), ~0);
2565                 }
2566         } else {
2567                 for (i = 0; i < RTE_ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2568                         uta_info->uta_shadow[i] = 0;
2569                         wr32(hw, NGBE_UCADDRTBL(i), 0);
2570                 }
2571         }
2572
2573         psrctl = rd32(hw, NGBE_PSRCTL);
2574         if (on)
2575                 psrctl |= NGBE_PSRCTL_UCHFENA;
2576         else
2577                 psrctl &= ~NGBE_PSRCTL_UCHFENA;
2578
2579         psrctl &= ~NGBE_PSRCTL_ADHF12_MASK;
2580         psrctl |= NGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
2581         wr32(hw, NGBE_PSRCTL, psrctl);
2582
2583         return 0;
2584 }
2585
2586 /**
2587  * Set the IVAR registers, mapping interrupt causes to vectors
2588  * @param hw
2589  *  pointer to ngbe_hw struct
2590  * @direction
2591  *  0 for Rx, 1 for Tx, -1 for other causes
2592  * @queue
2593  *  queue to map the corresponding interrupt to
2594  * @msix_vector
2595  *  the vector to map to the corresponding queue
2596  */
2597 void
2598 ngbe_set_ivar_map(struct ngbe_hw *hw, int8_t direction,
2599                    uint8_t queue, uint8_t msix_vector)
2600 {
2601         uint32_t tmp, idx;
2602
2603         if (direction == -1) {
2604                 /* other causes */
2605                 msix_vector |= NGBE_IVARMISC_VLD;
2606                 idx = 0;
2607                 tmp = rd32(hw, NGBE_IVARMISC);
2608                 tmp &= ~(0xFF << idx);
2609                 tmp |= (msix_vector << idx);
2610                 wr32(hw, NGBE_IVARMISC, tmp);
2611         } else {
2612                 /* rx or tx causes */
2613                 /* Workaround for ICR lost */
2614                 idx = ((16 * (queue & 1)) + (8 * direction));
2615                 tmp = rd32(hw, NGBE_IVAR(queue >> 1));
2616                 tmp &= ~(0xFF << idx);
2617                 tmp |= (msix_vector << idx);
2618                 wr32(hw, NGBE_IVAR(queue >> 1), tmp);
2619         }
2620 }
2621
2622 /**
2623  * Sets up the hardware to properly generate MSI-X interrupts
2624  * @hw
2625  *  board private structure
2626  */
2627 static void
2628 ngbe_configure_msix(struct rte_eth_dev *dev)
2629 {
2630         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2631         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
2632         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2633         uint32_t queue_id, base = NGBE_MISC_VEC_ID;
2634         uint32_t vec = NGBE_MISC_VEC_ID;
2635         uint32_t gpie;
2636
2637         /*
2638          * Won't configure MSI-X register if no mapping is done
2639          * between intr vector and event fd
2640          * but if MSI-X has been enabled already, need to configure
2641          * auto clean, auto mask and throttling.
2642          */
2643         gpie = rd32(hw, NGBE_GPIE);
2644         if (!rte_intr_dp_is_en(intr_handle) &&
2645             !(gpie & NGBE_GPIE_MSIX))
2646                 return;
2647
2648         if (rte_intr_allow_others(intr_handle)) {
2649                 base = NGBE_RX_VEC_START;
2650                 vec = base;
2651         }
2652
2653         /* setup GPIE for MSI-X mode */
2654         gpie = rd32(hw, NGBE_GPIE);
2655         gpie |= NGBE_GPIE_MSIX;
2656         wr32(hw, NGBE_GPIE, gpie);
2657
2658         /* Populate the IVAR table and set the ITR values to the
2659          * corresponding register.
2660          */
2661         if (rte_intr_dp_is_en(intr_handle)) {
2662                 for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
2663                         queue_id++) {
2664                         /* by default, 1:1 mapping */
2665                         ngbe_set_ivar_map(hw, 0, queue_id, vec);
2666                         rte_intr_vec_list_index_set(intr_handle,
2667                                                            queue_id, vec);
2668                         if (vec < base + rte_intr_nb_efd_get(intr_handle)
2669                             - 1)
2670                                 vec++;
2671                 }
2672
2673                 ngbe_set_ivar_map(hw, -1, 1, NGBE_MISC_VEC_ID);
2674         }
2675         wr32(hw, NGBE_ITR(NGBE_MISC_VEC_ID),
2676                         NGBE_ITR_IVAL_1G(NGBE_QUEUE_ITR_INTERVAL_DEFAULT)
2677                         | NGBE_ITR_WRDSA);
2678 }
2679
2680 static u8 *
2681 ngbe_dev_addr_list_itr(__rte_unused struct ngbe_hw *hw,
2682                         u8 **mc_addr_ptr, u32 *vmdq)
2683 {
2684         u8 *mc_addr;
2685
2686         *vmdq = 0;
2687         mc_addr = *mc_addr_ptr;
2688         *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr));
2689         return mc_addr;
2690 }
2691
2692 int
2693 ngbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
2694                           struct rte_ether_addr *mc_addr_set,
2695                           uint32_t nb_mc_addr)
2696 {
2697         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2698         u8 *mc_addr_list;
2699
2700         mc_addr_list = (u8 *)mc_addr_set;
2701         return hw->mac.update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
2702                                          ngbe_dev_addr_list_itr, TRUE);
2703 }
2704
2705 static uint64_t
2706 ngbe_read_systime_cyclecounter(struct rte_eth_dev *dev)
2707 {
2708         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2709         uint64_t systime_cycles;
2710
2711         systime_cycles = (uint64_t)rd32(hw, NGBE_TSTIMEL);
2712         systime_cycles |= (uint64_t)rd32(hw, NGBE_TSTIMEH) << 32;
2713
2714         return systime_cycles;
2715 }
2716
2717 static uint64_t
2718 ngbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
2719 {
2720         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2721         uint64_t rx_tstamp_cycles;
2722
2723         /* TSRXSTMPL stores ns and TSRXSTMPH stores seconds. */
2724         rx_tstamp_cycles = (uint64_t)rd32(hw, NGBE_TSRXSTMPL);
2725         rx_tstamp_cycles |= (uint64_t)rd32(hw, NGBE_TSRXSTMPH) << 32;
2726
2727         return rx_tstamp_cycles;
2728 }
2729
2730 static uint64_t
2731 ngbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
2732 {
2733         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2734         uint64_t tx_tstamp_cycles;
2735
2736         /* TSTXSTMPL stores ns and TSTXSTMPH stores seconds. */
2737         tx_tstamp_cycles = (uint64_t)rd32(hw, NGBE_TSTXSTMPL);
2738         tx_tstamp_cycles |= (uint64_t)rd32(hw, NGBE_TSTXSTMPH) << 32;
2739
2740         return tx_tstamp_cycles;
2741 }
2742
2743 static void
2744 ngbe_start_timecounters(struct rte_eth_dev *dev)
2745 {
2746         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2747         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2748         uint32_t incval = 0;
2749         uint32_t shift = 0;
2750
2751         incval = NGBE_INCVAL_1GB;
2752         shift = NGBE_INCVAL_SHIFT_1GB;
2753
2754         wr32(hw, NGBE_TSTIMEINC, NGBE_TSTIMEINC_IV(incval));
2755
2756         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
2757         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
2758         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
2759
2760         adapter->systime_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2761         adapter->systime_tc.cc_shift = shift;
2762         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
2763
2764         adapter->rx_tstamp_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2765         adapter->rx_tstamp_tc.cc_shift = shift;
2766         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
2767
2768         adapter->tx_tstamp_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2769         adapter->tx_tstamp_tc.cc_shift = shift;
2770         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
2771 }
2772
2773 static int
2774 ngbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
2775 {
2776         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2777
2778         adapter->systime_tc.nsec += delta;
2779         adapter->rx_tstamp_tc.nsec += delta;
2780         adapter->tx_tstamp_tc.nsec += delta;
2781
2782         return 0;
2783 }
2784
2785 static int
2786 ngbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
2787 {
2788         uint64_t ns;
2789         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2790
2791         ns = rte_timespec_to_ns(ts);
2792         /* Set the timecounters to a new value. */
2793         adapter->systime_tc.nsec = ns;
2794         adapter->rx_tstamp_tc.nsec = ns;
2795         adapter->tx_tstamp_tc.nsec = ns;
2796
2797         return 0;
2798 }
2799
2800 static int
2801 ngbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
2802 {
2803         uint64_t ns, systime_cycles;
2804         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2805
2806         systime_cycles = ngbe_read_systime_cyclecounter(dev);
2807         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
2808         *ts = rte_ns_to_timespec(ns);
2809
2810         return 0;
2811 }
2812
2813 static int
2814 ngbe_timesync_enable(struct rte_eth_dev *dev)
2815 {
2816         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2817         uint32_t tsync_ctl;
2818
2819         /* Stop the timesync system time. */
2820         wr32(hw, NGBE_TSTIMEINC, 0x0);
2821         /* Reset the timesync system time value. */
2822         wr32(hw, NGBE_TSTIMEL, 0x0);
2823         wr32(hw, NGBE_TSTIMEH, 0x0);
2824
2825         ngbe_start_timecounters(dev);
2826
2827         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
2828         wr32(hw, NGBE_ETFLT(NGBE_ETF_ID_1588),
2829                 RTE_ETHER_TYPE_1588 | NGBE_ETFLT_ENA | NGBE_ETFLT_1588);
2830
2831         /* Enable timestamping of received PTP packets. */
2832         tsync_ctl = rd32(hw, NGBE_TSRXCTL);
2833         tsync_ctl |= NGBE_TSRXCTL_ENA;
2834         wr32(hw, NGBE_TSRXCTL, tsync_ctl);
2835
2836         /* Enable timestamping of transmitted PTP packets. */
2837         tsync_ctl = rd32(hw, NGBE_TSTXCTL);
2838         tsync_ctl |= NGBE_TSTXCTL_ENA;
2839         wr32(hw, NGBE_TSTXCTL, tsync_ctl);
2840
2841         ngbe_flush(hw);
2842
2843         return 0;
2844 }
2845
2846 static int
2847 ngbe_timesync_disable(struct rte_eth_dev *dev)
2848 {
2849         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2850         uint32_t tsync_ctl;
2851
2852         /* Disable timestamping of transmitted PTP packets. */
2853         tsync_ctl = rd32(hw, NGBE_TSTXCTL);
2854         tsync_ctl &= ~NGBE_TSTXCTL_ENA;
2855         wr32(hw, NGBE_TSTXCTL, tsync_ctl);
2856
2857         /* Disable timestamping of received PTP packets. */
2858         tsync_ctl = rd32(hw, NGBE_TSRXCTL);
2859         tsync_ctl &= ~NGBE_TSRXCTL_ENA;
2860         wr32(hw, NGBE_TSRXCTL, tsync_ctl);
2861
2862         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
2863         wr32(hw, NGBE_ETFLT(NGBE_ETF_ID_1588), 0);
2864
2865         /* Stop incrementing the System Time registers. */
2866         wr32(hw, NGBE_TSTIMEINC, 0);
2867
2868         return 0;
2869 }
2870
2871 static int
2872 ngbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
2873                                  struct timespec *timestamp,
2874                                  uint32_t flags __rte_unused)
2875 {
2876         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2877         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2878         uint32_t tsync_rxctl;
2879         uint64_t rx_tstamp_cycles;
2880         uint64_t ns;
2881
2882         tsync_rxctl = rd32(hw, NGBE_TSRXCTL);
2883         if ((tsync_rxctl & NGBE_TSRXCTL_VLD) == 0)
2884                 return -EINVAL;
2885
2886         rx_tstamp_cycles = ngbe_read_rx_tstamp_cyclecounter(dev);
2887         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
2888         *timestamp = rte_ns_to_timespec(ns);
2889
2890         return  0;
2891 }
2892
2893 static int
2894 ngbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
2895                                  struct timespec *timestamp)
2896 {
2897         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2898         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2899         uint32_t tsync_txctl;
2900         uint64_t tx_tstamp_cycles;
2901         uint64_t ns;
2902
2903         tsync_txctl = rd32(hw, NGBE_TSTXCTL);
2904         if ((tsync_txctl & NGBE_TSTXCTL_VLD) == 0)
2905                 return -EINVAL;
2906
2907         tx_tstamp_cycles = ngbe_read_tx_tstamp_cyclecounter(dev);
2908         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
2909         *timestamp = rte_ns_to_timespec(ns);
2910
2911         return 0;
2912 }
2913
2914 static int
2915 ngbe_get_reg_length(struct rte_eth_dev *dev __rte_unused)
2916 {
2917         int count = 0;
2918         int g_ind = 0;
2919         const struct reg_info *reg_group;
2920         const struct reg_info **reg_set = ngbe_regs_others;
2921
2922         while ((reg_group = reg_set[g_ind++]))
2923                 count += ngbe_regs_group_count(reg_group);
2924
2925         return count;
2926 }
2927
2928 static int
2929 ngbe_get_regs(struct rte_eth_dev *dev,
2930               struct rte_dev_reg_info *regs)
2931 {
2932         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2933         uint32_t *data = regs->data;
2934         int g_ind = 0;
2935         int count = 0;
2936         const struct reg_info *reg_group;
2937         const struct reg_info **reg_set = ngbe_regs_others;
2938
2939         if (data == NULL) {
2940                 regs->length = ngbe_get_reg_length(dev);
2941                 regs->width = sizeof(uint32_t);
2942                 return 0;
2943         }
2944
2945         /* Support only full register dump */
2946         if (regs->length == 0 ||
2947             regs->length == (uint32_t)ngbe_get_reg_length(dev)) {
2948                 regs->version = hw->mac.type << 24 |
2949                                 hw->revision_id << 16 |
2950                                 hw->device_id;
2951                 while ((reg_group = reg_set[g_ind++]))
2952                         count += ngbe_read_regs_group(dev, &data[count],
2953                                                       reg_group);
2954                 return 0;
2955         }
2956
2957         return -ENOTSUP;
2958 }
2959
2960 static int
2961 ngbe_get_eeprom_length(struct rte_eth_dev *dev)
2962 {
2963         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2964
2965         /* Return unit is byte count */
2966         return hw->rom.word_size * 2;
2967 }
2968
2969 static int
2970 ngbe_get_eeprom(struct rte_eth_dev *dev,
2971                 struct rte_dev_eeprom_info *in_eeprom)
2972 {
2973         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2974         struct ngbe_rom_info *eeprom = &hw->rom;
2975         uint16_t *data = in_eeprom->data;
2976         int first, length;
2977
2978         first = in_eeprom->offset >> 1;
2979         length = in_eeprom->length >> 1;
2980         if (first > hw->rom.word_size ||
2981             ((first + length) > hw->rom.word_size))
2982                 return -EINVAL;
2983
2984         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
2985
2986         return eeprom->readw_buffer(hw, first, length, data);
2987 }
2988
2989 static int
2990 ngbe_set_eeprom(struct rte_eth_dev *dev,
2991                 struct rte_dev_eeprom_info *in_eeprom)
2992 {
2993         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2994         struct ngbe_rom_info *eeprom = &hw->rom;
2995         uint16_t *data = in_eeprom->data;
2996         int first, length;
2997
2998         first = in_eeprom->offset >> 1;
2999         length = in_eeprom->length >> 1;
3000         if (first > hw->rom.word_size ||
3001             ((first + length) > hw->rom.word_size))
3002                 return -EINVAL;
3003
3004         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
3005
3006         return eeprom->writew_buffer(hw,  first, length, data);
3007 }
3008
3009 static const struct eth_dev_ops ngbe_eth_dev_ops = {
3010         .dev_configure              = ngbe_dev_configure,
3011         .dev_infos_get              = ngbe_dev_info_get,
3012         .dev_start                  = ngbe_dev_start,
3013         .dev_stop                   = ngbe_dev_stop,
3014         .dev_close                  = ngbe_dev_close,
3015         .dev_reset                  = ngbe_dev_reset,
3016         .promiscuous_enable         = ngbe_dev_promiscuous_enable,
3017         .promiscuous_disable        = ngbe_dev_promiscuous_disable,
3018         .allmulticast_enable        = ngbe_dev_allmulticast_enable,
3019         .allmulticast_disable       = ngbe_dev_allmulticast_disable,
3020         .link_update                = ngbe_dev_link_update,
3021         .stats_get                  = ngbe_dev_stats_get,
3022         .xstats_get                 = ngbe_dev_xstats_get,
3023         .xstats_get_by_id           = ngbe_dev_xstats_get_by_id,
3024         .stats_reset                = ngbe_dev_stats_reset,
3025         .xstats_reset               = ngbe_dev_xstats_reset,
3026         .xstats_get_names           = ngbe_dev_xstats_get_names,
3027         .xstats_get_names_by_id     = ngbe_dev_xstats_get_names_by_id,
3028         .fw_version_get             = ngbe_fw_version_get,
3029         .dev_supported_ptypes_get   = ngbe_dev_supported_ptypes_get,
3030         .mtu_set                    = ngbe_dev_mtu_set,
3031         .vlan_filter_set            = ngbe_vlan_filter_set,
3032         .vlan_tpid_set              = ngbe_vlan_tpid_set,
3033         .vlan_offload_set           = ngbe_vlan_offload_set,
3034         .vlan_strip_queue_set       = ngbe_vlan_strip_queue_set,
3035         .rx_queue_start             = ngbe_dev_rx_queue_start,
3036         .rx_queue_stop              = ngbe_dev_rx_queue_stop,
3037         .tx_queue_start             = ngbe_dev_tx_queue_start,
3038         .tx_queue_stop              = ngbe_dev_tx_queue_stop,
3039         .rx_queue_setup             = ngbe_dev_rx_queue_setup,
3040         .rx_queue_release           = ngbe_dev_rx_queue_release,
3041         .tx_queue_setup             = ngbe_dev_tx_queue_setup,
3042         .tx_queue_release           = ngbe_dev_tx_queue_release,
3043         .dev_led_on                 = ngbe_dev_led_on,
3044         .dev_led_off                = ngbe_dev_led_off,
3045         .flow_ctrl_get              = ngbe_flow_ctrl_get,
3046         .flow_ctrl_set              = ngbe_flow_ctrl_set,
3047         .mac_addr_add               = ngbe_add_rar,
3048         .mac_addr_remove            = ngbe_remove_rar,
3049         .mac_addr_set               = ngbe_set_default_mac_addr,
3050         .uc_hash_table_set          = ngbe_uc_hash_table_set,
3051         .uc_all_hash_table_set      = ngbe_uc_all_hash_table_set,
3052         .reta_update                = ngbe_dev_rss_reta_update,
3053         .reta_query                 = ngbe_dev_rss_reta_query,
3054         .rss_hash_update            = ngbe_dev_rss_hash_update,
3055         .rss_hash_conf_get          = ngbe_dev_rss_hash_conf_get,
3056         .set_mc_addr_list           = ngbe_dev_set_mc_addr_list,
3057         .rxq_info_get               = ngbe_rxq_info_get,
3058         .txq_info_get               = ngbe_txq_info_get,
3059         .rx_burst_mode_get          = ngbe_rx_burst_mode_get,
3060         .tx_burst_mode_get          = ngbe_tx_burst_mode_get,
3061         .timesync_enable            = ngbe_timesync_enable,
3062         .timesync_disable           = ngbe_timesync_disable,
3063         .timesync_read_rx_timestamp = ngbe_timesync_read_rx_timestamp,
3064         .timesync_read_tx_timestamp = ngbe_timesync_read_tx_timestamp,
3065         .get_reg                    = ngbe_get_regs,
3066         .get_eeprom_length          = ngbe_get_eeprom_length,
3067         .get_eeprom                 = ngbe_get_eeprom,
3068         .set_eeprom                 = ngbe_set_eeprom,
3069         .timesync_adjust_time       = ngbe_timesync_adjust_time,
3070         .timesync_read_time         = ngbe_timesync_read_time,
3071         .timesync_write_time        = ngbe_timesync_write_time,
3072         .tx_done_cleanup            = ngbe_dev_tx_done_cleanup,
3073 };
3074
3075 RTE_PMD_REGISTER_PCI(net_ngbe, rte_ngbe_pmd);
3076 RTE_PMD_REGISTER_PCI_TABLE(net_ngbe, pci_id_ngbe_map);
3077 RTE_PMD_REGISTER_KMOD_DEP(net_ngbe, "* igb_uio | uio_pci_generic | vfio-pci");
3078
3079 RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_init, init, NOTICE);
3080 RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_driver, driver, NOTICE);
3081
3082 #ifdef RTE_ETHDEV_DEBUG_RX
3083         RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_rx, rx, DEBUG);
3084 #endif
3085 #ifdef RTE_ETHDEV_DEBUG_TX
3086         RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_tx, tx, DEBUG);
3087 #endif