drivers/net/ngbe/ngbe_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2021 Beijing WangXun Technology Co., Ltd.
   3  * Copyright(c) 2010-2017 Intel Corporation
   4  */
   5
   6 #include <errno.h>
   7 #include <rte_common.h>
   8 #include <ethdev_pci.h>
   9
  10 #include <rte_alarm.h>
  11
  12 #include "ngbe_logs.h"
  13 #include "ngbe.h"
  14 #include "ngbe_ethdev.h"
  15 #include "ngbe_rxtx.h"
  16 #include "ngbe_regs_group.h"
  17
  18 static const struct reg_info ngbe_regs_general[] = {
  19         {NGBE_RST, 1, 1, "NGBE_RST"},
  20         {NGBE_STAT, 1, 1, "NGBE_STAT"},
  21         {NGBE_PORTCTL, 1, 1, "NGBE_PORTCTL"},
  22         {NGBE_GPIODATA, 1, 1, "NGBE_GPIODATA"},
  23         {NGBE_GPIOCTL, 1, 1, "NGBE_GPIOCTL"},
  24         {NGBE_LEDCTL, 1, 1, "NGBE_LEDCTL"},
  25         {0, 0, 0, ""}
  26 };
  27
  28 static const struct reg_info ngbe_regs_nvm[] = {
  29         {0, 0, 0, ""}
  30 };
  31
  32 static const struct reg_info ngbe_regs_interrupt[] = {
  33         {0, 0, 0, ""}
  34 };
  35
  36 static const struct reg_info ngbe_regs_fctl_others[] = {
  37         {0, 0, 0, ""}
  38 };
  39
  40 static const struct reg_info ngbe_regs_rxdma[] = {
  41         {0, 0, 0, ""}
  42 };
  43
  44 static const struct reg_info ngbe_regs_rx[] = {
  45         {0, 0, 0, ""}
  46 };
  47
  48 static struct reg_info ngbe_regs_tx[] = {
  49         {0, 0, 0, ""}
  50 };
  51
  52 static const struct reg_info ngbe_regs_wakeup[] = {
  53         {0, 0, 0, ""}
  54 };
  55
  56 static const struct reg_info ngbe_regs_mac[] = {
  57         {0, 0, 0, ""}
  58 };
  59
  60 static const struct reg_info ngbe_regs_diagnostic[] = {
  61         {0, 0, 0, ""},
  62 };
  63
  64 /* PF registers */
  65 static const struct reg_info *ngbe_regs_others[] = {
  66                                 ngbe_regs_general,
  67                                 ngbe_regs_nvm,
  68                                 ngbe_regs_interrupt,
  69                                 ngbe_regs_fctl_others,
  70                                 ngbe_regs_rxdma,
  71                                 ngbe_regs_rx,
  72                                 ngbe_regs_tx,
  73                                 ngbe_regs_wakeup,
  74                                 ngbe_regs_mac,
  75                                 ngbe_regs_diagnostic,
  76                                 NULL};
  77
  78 static int ngbe_dev_close(struct rte_eth_dev *dev);
  79 static int ngbe_dev_link_update(struct rte_eth_dev *dev,
  80                                 int wait_to_complete);
  81 static int ngbe_dev_stats_reset(struct rte_eth_dev *dev);
  82 static void ngbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
  83 static void ngbe_vlan_hw_strip_disable(struct rte_eth_dev *dev,
  84                                         uint16_t queue);
  85
  86 static void ngbe_dev_link_status_print(struct rte_eth_dev *dev);
  87 static int ngbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on);
  88 static int ngbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev);
  89 static int ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev);
  90 static int ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
  91 static void ngbe_dev_interrupt_handler(void *param);
  92 static void ngbe_configure_msix(struct rte_eth_dev *dev);
  93
  94 #define NGBE_SET_HWSTRIP(h, q) do {\
  95                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
  96                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
  97                 (h)->bitmap[idx] |= 1 << bit;\
  98         } while (0)
  99
 100 #define NGBE_CLEAR_HWSTRIP(h, q) do {\
 101                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 102                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 103                 (h)->bitmap[idx] &= ~(1 << bit);\
 104         } while (0)
 105
 106 #define NGBE_GET_HWSTRIP(h, q, r) do {\
 107                 uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
 108                 uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
 109                 (r) = (h)->bitmap[idx] >> bit & 1;\
 110         } while (0)
 111
 112 /*
 113  * The set of PCI devices this driver supports
 114  */
 115 static const struct rte_pci_id pci_id_ngbe_map[] = {
 116         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A2) },
 117         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A2S) },
 118         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A4) },
 119         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A4S) },
 120         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL2) },
 121         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S) },
 122         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL4) },
 123         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S) },
 124         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860NCSI) },
 125         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A1) },
 126         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860A1L) },
 127         { RTE_PCI_DEVICE(PCI_VENDOR_ID_WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W) },
 128         { .vendor_id = 0, /* sentinel */ },
 129 };
 130
 131 static const struct rte_eth_desc_lim rx_desc_lim = {
 132         .nb_max = NGBE_RING_DESC_MAX,
 133         .nb_min = NGBE_RING_DESC_MIN,
 134         .nb_align = NGBE_RXD_ALIGN,
 135 };
 136
 137 static const struct rte_eth_desc_lim tx_desc_lim = {
 138         .nb_max = NGBE_RING_DESC_MAX,
 139         .nb_min = NGBE_RING_DESC_MIN,
 140         .nb_align = NGBE_TXD_ALIGN,
 141         .nb_seg_max = NGBE_TX_MAX_SEG,
 142         .nb_mtu_seg_max = NGBE_TX_MAX_SEG,
 143 };
 144
 145 static const struct eth_dev_ops ngbe_eth_dev_ops;
 146
 147 #define HW_XSTAT(m) {#m, offsetof(struct ngbe_hw_stats, m)}
 148 #define HW_XSTAT_NAME(m, n) {n, offsetof(struct ngbe_hw_stats, m)}
 149 static const struct rte_ngbe_xstats_name_off rte_ngbe_stats_strings[] = {
 150         /* MNG RxTx */
 151         HW_XSTAT(mng_bmc2host_packets),
 152         HW_XSTAT(mng_host2bmc_packets),
 153         /* Basic RxTx */
 154         HW_XSTAT(rx_packets),
 155         HW_XSTAT(tx_packets),
 156         HW_XSTAT(rx_bytes),
 157         HW_XSTAT(tx_bytes),
 158         HW_XSTAT(rx_total_bytes),
 159         HW_XSTAT(rx_total_packets),
 160         HW_XSTAT(tx_total_packets),
 161         HW_XSTAT(rx_total_missed_packets),
 162         HW_XSTAT(rx_broadcast_packets),
 163         HW_XSTAT(rx_multicast_packets),
 164         HW_XSTAT(rx_management_packets),
 165         HW_XSTAT(tx_management_packets),
 166         HW_XSTAT(rx_management_dropped),
 167
 168         /* Basic Error */
 169         HW_XSTAT(rx_crc_errors),
 170         HW_XSTAT(rx_illegal_byte_errors),
 171         HW_XSTAT(rx_error_bytes),
 172         HW_XSTAT(rx_mac_short_packet_dropped),
 173         HW_XSTAT(rx_length_errors),
 174         HW_XSTAT(rx_undersize_errors),
 175         HW_XSTAT(rx_fragment_errors),
 176         HW_XSTAT(rx_oversize_errors),
 177         HW_XSTAT(rx_jabber_errors),
 178         HW_XSTAT(rx_l3_l4_xsum_error),
 179         HW_XSTAT(mac_local_errors),
 180         HW_XSTAT(mac_remote_errors),
 181
 182         /* MACSEC */
 183         HW_XSTAT(tx_macsec_pkts_untagged),
 184         HW_XSTAT(tx_macsec_pkts_encrypted),
 185         HW_XSTAT(tx_macsec_pkts_protected),
 186         HW_XSTAT(tx_macsec_octets_encrypted),
 187         HW_XSTAT(tx_macsec_octets_protected),
 188         HW_XSTAT(rx_macsec_pkts_untagged),
 189         HW_XSTAT(rx_macsec_pkts_badtag),
 190         HW_XSTAT(rx_macsec_pkts_nosci),
 191         HW_XSTAT(rx_macsec_pkts_unknownsci),
 192         HW_XSTAT(rx_macsec_octets_decrypted),
 193         HW_XSTAT(rx_macsec_octets_validated),
 194         HW_XSTAT(rx_macsec_sc_pkts_unchecked),
 195         HW_XSTAT(rx_macsec_sc_pkts_delayed),
 196         HW_XSTAT(rx_macsec_sc_pkts_late),
 197         HW_XSTAT(rx_macsec_sa_pkts_ok),
 198         HW_XSTAT(rx_macsec_sa_pkts_invalid),
 199         HW_XSTAT(rx_macsec_sa_pkts_notvalid),
 200         HW_XSTAT(rx_macsec_sa_pkts_unusedsa),
 201         HW_XSTAT(rx_macsec_sa_pkts_notusingsa),
 202
 203         /* MAC RxTx */
 204         HW_XSTAT(rx_size_64_packets),
 205         HW_XSTAT(rx_size_65_to_127_packets),
 206         HW_XSTAT(rx_size_128_to_255_packets),
 207         HW_XSTAT(rx_size_256_to_511_packets),
 208         HW_XSTAT(rx_size_512_to_1023_packets),
 209         HW_XSTAT(rx_size_1024_to_max_packets),
 210         HW_XSTAT(tx_size_64_packets),
 211         HW_XSTAT(tx_size_65_to_127_packets),
 212         HW_XSTAT(tx_size_128_to_255_packets),
 213         HW_XSTAT(tx_size_256_to_511_packets),
 214         HW_XSTAT(tx_size_512_to_1023_packets),
 215         HW_XSTAT(tx_size_1024_to_max_packets),
 216
 217         /* Flow Control */
 218         HW_XSTAT(tx_xon_packets),
 219         HW_XSTAT(rx_xon_packets),
 220         HW_XSTAT(tx_xoff_packets),
 221         HW_XSTAT(rx_xoff_packets),
 222
 223         HW_XSTAT_NAME(tx_xon_packets, "tx_flow_control_xon_packets"),
 224         HW_XSTAT_NAME(rx_xon_packets, "rx_flow_control_xon_packets"),
 225         HW_XSTAT_NAME(tx_xoff_packets, "tx_flow_control_xoff_packets"),
 226         HW_XSTAT_NAME(rx_xoff_packets, "rx_flow_control_xoff_packets"),
 227 };
 228
 229 #define NGBE_NB_HW_STATS (sizeof(rte_ngbe_stats_strings) / \
 230                            sizeof(rte_ngbe_stats_strings[0]))
 231
 232 /* Per-queue statistics */
 233 #define QP_XSTAT(m) {#m, offsetof(struct ngbe_hw_stats, qp[0].m)}
 234 static const struct rte_ngbe_xstats_name_off rte_ngbe_qp_strings[] = {
 235         QP_XSTAT(rx_qp_packets),
 236         QP_XSTAT(tx_qp_packets),
 237         QP_XSTAT(rx_qp_bytes),
 238         QP_XSTAT(tx_qp_bytes),
 239         QP_XSTAT(rx_qp_mc_packets),
 240 };
 241
 242 #define NGBE_NB_QP_STATS (sizeof(rte_ngbe_qp_strings) / \
 243                            sizeof(rte_ngbe_qp_strings[0]))
 244
 245 static inline int32_t
 246 ngbe_pf_reset_hw(struct ngbe_hw *hw)
 247 {
 248         uint32_t ctrl_ext;
 249         int32_t status;
 250
 251         status = hw->mac.reset_hw(hw);
 252
 253         ctrl_ext = rd32(hw, NGBE_PORTCTL);
 254         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 255         ctrl_ext |= NGBE_PORTCTL_RSTDONE;
 256         wr32(hw, NGBE_PORTCTL, ctrl_ext);
 257         ngbe_flush(hw);
 258
 259         if (status == NGBE_ERR_SFP_NOT_PRESENT)
 260                 status = 0;
 261         return status;
 262 }
 263
 264 static inline void
 265 ngbe_enable_intr(struct rte_eth_dev *dev)
 266 {
 267         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 268         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 269
 270         wr32(hw, NGBE_IENMISC, intr->mask_misc);
 271         wr32(hw, NGBE_IMC(0), intr->mask & BIT_MASK32);
 272         ngbe_flush(hw);
 273 }
 274
 275 static void
 276 ngbe_disable_intr(struct ngbe_hw *hw)
 277 {
 278         PMD_INIT_FUNC_TRACE();
 279
 280         wr32(hw, NGBE_IMS(0), NGBE_IMS_MASK);
 281         ngbe_flush(hw);
 282 }
 283
 284 /*
 285  * Ensure that all locks are released before first NVM or PHY access
 286  */
 287 static void
 288 ngbe_swfw_lock_reset(struct ngbe_hw *hw)
 289 {
 290         uint16_t mask;
 291
 292         /*
 293          * These ones are more tricky since they are common to all ports; but
 294          * swfw_sync retries last long enough (1s) to be almost sure that if
 295          * lock can not be taken it is due to an improper lock of the
 296          * semaphore.
 297          */
 298         mask = NGBE_MNGSEM_SWPHY |
 299                NGBE_MNGSEM_SWMBX |
 300                NGBE_MNGSEM_SWFLASH;
 301         if (hw->mac.acquire_swfw_sync(hw, mask) < 0)
 302                 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 303
 304         hw->mac.release_swfw_sync(hw, mask);
 305 }
 306
 307 static int
 308 eth_ngbe_dev_init(struct rte_eth_dev *eth_dev, void *init_params __rte_unused)
 309 {
 310         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 311         struct ngbe_hw *hw = ngbe_dev_hw(eth_dev);
 312         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(eth_dev);
 313         struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(eth_dev);
 314         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 315         const struct rte_memzone *mz;
 316         uint32_t ctrl_ext;
 317         int err, ret;
 318
 319         PMD_INIT_FUNC_TRACE();
 320
 321         eth_dev->dev_ops = &ngbe_eth_dev_ops;
 322         eth_dev->rx_queue_count       = ngbe_dev_rx_queue_count;
 323         eth_dev->rx_descriptor_status = ngbe_dev_rx_descriptor_status;
 324         eth_dev->tx_descriptor_status = ngbe_dev_tx_descriptor_status;
 325         eth_dev->rx_pkt_burst = &ngbe_recv_pkts;
 326         eth_dev->tx_pkt_burst = &ngbe_xmit_pkts;
 327         eth_dev->tx_pkt_prepare = &ngbe_prep_pkts;
 328
 329         /*
 330          * For secondary processes, we don't initialise any further as primary
 331          * has already done this work. Only check we don't need a different
 332          * Rx and Tx function.
 333          */
 334         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 335                 struct ngbe_tx_queue *txq;
 336                 /* Tx queue function in primary, set by last queue initialized
 337                  * Tx queue may not initialized by primary process
 338                  */
 339                 if (eth_dev->data->tx_queues) {
 340                         uint16_t nb_tx_queues = eth_dev->data->nb_tx_queues;
 341                         txq = eth_dev->data->tx_queues[nb_tx_queues - 1];
 342                         ngbe_set_tx_function(eth_dev, txq);
 343                 } else {
 344                         /* Use default Tx function if we get here */
 345                         PMD_INIT_LOG(NOTICE,
 346                                 "No Tx queues configured yet. Using default Tx function.");
 347                 }
 348
 349                 ngbe_set_rx_function(eth_dev);
 350
 351                 return 0;
 352         }
 353
 354         rte_eth_copy_pci_info(eth_dev, pci_dev);
 355         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 356
 357         /* Vendor and Device ID need to be set before init of shared code */
 358         hw->device_id = pci_dev->id.device_id;
 359         hw->vendor_id = pci_dev->id.vendor_id;
 360         hw->sub_system_id = pci_dev->id.subsystem_device_id;
 361         ngbe_map_device_id(hw);
 362         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 363
 364         /* Reserve memory for interrupt status block */
 365         mz = rte_eth_dma_zone_reserve(eth_dev, "ngbe_driver", -1,
 366                 NGBE_ISB_SIZE, NGBE_ALIGN, SOCKET_ID_ANY);
 367         if (mz == NULL)
 368                 return -ENOMEM;
 369
 370         hw->isb_dma = TMZ_PADDR(mz);
 371         hw->isb_mem = TMZ_VADDR(mz);
 372
 373         /* Initialize the shared code (base driver) */
 374         err = ngbe_init_shared_code(hw);
 375         if (err != 0) {
 376                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", err);
 377                 return -EIO;
 378         }
 379
 380         /* Unlock any pending hardware semaphore */
 381         ngbe_swfw_lock_reset(hw);
 382
 383         /* Get Hardware Flow Control setting */
 384         hw->fc.requested_mode = ngbe_fc_full;
 385         hw->fc.current_mode = ngbe_fc_full;
 386         hw->fc.pause_time = NGBE_FC_PAUSE_TIME;
 387         hw->fc.low_water = NGBE_FC_XON_LOTH;
 388         hw->fc.high_water = NGBE_FC_XOFF_HITH;
 389         hw->fc.send_xon = 1;
 390
 391         err = hw->rom.init_params(hw);
 392         if (err != 0) {
 393                 PMD_INIT_LOG(ERR, "The EEPROM init failed: %d", err);
 394                 return -EIO;
 395         }
 396
 397         /* Make sure we have a good EEPROM before we read from it */
 398         err = hw->rom.validate_checksum(hw, NULL);
 399         if (err != 0) {
 400                 PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", err);
 401                 return -EIO;
 402         }
 403
 404         err = hw->mac.init_hw(hw);
 405         if (err != 0) {
 406                 PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", err);
 407                 return -EIO;
 408         }
 409
 410         /* Reset the hw statistics */
 411         ngbe_dev_stats_reset(eth_dev);
 412
 413         /* disable interrupt */
 414         ngbe_disable_intr(hw);
 415
 416         /* Allocate memory for storing MAC addresses */
 417         eth_dev->data->mac_addrs = rte_zmalloc("ngbe", RTE_ETHER_ADDR_LEN *
 418                                                hw->mac.num_rar_entries, 0);
 419         if (eth_dev->data->mac_addrs == NULL) {
 420                 PMD_INIT_LOG(ERR,
 421                              "Failed to allocate %u bytes needed to store MAC addresses",
 422                              RTE_ETHER_ADDR_LEN * hw->mac.num_rar_entries);
 423                 return -ENOMEM;
 424         }
 425
 426         /* Copy the permanent MAC address */
 427         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.perm_addr,
 428                         &eth_dev->data->mac_addrs[0]);
 429
 430         /* Allocate memory for storing hash filter MAC addresses */
 431         eth_dev->data->hash_mac_addrs = rte_zmalloc("ngbe",
 432                         RTE_ETHER_ADDR_LEN * NGBE_VMDQ_NUM_UC_MAC, 0);
 433         if (eth_dev->data->hash_mac_addrs == NULL) {
 434                 PMD_INIT_LOG(ERR,
 435                              "Failed to allocate %d bytes needed to store MAC addresses",
 436                              RTE_ETHER_ADDR_LEN * NGBE_VMDQ_NUM_UC_MAC);
 437                 rte_free(eth_dev->data->mac_addrs);
 438                 eth_dev->data->mac_addrs = NULL;
 439                 return -ENOMEM;
 440         }
 441
 442         /* initialize the vfta */
 443         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 444
 445         /* initialize the hw strip bitmap*/
 446         memset(hwstrip, 0, sizeof(*hwstrip));
 447
 448         /* initialize PF if max_vfs not zero */
 449         ret = ngbe_pf_host_init(eth_dev);
 450         if (ret) {
 451                 rte_free(eth_dev->data->mac_addrs);
 452                 eth_dev->data->mac_addrs = NULL;
 453                 rte_free(eth_dev->data->hash_mac_addrs);
 454                 eth_dev->data->hash_mac_addrs = NULL;
 455                 return ret;
 456         }
 457
 458         ctrl_ext = rd32(hw, NGBE_PORTCTL);
 459         /* let hardware know driver is loaded */
 460         ctrl_ext |= NGBE_PORTCTL_DRVLOAD;
 461         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 462         ctrl_ext |= NGBE_PORTCTL_RSTDONE;
 463         wr32(hw, NGBE_PORTCTL, ctrl_ext);
 464         ngbe_flush(hw);
 465
 466         PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
 467                         (int)hw->mac.type, (int)hw->phy.type);
 468
 469         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
 470                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 471                      pci_dev->id.device_id);
 472
 473         rte_intr_callback_register(intr_handle,
 474                                    ngbe_dev_interrupt_handler, eth_dev);
 475
 476         /* enable uio/vfio intr/eventfd mapping */
 477         rte_intr_enable(intr_handle);
 478
 479         /* enable support intr */
 480         ngbe_enable_intr(eth_dev);
 481
 482         return 0;
 483 }
 484
 485 static int
 486 eth_ngbe_dev_uninit(struct rte_eth_dev *eth_dev)
 487 {
 488         PMD_INIT_FUNC_TRACE();
 489
 490         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 491                 return 0;
 492
 493         ngbe_dev_close(eth_dev);
 494
 495         return 0;
 496 }
 497
 498 static int
 499 eth_ngbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 500                 struct rte_pci_device *pci_dev)
 501 {
 502         return rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
 503                         sizeof(struct ngbe_adapter),
 504                         eth_dev_pci_specific_init, pci_dev,
 505                         eth_ngbe_dev_init, NULL);
 506 }
 507
 508 static int eth_ngbe_pci_remove(struct rte_pci_device *pci_dev)
 509 {
 510         struct rte_eth_dev *ethdev;
 511
 512         ethdev = rte_eth_dev_allocated(pci_dev->device.name);
 513         if (ethdev == NULL)
 514                 return 0;
 515
 516         return rte_eth_dev_destroy(ethdev, eth_ngbe_dev_uninit);
 517 }
 518
 519 static struct rte_pci_driver rte_ngbe_pmd = {
 520         .id_table = pci_id_ngbe_map,
 521         .drv_flags = RTE_PCI_DRV_NEED_MAPPING |
 522                      RTE_PCI_DRV_INTR_LSC,
 523         .probe = eth_ngbe_pci_probe,
 524         .remove = eth_ngbe_pci_remove,
 525 };
 526
 527 static int
 528 ngbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 529 {
 530         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 531         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(dev);
 532         uint32_t vfta;
 533         uint32_t vid_idx;
 534         uint32_t vid_bit;
 535
 536         vid_idx = (uint32_t)((vlan_id >> 5) & 0x7F);
 537         vid_bit = (uint32_t)(1 << (vlan_id & 0x1F));
 538         vfta = rd32(hw, NGBE_VLANTBL(vid_idx));
 539         if (on)
 540                 vfta |= vid_bit;
 541         else
 542                 vfta &= ~vid_bit;
 543         wr32(hw, NGBE_VLANTBL(vid_idx), vfta);
 544
 545         /* update local VFTA copy */
 546         shadow_vfta->vfta[vid_idx] = vfta;
 547
 548         return 0;
 549 }
 550
 551 static void
 552 ngbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
 553 {
 554         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 555         struct ngbe_rx_queue *rxq;
 556         bool restart;
 557         uint32_t rxcfg, rxbal, rxbah;
 558
 559         if (on)
 560                 ngbe_vlan_hw_strip_enable(dev, queue);
 561         else
 562                 ngbe_vlan_hw_strip_disable(dev, queue);
 563
 564         rxq = dev->data->rx_queues[queue];
 565         rxbal = rd32(hw, NGBE_RXBAL(rxq->reg_idx));
 566         rxbah = rd32(hw, NGBE_RXBAH(rxq->reg_idx));
 567         rxcfg = rd32(hw, NGBE_RXCFG(rxq->reg_idx));
 568         if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) {
 569                 restart = (rxcfg & NGBE_RXCFG_ENA) &&
 570                         !(rxcfg & NGBE_RXCFG_VLAN);
 571                 rxcfg |= NGBE_RXCFG_VLAN;
 572         } else {
 573                 restart = (rxcfg & NGBE_RXCFG_ENA) &&
 574                         (rxcfg & NGBE_RXCFG_VLAN);
 575                 rxcfg &= ~NGBE_RXCFG_VLAN;
 576         }
 577         rxcfg &= ~NGBE_RXCFG_ENA;
 578
 579         if (restart) {
 580                 /* set vlan strip for ring */
 581                 ngbe_dev_rx_queue_stop(dev, queue);
 582                 wr32(hw, NGBE_RXBAL(rxq->reg_idx), rxbal);
 583                 wr32(hw, NGBE_RXBAH(rxq->reg_idx), rxbah);
 584                 wr32(hw, NGBE_RXCFG(rxq->reg_idx), rxcfg);
 585                 ngbe_dev_rx_queue_start(dev, queue);
 586         }
 587 }
 588
 589 static int
 590 ngbe_vlan_tpid_set(struct rte_eth_dev *dev,
 591                     enum rte_vlan_type vlan_type,
 592                     uint16_t tpid)
 593 {
 594         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 595         int ret = 0;
 596         uint32_t portctrl, vlan_ext, qinq;
 597
 598         portctrl = rd32(hw, NGBE_PORTCTL);
 599
 600         vlan_ext = (portctrl & NGBE_PORTCTL_VLANEXT);
 601         qinq = vlan_ext && (portctrl & NGBE_PORTCTL_QINQ);
 602         switch (vlan_type) {
 603         case RTE_ETH_VLAN_TYPE_INNER:
 604                 if (vlan_ext) {
 605                         wr32m(hw, NGBE_VLANCTL,
 606                                 NGBE_VLANCTL_TPID_MASK,
 607                                 NGBE_VLANCTL_TPID(tpid));
 608                         wr32m(hw, NGBE_DMATXCTRL,
 609                                 NGBE_DMATXCTRL_TPID_MASK,
 610                                 NGBE_DMATXCTRL_TPID(tpid));
 611                 } else {
 612                         ret = -ENOTSUP;
 613                         PMD_DRV_LOG(ERR,
 614                                 "Inner type is not supported by single VLAN");
 615                 }
 616
 617                 if (qinq) {
 618                         wr32m(hw, NGBE_TAGTPID(0),
 619                                 NGBE_TAGTPID_LSB_MASK,
 620                                 NGBE_TAGTPID_LSB(tpid));
 621                 }
 622                 break;
 623         case RTE_ETH_VLAN_TYPE_OUTER:
 624                 if (vlan_ext) {
 625                         /* Only the high 16-bits is valid */
 626                         wr32m(hw, NGBE_EXTAG,
 627                                 NGBE_EXTAG_VLAN_MASK,
 628                                 NGBE_EXTAG_VLAN(tpid));
 629                 } else {
 630                         wr32m(hw, NGBE_VLANCTL,
 631                                 NGBE_VLANCTL_TPID_MASK,
 632                                 NGBE_VLANCTL_TPID(tpid));
 633                         wr32m(hw, NGBE_DMATXCTRL,
 634                                 NGBE_DMATXCTRL_TPID_MASK,
 635                                 NGBE_DMATXCTRL_TPID(tpid));
 636                 }
 637
 638                 if (qinq) {
 639                         wr32m(hw, NGBE_TAGTPID(0),
 640                                 NGBE_TAGTPID_MSB_MASK,
 641                                 NGBE_TAGTPID_MSB(tpid));
 642                 }
 643                 break;
 644         default:
 645                 PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type);
 646                 return -EINVAL;
 647         }
 648
 649         return ret;
 650 }
 651
 652 void
 653 ngbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
 654 {
 655         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 656         uint32_t vlnctrl;
 657
 658         PMD_INIT_FUNC_TRACE();
 659
 660         /* Filter Table Disable */
 661         vlnctrl = rd32(hw, NGBE_VLANCTL);
 662         vlnctrl &= ~NGBE_VLANCTL_VFE;
 663         wr32(hw, NGBE_VLANCTL, vlnctrl);
 664 }
 665
 666 void
 667 ngbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 668 {
 669         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 670         struct ngbe_vfta *shadow_vfta = NGBE_DEV_VFTA(dev);
 671         uint32_t vlnctrl;
 672         uint16_t i;
 673
 674         PMD_INIT_FUNC_TRACE();
 675
 676         /* Filter Table Enable */
 677         vlnctrl = rd32(hw, NGBE_VLANCTL);
 678         vlnctrl &= ~NGBE_VLANCTL_CFIENA;
 679         vlnctrl |= NGBE_VLANCTL_VFE;
 680         wr32(hw, NGBE_VLANCTL, vlnctrl);
 681
 682         /* write whatever is in local vfta copy */
 683         for (i = 0; i < NGBE_VFTA_SIZE; i++)
 684                 wr32(hw, NGBE_VLANTBL(i), shadow_vfta->vfta[i]);
 685 }
 686
 687 void
 688 ngbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
 689 {
 690         struct ngbe_hwstrip *hwstrip = NGBE_DEV_HWSTRIP(dev);
 691         struct ngbe_rx_queue *rxq;
 692
 693         if (queue >= NGBE_MAX_RX_QUEUE_NUM)
 694                 return;
 695
 696         if (on)
 697                 NGBE_SET_HWSTRIP(hwstrip, queue);
 698         else
 699                 NGBE_CLEAR_HWSTRIP(hwstrip, queue);
 700
 701         if (queue >= dev->data->nb_rx_queues)
 702                 return;
 703
 704         rxq = dev->data->rx_queues[queue];
 705
 706         if (on) {
 707                 rxq->vlan_flags = RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
 708                 rxq->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 709         } else {
 710                 rxq->vlan_flags = RTE_MBUF_F_RX_VLAN;
 711                 rxq->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 712         }
 713 }
 714
 715 static void
 716 ngbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
 717 {
 718         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 719         uint32_t ctrl;
 720
 721         PMD_INIT_FUNC_TRACE();
 722
 723         ctrl = rd32(hw, NGBE_RXCFG(queue));
 724         ctrl &= ~NGBE_RXCFG_VLAN;
 725         wr32(hw, NGBE_RXCFG(queue), ctrl);
 726
 727         /* record those setting for HW strip per queue */
 728         ngbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
 729 }
 730
 731 static void
 732 ngbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
 733 {
 734         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 735         uint32_t ctrl;
 736
 737         PMD_INIT_FUNC_TRACE();
 738
 739         ctrl = rd32(hw, NGBE_RXCFG(queue));
 740         ctrl |= NGBE_RXCFG_VLAN;
 741         wr32(hw, NGBE_RXCFG(queue), ctrl);
 742
 743         /* record those setting for HW strip per queue */
 744         ngbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
 745 }
 746
 747 static void
 748 ngbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
 749 {
 750         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 751         uint32_t ctrl;
 752
 753         PMD_INIT_FUNC_TRACE();
 754
 755         ctrl = rd32(hw, NGBE_PORTCTL);
 756         ctrl &= ~NGBE_PORTCTL_VLANEXT;
 757         ctrl &= ~NGBE_PORTCTL_QINQ;
 758         wr32(hw, NGBE_PORTCTL, ctrl);
 759 }
 760
 761 static void
 762 ngbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
 763 {
 764         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 765         uint32_t ctrl;
 766
 767         PMD_INIT_FUNC_TRACE();
 768
 769         ctrl  = rd32(hw, NGBE_PORTCTL);
 770         ctrl |= NGBE_PORTCTL_VLANEXT | NGBE_PORTCTL_QINQ;
 771         wr32(hw, NGBE_PORTCTL, ctrl);
 772 }
 773
 774 static void
 775 ngbe_qinq_hw_strip_disable(struct rte_eth_dev *dev)
 776 {
 777         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 778         uint32_t ctrl;
 779
 780         PMD_INIT_FUNC_TRACE();
 781
 782         ctrl = rd32(hw, NGBE_PORTCTL);
 783         ctrl &= ~NGBE_PORTCTL_QINQ;
 784         wr32(hw, NGBE_PORTCTL, ctrl);
 785 }
 786
 787 static void
 788 ngbe_qinq_hw_strip_enable(struct rte_eth_dev *dev)
 789 {
 790         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 791         uint32_t ctrl;
 792
 793         PMD_INIT_FUNC_TRACE();
 794
 795         ctrl  = rd32(hw, NGBE_PORTCTL);
 796         ctrl |= NGBE_PORTCTL_QINQ | NGBE_PORTCTL_VLANEXT;
 797         wr32(hw, NGBE_PORTCTL, ctrl);
 798 }
 799
 800 void
 801 ngbe_vlan_hw_strip_config(struct rte_eth_dev *dev)
 802 {
 803         struct ngbe_rx_queue *rxq;
 804         uint16_t i;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 809                 rxq = dev->data->rx_queues[i];
 810
 811                 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 812                         ngbe_vlan_hw_strip_enable(dev, i);
 813                 else
 814                         ngbe_vlan_hw_strip_disable(dev, i);
 815         }
 816 }
 817
 818 void
 819 ngbe_config_vlan_strip_on_all_queues(struct rte_eth_dev *dev, int mask)
 820 {
 821         uint16_t i;
 822         struct rte_eth_rxmode *rxmode;
 823         struct ngbe_rx_queue *rxq;
 824
 825         if (mask & RTE_ETH_VLAN_STRIP_MASK) {
 826                 rxmode = &dev->data->dev_conf.rxmode;
 827                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 828                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 829                                 rxq = dev->data->rx_queues[i];
 830                                 rxq->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 831                         }
 832                 else
 833                         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 834                                 rxq = dev->data->rx_queues[i];
 835                                 rxq->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 836                         }
 837         }
 838 }
 839
 840 static int
 841 ngbe_vlan_offload_config(struct rte_eth_dev *dev, int mask)
 842 {
 843         struct rte_eth_rxmode *rxmode;
 844         rxmode = &dev->data->dev_conf.rxmode;
 845
 846         if (mask & RTE_ETH_VLAN_STRIP_MASK)
 847                 ngbe_vlan_hw_strip_config(dev);
 848
 849         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
 850                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER)
 851                         ngbe_vlan_hw_filter_enable(dev);
 852                 else
 853                         ngbe_vlan_hw_filter_disable(dev);
 854         }
 855
 856         if (mask & RTE_ETH_VLAN_EXTEND_MASK) {
 857                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_EXTEND)
 858                         ngbe_vlan_hw_extend_enable(dev);
 859                 else
 860                         ngbe_vlan_hw_extend_disable(dev);
 861         }
 862
 863         if (mask & RTE_ETH_QINQ_STRIP_MASK) {
 864                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_QINQ_STRIP)
 865                         ngbe_qinq_hw_strip_enable(dev);
 866                 else
 867                         ngbe_qinq_hw_strip_disable(dev);
 868         }
 869
 870         return 0;
 871 }
 872
 873 static int
 874 ngbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
 875 {
 876         ngbe_config_vlan_strip_on_all_queues(dev, mask);
 877
 878         ngbe_vlan_offload_config(dev, mask);
 879
 880         return 0;
 881 }
 882
 883 static int
 884 ngbe_dev_configure(struct rte_eth_dev *dev)
 885 {
 886         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 887         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
 888
 889         PMD_INIT_FUNC_TRACE();
 890
 891         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
 892                 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
 893
 894         /* set flag to update link status after init */
 895         intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
 896
 897         /*
 898          * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
 899          * allocation Rx preconditions we will reset it.
 900          */
 901         adapter->rx_bulk_alloc_allowed = true;
 902
 903         return 0;
 904 }
 905
 906 static void
 907 ngbe_dev_phy_intr_setup(struct rte_eth_dev *dev)
 908 {
 909         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 910         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
 911
 912         wr32(hw, NGBE_GPIODIR, NGBE_GPIODIR_DDR(1));
 913         wr32(hw, NGBE_GPIOINTEN, NGBE_GPIOINTEN_INT(3));
 914         wr32(hw, NGBE_GPIOINTTYPE, NGBE_GPIOINTTYPE_LEVEL(0));
 915         if (hw->phy.type == ngbe_phy_yt8521s_sfi)
 916                 wr32(hw, NGBE_GPIOINTPOL, NGBE_GPIOINTPOL_ACT(0));
 917         else
 918                 wr32(hw, NGBE_GPIOINTPOL, NGBE_GPIOINTPOL_ACT(3));
 919
 920         intr->mask_misc |= NGBE_ICRMISC_GPIO;
 921 }
 922
 923 /*
 924  * Configure device link speed and setup link.
 925  * It returns 0 on success.
 926  */
 927 static int
 928 ngbe_dev_start(struct rte_eth_dev *dev)
 929 {
 930         struct ngbe_hw *hw = ngbe_dev_hw(dev);
 931         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
 932         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 933         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 934         uint32_t intr_vector = 0;
 935         int err;
 936         bool link_up = false, negotiate = false;
 937         uint32_t speed = 0;
 938         uint32_t allowed_speeds = 0;
 939         int mask = 0;
 940         int status;
 941         uint32_t *link_speeds;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Stop the link setup handler before resetting the HW. */
 946         rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev);
 947
 948         /* disable uio/vfio intr/eventfd mapping */
 949         rte_intr_disable(intr_handle);
 950
 951         /* stop adapter */
 952         hw->adapter_stopped = 0;
 953         ngbe_stop_hw(hw);
 954
 955         /* reinitialize adapter, this calls reset and start */
 956         hw->nb_rx_queues = dev->data->nb_rx_queues;
 957         hw->nb_tx_queues = dev->data->nb_tx_queues;
 958         status = ngbe_pf_reset_hw(hw);
 959         if (status != 0)
 960                 return -1;
 961         hw->mac.start_hw(hw);
 962         hw->mac.get_link_status = true;
 963
 964         /* configure PF module if SRIOV enabled */
 965         ngbe_pf_host_configure(dev);
 966
 967         ngbe_dev_phy_intr_setup(dev);
 968
 969         /* check and configure queue intr-vector mapping */
 970         if ((rte_intr_cap_multiple(intr_handle) ||
 971              !RTE_ETH_DEV_SRIOV(dev).active) &&
 972             dev->data->dev_conf.intr_conf.rxq != 0) {
 973                 intr_vector = dev->data->nb_rx_queues;
 974                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 975                         return -1;
 976         }
 977
 978         if (rte_intr_dp_is_en(intr_handle)) {
 979                 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec",
 980                                                    dev->data->nb_rx_queues)) {
 981                         PMD_INIT_LOG(ERR,
 982                                      "Failed to allocate %d rx_queues intr_vec",
 983                                      dev->data->nb_rx_queues);
 984                         return -ENOMEM;
 985                 }
 986         }
 987
 988         /* configure MSI-X for sleep until Rx interrupt */
 989         ngbe_configure_msix(dev);
 990
 991         /* initialize transmission unit */
 992         ngbe_dev_tx_init(dev);
 993
 994         /* This can fail when allocating mbufs for descriptor rings */
 995         err = ngbe_dev_rx_init(dev);
 996         if (err != 0) {
 997                 PMD_INIT_LOG(ERR, "Unable to initialize Rx hardware");
 998                 goto error;
 999         }
1000
1001         mask = RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK |
1002                 RTE_ETH_VLAN_EXTEND_MASK;
1003         err = ngbe_vlan_offload_config(dev, mask);
1004         if (err != 0) {
1005                 PMD_INIT_LOG(ERR, "Unable to set VLAN offload");
1006                 goto error;
1007         }
1008
1009         hw->mac.setup_pba(hw);
1010         ngbe_configure_port(dev);
1011
1012         err = ngbe_dev_rxtx_start(dev);
1013         if (err < 0) {
1014                 PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
1015                 goto error;
1016         }
1017
1018         /* Skip link setup if loopback mode is enabled. */
1019         if (hw->is_pf && dev->data->dev_conf.lpbk_mode)
1020                 goto skip_link_setup;
1021
1022         err = hw->mac.check_link(hw, &speed, &link_up, 0);
1023         if (err != 0)
1024                 goto error;
1025         dev->data->dev_link.link_status = link_up;
1026
1027         link_speeds = &dev->data->dev_conf.link_speeds;
1028         if (*link_speeds == RTE_ETH_LINK_SPEED_AUTONEG)
1029                 negotiate = true;
1030
1031         err = hw->mac.get_link_capabilities(hw, &speed, &negotiate);
1032         if (err != 0)
1033                 goto error;
1034
1035         allowed_speeds = 0;
1036         if (hw->mac.default_speeds & NGBE_LINK_SPEED_1GB_FULL)
1037                 allowed_speeds |= RTE_ETH_LINK_SPEED_1G;
1038         if (hw->mac.default_speeds & NGBE_LINK_SPEED_100M_FULL)
1039                 allowed_speeds |= RTE_ETH_LINK_SPEED_100M;
1040         if (hw->mac.default_speeds & NGBE_LINK_SPEED_10M_FULL)
1041                 allowed_speeds |= RTE_ETH_LINK_SPEED_10M;
1042
1043         if (*link_speeds & ~allowed_speeds) {
1044                 PMD_INIT_LOG(ERR, "Invalid link setting");
1045                 goto error;
1046         }
1047
1048         speed = 0x0;
1049         if (*link_speeds == RTE_ETH_LINK_SPEED_AUTONEG) {
1050                 speed = hw->mac.default_speeds;
1051         } else {
1052                 if (*link_speeds & RTE_ETH_LINK_SPEED_1G)
1053                         speed |= NGBE_LINK_SPEED_1GB_FULL;
1054                 if (*link_speeds & RTE_ETH_LINK_SPEED_100M)
1055                         speed |= NGBE_LINK_SPEED_100M_FULL;
1056                 if (*link_speeds & RTE_ETH_LINK_SPEED_10M)
1057                         speed |= NGBE_LINK_SPEED_10M_FULL;
1058         }
1059
1060         hw->phy.init_hw(hw);
1061         err = hw->mac.setup_link(hw, speed, link_up);
1062         if (err != 0)
1063                 goto error;
1064
1065 skip_link_setup:
1066
1067         if (rte_intr_allow_others(intr_handle)) {
1068                 ngbe_dev_misc_interrupt_setup(dev);
1069                 /* check if lsc interrupt is enabled */
1070                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1071                         ngbe_dev_lsc_interrupt_setup(dev, TRUE);
1072                 else
1073                         ngbe_dev_lsc_interrupt_setup(dev, FALSE);
1074                 ngbe_dev_macsec_interrupt_setup(dev);
1075                 ngbe_set_ivar_map(hw, -1, 1, NGBE_MISC_VEC_ID);
1076         } else {
1077                 rte_intr_callback_unregister(intr_handle,
1078                                              ngbe_dev_interrupt_handler, dev);
1079                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1080                         PMD_INIT_LOG(INFO,
1081                                      "LSC won't enable because of no intr multiplex");
1082         }
1083
1084         /* check if rxq interrupt is enabled */
1085         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1086             rte_intr_dp_is_en(intr_handle))
1087                 ngbe_dev_rxq_interrupt_setup(dev);
1088
1089         /* enable UIO/VFIO intr/eventfd mapping */
1090         rte_intr_enable(intr_handle);
1091
1092         /* resume enabled intr since HW reset */
1093         ngbe_enable_intr(dev);
1094
1095         if ((hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_M88E1512_SFP ||
1096                 (hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_YT8521S_SFP) {
1097                 /* gpio0 is used to power on/off control*/
1098                 wr32(hw, NGBE_GPIODATA, 0);
1099         }
1100
1101         /*
1102          * Update link status right before return, because it may
1103          * start link configuration process in a separate thread.
1104          */
1105         ngbe_dev_link_update(dev, 0);
1106
1107         ngbe_read_stats_registers(hw, hw_stats);
1108         hw->offset_loaded = 1;
1109
1110         return 0;
1111
1112 error:
1113         PMD_INIT_LOG(ERR, "failure in dev start: %d", err);
1114         ngbe_dev_clear_queues(dev);
1115         return -EIO;
1116 }
1117
1118 /*
1119  * Stop device: disable rx and tx functions to allow for reconfiguring.
1120  */
1121 static int
1122 ngbe_dev_stop(struct rte_eth_dev *dev)
1123 {
1124         struct rte_eth_link link;
1125         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
1126         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1127         struct ngbe_vf_info *vfinfo = *NGBE_DEV_VFDATA(dev);
1128         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1129         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1130         int vf;
1131
1132         if (hw->adapter_stopped)
1133                 return 0;
1134
1135         PMD_INIT_FUNC_TRACE();
1136
1137         rte_eal_alarm_cancel(ngbe_dev_setup_link_alarm_handler, dev);
1138
1139         if ((hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_M88E1512_SFP ||
1140                 (hw->sub_system_id & NGBE_OEM_MASK) == NGBE_LY_YT8521S_SFP) {
1141                 /* gpio0 is used to power on/off control*/
1142                 wr32(hw, NGBE_GPIODATA, NGBE_GPIOBIT_0);
1143         }
1144
1145         /* disable interrupts */
1146         ngbe_disable_intr(hw);
1147
1148         /* reset the NIC */
1149         ngbe_pf_reset_hw(hw);
1150         hw->adapter_stopped = 0;
1151
1152         /* stop adapter */
1153         ngbe_stop_hw(hw);
1154
1155         for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++)
1156                 vfinfo[vf].clear_to_send = false;
1157
1158         ngbe_dev_clear_queues(dev);
1159
1160         /* Clear stored conf */
1161         dev->data->scattered_rx = 0;
1162
1163         /* Clear recorded link status */
1164         memset(&link, 0, sizeof(link));
1165         rte_eth_linkstatus_set(dev, &link);
1166
1167         if (!rte_intr_allow_others(intr_handle))
1168                 /* resume to the default handler */
1169                 rte_intr_callback_register(intr_handle,
1170                                            ngbe_dev_interrupt_handler,
1171                                            (void *)dev);
1172
1173         /* Clean datapath event and queue/vec mapping */
1174         rte_intr_efd_disable(intr_handle);
1175         rte_intr_vec_list_free(intr_handle);
1176
1177         adapter->rss_reta_updated = 0;
1178
1179         hw->adapter_stopped = true;
1180         dev->data->dev_started = 0;
1181
1182         return 0;
1183 }
1184
1185 /*
1186  * Reset and stop device.
1187  */
1188 static int
1189 ngbe_dev_close(struct rte_eth_dev *dev)
1190 {
1191         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1192         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1193         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1194         int retries = 0;
1195         int ret;
1196
1197         PMD_INIT_FUNC_TRACE();
1198
1199         ngbe_pf_reset_hw(hw);
1200
1201         ngbe_dev_stop(dev);
1202
1203         ngbe_dev_free_queues(dev);
1204
1205         /* reprogram the RAR[0] in case user changed it. */
1206         ngbe_set_rar(hw, 0, hw->mac.addr, 0, true);
1207
1208         /* Unlock any pending hardware semaphore */
1209         ngbe_swfw_lock_reset(hw);
1210
1211         /* disable uio intr before callback unregister */
1212         rte_intr_disable(intr_handle);
1213
1214         do {
1215                 ret = rte_intr_callback_unregister(intr_handle,
1216                                 ngbe_dev_interrupt_handler, dev);
1217                 if (ret >= 0 || ret == -ENOENT) {
1218                         break;
1219                 } else if (ret != -EAGAIN) {
1220                         PMD_INIT_LOG(ERR,
1221                                 "intr callback unregister failed: %d",
1222                                 ret);
1223                 }
1224                 rte_delay_ms(100);
1225         } while (retries++ < (10 + NGBE_LINK_UP_TIME));
1226
1227         /* uninitialize PF if max_vfs not zero */
1228         ngbe_pf_host_uninit(dev);
1229
1230         rte_free(dev->data->mac_addrs);
1231         dev->data->mac_addrs = NULL;
1232
1233         rte_free(dev->data->hash_mac_addrs);
1234         dev->data->hash_mac_addrs = NULL;
1235
1236         return ret;
1237 }
1238
1239 /*
1240  * Reset PF device.
1241  */
1242 static int
1243 ngbe_dev_reset(struct rte_eth_dev *dev)
1244 {
1245         int ret;
1246
1247         /* When a DPDK PMD PF begin to reset PF port, it should notify all
1248          * its VF to make them align with it. The detailed notification
1249          * mechanism is PMD specific. As to ngbe PF, it is rather complex.
1250          * To avoid unexpected behavior in VF, currently reset of PF with
1251          * SR-IOV activation is not supported. It might be supported later.
1252          */
1253         if (dev->data->sriov.active)
1254                 return -ENOTSUP;
1255
1256         ret = eth_ngbe_dev_uninit(dev);
1257         if (ret != 0)
1258                 return ret;
1259
1260         ret = eth_ngbe_dev_init(dev, NULL);
1261
1262         return ret;
1263 }
1264
1265 #define UPDATE_QP_COUNTER_32bit(reg, last_counter, counter)     \
1266         {                                                       \
1267                 uint32_t current_counter = rd32(hw, reg);       \
1268                 if (current_counter < last_counter)             \
1269                         current_counter += 0x100000000LL;       \
1270                 if (!hw->offset_loaded)                         \
1271                         last_counter = current_counter;         \
1272                 counter = current_counter - last_counter;       \
1273                 counter &= 0xFFFFFFFFLL;                        \
1274         }
1275
1276 #define UPDATE_QP_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
1277         {                                                                \
1278                 uint64_t current_counter_lsb = rd32(hw, reg_lsb);        \
1279                 uint64_t current_counter_msb = rd32(hw, reg_msb);        \
1280                 uint64_t current_counter = (current_counter_msb << 32) | \
1281                         current_counter_lsb;                             \
1282                 if (current_counter < last_counter)                      \
1283                         current_counter += 0x1000000000LL;               \
1284                 if (!hw->offset_loaded)                                  \
1285                         last_counter = current_counter;                  \
1286                 counter = current_counter - last_counter;                \
1287                 counter &= 0xFFFFFFFFFLL;                                \
1288         }
1289
1290 void
1291 ngbe_read_stats_registers(struct ngbe_hw *hw,
1292                            struct ngbe_hw_stats *hw_stats)
1293 {
1294         unsigned int i;
1295
1296         /* QP Stats */
1297         for (i = 0; i < hw->nb_rx_queues; i++) {
1298                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXPKT(i),
1299                         hw->qp_last[i].rx_qp_packets,
1300                         hw_stats->qp[i].rx_qp_packets);
1301                 UPDATE_QP_COUNTER_36bit(NGBE_QPRXOCTL(i), NGBE_QPRXOCTH(i),
1302                         hw->qp_last[i].rx_qp_bytes,
1303                         hw_stats->qp[i].rx_qp_bytes);
1304                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXMPKT(i),
1305                         hw->qp_last[i].rx_qp_mc_packets,
1306                         hw_stats->qp[i].rx_qp_mc_packets);
1307                 UPDATE_QP_COUNTER_32bit(NGBE_QPRXBPKT(i),
1308                         hw->qp_last[i].rx_qp_bc_packets,
1309                         hw_stats->qp[i].rx_qp_bc_packets);
1310         }
1311
1312         for (i = 0; i < hw->nb_tx_queues; i++) {
1313                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXPKT(i),
1314                         hw->qp_last[i].tx_qp_packets,
1315                         hw_stats->qp[i].tx_qp_packets);
1316                 UPDATE_QP_COUNTER_36bit(NGBE_QPTXOCTL(i), NGBE_QPTXOCTH(i),
1317                         hw->qp_last[i].tx_qp_bytes,
1318                         hw_stats->qp[i].tx_qp_bytes);
1319                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXMPKT(i),
1320                         hw->qp_last[i].tx_qp_mc_packets,
1321                         hw_stats->qp[i].tx_qp_mc_packets);
1322                 UPDATE_QP_COUNTER_32bit(NGBE_QPTXBPKT(i),
1323                         hw->qp_last[i].tx_qp_bc_packets,
1324                         hw_stats->qp[i].tx_qp_bc_packets);
1325         }
1326
1327         /* PB Stats */
1328         hw_stats->rx_up_dropped += rd32(hw, NGBE_PBRXMISS);
1329         hw_stats->rdb_pkt_cnt += rd32(hw, NGBE_PBRXPKT);
1330         hw_stats->rdb_repli_cnt += rd32(hw, NGBE_PBRXREP);
1331         hw_stats->rdb_drp_cnt += rd32(hw, NGBE_PBRXDROP);
1332         hw_stats->tx_xoff_packets += rd32(hw, NGBE_PBTXLNKXOFF);
1333         hw_stats->tx_xon_packets += rd32(hw, NGBE_PBTXLNKXON);
1334
1335         hw_stats->rx_xon_packets += rd32(hw, NGBE_PBRXLNKXON);
1336         hw_stats->rx_xoff_packets += rd32(hw, NGBE_PBRXLNKXOFF);
1337
1338         /* DMA Stats */
1339         hw_stats->rx_drop_packets += rd32(hw, NGBE_DMARXDROP);
1340         hw_stats->tx_drop_packets += rd32(hw, NGBE_DMATXDROP);
1341         hw_stats->rx_dma_drop += rd32(hw, NGBE_DMARXDROP);
1342         hw_stats->tx_secdrp_packets += rd32(hw, NGBE_DMATXSECDROP);
1343         hw_stats->rx_packets += rd32(hw, NGBE_DMARXPKT);
1344         hw_stats->tx_packets += rd32(hw, NGBE_DMATXPKT);
1345         hw_stats->rx_bytes += rd64(hw, NGBE_DMARXOCTL);
1346         hw_stats->tx_bytes += rd64(hw, NGBE_DMATXOCTL);
1347
1348         /* MAC Stats */
1349         hw_stats->rx_crc_errors += rd64(hw, NGBE_MACRXERRCRCL);
1350         hw_stats->rx_multicast_packets += rd64(hw, NGBE_MACRXMPKTL);
1351         hw_stats->tx_multicast_packets += rd64(hw, NGBE_MACTXMPKTL);
1352
1353         hw_stats->rx_total_packets += rd64(hw, NGBE_MACRXPKTL);
1354         hw_stats->tx_total_packets += rd64(hw, NGBE_MACTXPKTL);
1355         hw_stats->rx_total_bytes += rd64(hw, NGBE_MACRXGBOCTL);
1356
1357         hw_stats->rx_broadcast_packets += rd64(hw, NGBE_MACRXOCTL);
1358         hw_stats->tx_broadcast_packets += rd32(hw, NGBE_MACTXOCTL);
1359
1360         hw_stats->rx_size_64_packets += rd64(hw, NGBE_MACRX1TO64L);
1361         hw_stats->rx_size_65_to_127_packets += rd64(hw, NGBE_MACRX65TO127L);
1362         hw_stats->rx_size_128_to_255_packets += rd64(hw, NGBE_MACRX128TO255L);
1363         hw_stats->rx_size_256_to_511_packets += rd64(hw, NGBE_MACRX256TO511L);
1364         hw_stats->rx_size_512_to_1023_packets +=
1365                         rd64(hw, NGBE_MACRX512TO1023L);
1366         hw_stats->rx_size_1024_to_max_packets +=
1367                         rd64(hw, NGBE_MACRX1024TOMAXL);
1368         hw_stats->tx_size_64_packets += rd64(hw, NGBE_MACTX1TO64L);
1369         hw_stats->tx_size_65_to_127_packets += rd64(hw, NGBE_MACTX65TO127L);
1370         hw_stats->tx_size_128_to_255_packets += rd64(hw, NGBE_MACTX128TO255L);
1371         hw_stats->tx_size_256_to_511_packets += rd64(hw, NGBE_MACTX256TO511L);
1372         hw_stats->tx_size_512_to_1023_packets +=
1373                         rd64(hw, NGBE_MACTX512TO1023L);
1374         hw_stats->tx_size_1024_to_max_packets +=
1375                         rd64(hw, NGBE_MACTX1024TOMAXL);
1376
1377         hw_stats->rx_undersize_errors += rd64(hw, NGBE_MACRXERRLENL);
1378         hw_stats->rx_oversize_errors += rd32(hw, NGBE_MACRXOVERSIZE);
1379         hw_stats->rx_jabber_errors += rd32(hw, NGBE_MACRXJABBER);
1380
1381         /* MNG Stats */
1382         hw_stats->mng_bmc2host_packets = rd32(hw, NGBE_MNGBMC2OS);
1383         hw_stats->mng_host2bmc_packets = rd32(hw, NGBE_MNGOS2BMC);
1384         hw_stats->rx_management_packets = rd32(hw, NGBE_DMARXMNG);
1385         hw_stats->tx_management_packets = rd32(hw, NGBE_DMATXMNG);
1386
1387         /* MACsec Stats */
1388         hw_stats->tx_macsec_pkts_untagged += rd32(hw, NGBE_LSECTX_UTPKT);
1389         hw_stats->tx_macsec_pkts_encrypted +=
1390                         rd32(hw, NGBE_LSECTX_ENCPKT);
1391         hw_stats->tx_macsec_pkts_protected +=
1392                         rd32(hw, NGBE_LSECTX_PROTPKT);
1393         hw_stats->tx_macsec_octets_encrypted +=
1394                         rd32(hw, NGBE_LSECTX_ENCOCT);
1395         hw_stats->tx_macsec_octets_protected +=
1396                         rd32(hw, NGBE_LSECTX_PROTOCT);
1397         hw_stats->rx_macsec_pkts_untagged += rd32(hw, NGBE_LSECRX_UTPKT);
1398         hw_stats->rx_macsec_pkts_badtag += rd32(hw, NGBE_LSECRX_BTPKT);
1399         hw_stats->rx_macsec_pkts_nosci += rd32(hw, NGBE_LSECRX_NOSCIPKT);
1400         hw_stats->rx_macsec_pkts_unknownsci += rd32(hw, NGBE_LSECRX_UNSCIPKT);
1401         hw_stats->rx_macsec_octets_decrypted += rd32(hw, NGBE_LSECRX_DECOCT);
1402         hw_stats->rx_macsec_octets_validated += rd32(hw, NGBE_LSECRX_VLDOCT);
1403         hw_stats->rx_macsec_sc_pkts_unchecked +=
1404                         rd32(hw, NGBE_LSECRX_UNCHKPKT);
1405         hw_stats->rx_macsec_sc_pkts_delayed += rd32(hw, NGBE_LSECRX_DLYPKT);
1406         hw_stats->rx_macsec_sc_pkts_late += rd32(hw, NGBE_LSECRX_LATEPKT);
1407         for (i = 0; i < 2; i++) {
1408                 hw_stats->rx_macsec_sa_pkts_ok +=
1409                         rd32(hw, NGBE_LSECRX_OKPKT(i));
1410                 hw_stats->rx_macsec_sa_pkts_invalid +=
1411                         rd32(hw, NGBE_LSECRX_INVPKT(i));
1412                 hw_stats->rx_macsec_sa_pkts_notvalid +=
1413                         rd32(hw, NGBE_LSECRX_BADPKT(i));
1414         }
1415         for (i = 0; i < 4; i++) {
1416                 hw_stats->rx_macsec_sa_pkts_unusedsa +=
1417                         rd32(hw, NGBE_LSECRX_INVSAPKT(i));
1418                 hw_stats->rx_macsec_sa_pkts_notusingsa +=
1419                         rd32(hw, NGBE_LSECRX_BADSAPKT(i));
1420         }
1421         hw_stats->rx_total_missed_packets =
1422                         hw_stats->rx_up_dropped;
1423 }
1424
1425 static int
1426 ngbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1427 {
1428         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1429         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1430         struct ngbe_stat_mappings *stat_mappings =
1431                         NGBE_DEV_STAT_MAPPINGS(dev);
1432         uint32_t i, j;
1433
1434         ngbe_read_stats_registers(hw, hw_stats);
1435
1436         if (stats == NULL)
1437                 return -EINVAL;
1438
1439         /* Fill out the rte_eth_stats statistics structure */
1440         stats->ipackets = hw_stats->rx_packets;
1441         stats->ibytes = hw_stats->rx_bytes;
1442         stats->opackets = hw_stats->tx_packets;
1443         stats->obytes = hw_stats->tx_bytes;
1444
1445         memset(&stats->q_ipackets, 0, sizeof(stats->q_ipackets));
1446         memset(&stats->q_opackets, 0, sizeof(stats->q_opackets));
1447         memset(&stats->q_ibytes, 0, sizeof(stats->q_ibytes));
1448         memset(&stats->q_obytes, 0, sizeof(stats->q_obytes));
1449         memset(&stats->q_errors, 0, sizeof(stats->q_errors));
1450         for (i = 0; i < NGBE_MAX_QP; i++) {
1451                 uint32_t n = i / NB_QMAP_FIELDS_PER_QSM_REG;
1452                 uint32_t offset = (i % NB_QMAP_FIELDS_PER_QSM_REG) * 8;
1453                 uint32_t q_map;
1454
1455                 q_map = (stat_mappings->rqsm[n] >> offset)
1456                                 & QMAP_FIELD_RESERVED_BITS_MASK;
1457                 j = (q_map < RTE_ETHDEV_QUEUE_STAT_CNTRS
1458                      ? q_map : q_map % RTE_ETHDEV_QUEUE_STAT_CNTRS);
1459                 stats->q_ipackets[j] += hw_stats->qp[i].rx_qp_packets;
1460                 stats->q_ibytes[j] += hw_stats->qp[i].rx_qp_bytes;
1461
1462                 q_map = (stat_mappings->tqsm[n] >> offset)
1463                                 & QMAP_FIELD_RESERVED_BITS_MASK;
1464                 j = (q_map < RTE_ETHDEV_QUEUE_STAT_CNTRS
1465                      ? q_map : q_map % RTE_ETHDEV_QUEUE_STAT_CNTRS);
1466                 stats->q_opackets[j] += hw_stats->qp[i].tx_qp_packets;
1467                 stats->q_obytes[j] += hw_stats->qp[i].tx_qp_bytes;
1468         }
1469
1470         /* Rx Errors */
1471         stats->imissed  = hw_stats->rx_total_missed_packets +
1472                           hw_stats->rx_dma_drop;
1473         stats->ierrors  = hw_stats->rx_crc_errors +
1474                           hw_stats->rx_mac_short_packet_dropped +
1475                           hw_stats->rx_length_errors +
1476                           hw_stats->rx_undersize_errors +
1477                           hw_stats->rx_oversize_errors +
1478                           hw_stats->rx_illegal_byte_errors +
1479                           hw_stats->rx_error_bytes +
1480                           hw_stats->rx_fragment_errors;
1481
1482         /* Tx Errors */
1483         stats->oerrors  = 0;
1484         return 0;
1485 }
1486
1487 static int
1488 ngbe_dev_stats_reset(struct rte_eth_dev *dev)
1489 {
1490         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1491         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1492
1493         /* HW registers are cleared on read */
1494         hw->offset_loaded = 0;
1495         ngbe_dev_stats_get(dev, NULL);
1496         hw->offset_loaded = 1;
1497
1498         /* Reset software totals */
1499         memset(hw_stats, 0, sizeof(*hw_stats));
1500
1501         return 0;
1502 }
1503
1504 /* This function calculates the number of xstats based on the current config */
1505 static unsigned
1506 ngbe_xstats_calc_num(struct rte_eth_dev *dev)
1507 {
1508         int nb_queues = max(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
1509         return NGBE_NB_HW_STATS +
1510                NGBE_NB_QP_STATS * nb_queues;
1511 }
1512
1513 static inline int
1514 ngbe_get_name_by_id(uint32_t id, char *name, uint32_t size)
1515 {
1516         int nb, st;
1517
1518         /* Extended stats from ngbe_hw_stats */
1519         if (id < NGBE_NB_HW_STATS) {
1520                 snprintf(name, size, "[hw]%s",
1521                         rte_ngbe_stats_strings[id].name);
1522                 return 0;
1523         }
1524         id -= NGBE_NB_HW_STATS;
1525
1526         /* Queue Stats */
1527         if (id < NGBE_NB_QP_STATS * NGBE_MAX_QP) {
1528                 nb = id / NGBE_NB_QP_STATS;
1529                 st = id % NGBE_NB_QP_STATS;
1530                 snprintf(name, size, "[q%u]%s", nb,
1531                         rte_ngbe_qp_strings[st].name);
1532                 return 0;
1533         }
1534         id -= NGBE_NB_QP_STATS * NGBE_MAX_QP;
1535
1536         return -(int)(id + 1);
1537 }
1538
1539 static inline int
1540 ngbe_get_offset_by_id(uint32_t id, uint32_t *offset)
1541 {
1542         int nb, st;
1543
1544         /* Extended stats from ngbe_hw_stats */
1545         if (id < NGBE_NB_HW_STATS) {
1546                 *offset = rte_ngbe_stats_strings[id].offset;
1547                 return 0;
1548         }
1549         id -= NGBE_NB_HW_STATS;
1550
1551         /* Queue Stats */
1552         if (id < NGBE_NB_QP_STATS * NGBE_MAX_QP) {
1553                 nb = id / NGBE_NB_QP_STATS;
1554                 st = id % NGBE_NB_QP_STATS;
1555                 *offset = rte_ngbe_qp_strings[st].offset +
1556                         nb * (NGBE_NB_QP_STATS * sizeof(uint64_t));
1557                 return 0;
1558         }
1559
1560         return -1;
1561 }
1562
1563 static int ngbe_dev_xstats_get_names(struct rte_eth_dev *dev,
1564         struct rte_eth_xstat_name *xstats_names, unsigned int limit)
1565 {
1566         unsigned int i, count;
1567
1568         count = ngbe_xstats_calc_num(dev);
1569         if (xstats_names == NULL)
1570                 return count;
1571
1572         /* Note: limit >= cnt_stats checked upstream
1573          * in rte_eth_xstats_names()
1574          */
1575         limit = min(limit, count);
1576
1577         /* Extended stats from ngbe_hw_stats */
1578         for (i = 0; i < limit; i++) {
1579                 if (ngbe_get_name_by_id(i, xstats_names[i].name,
1580                         sizeof(xstats_names[i].name))) {
1581                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1582                         break;
1583                 }
1584         }
1585
1586         return i;
1587 }
1588
1589 static int ngbe_dev_xstats_get_names_by_id(struct rte_eth_dev *dev,
1590         const uint64_t *ids,
1591         struct rte_eth_xstat_name *xstats_names,
1592         unsigned int limit)
1593 {
1594         unsigned int i;
1595
1596         if (ids == NULL)
1597                 return ngbe_dev_xstats_get_names(dev, xstats_names, limit);
1598
1599         for (i = 0; i < limit; i++) {
1600                 if (ngbe_get_name_by_id(ids[i], xstats_names[i].name,
1601                                 sizeof(xstats_names[i].name))) {
1602                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1603                         return -1;
1604                 }
1605         }
1606
1607         return i;
1608 }
1609
1610 static int
1611 ngbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1612                                          unsigned int limit)
1613 {
1614         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1615         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1616         unsigned int i, count;
1617
1618         ngbe_read_stats_registers(hw, hw_stats);
1619
1620         /* If this is a reset xstats is NULL, and we have cleared the
1621          * registers by reading them.
1622          */
1623         count = ngbe_xstats_calc_num(dev);
1624         if (xstats == NULL)
1625                 return count;
1626
1627         limit = min(limit, ngbe_xstats_calc_num(dev));
1628
1629         /* Extended stats from ngbe_hw_stats */
1630         for (i = 0; i < limit; i++) {
1631                 uint32_t offset = 0;
1632
1633                 if (ngbe_get_offset_by_id(i, &offset)) {
1634                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1635                         break;
1636                 }
1637                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) + offset);
1638                 xstats[i].id = i;
1639         }
1640
1641         return i;
1642 }
1643
1644 static int
1645 ngbe_dev_xstats_get_(struct rte_eth_dev *dev, uint64_t *values,
1646                                          unsigned int limit)
1647 {
1648         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1649         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1650         unsigned int i, count;
1651
1652         ngbe_read_stats_registers(hw, hw_stats);
1653
1654         /* If this is a reset xstats is NULL, and we have cleared the
1655          * registers by reading them.
1656          */
1657         count = ngbe_xstats_calc_num(dev);
1658         if (values == NULL)
1659                 return count;
1660
1661         limit = min(limit, ngbe_xstats_calc_num(dev));
1662
1663         /* Extended stats from ngbe_hw_stats */
1664         for (i = 0; i < limit; i++) {
1665                 uint32_t offset;
1666
1667                 if (ngbe_get_offset_by_id(i, &offset)) {
1668                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1669                         break;
1670                 }
1671                 values[i] = *(uint64_t *)(((char *)hw_stats) + offset);
1672         }
1673
1674         return i;
1675 }
1676
1677 static int
1678 ngbe_dev_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1679                 uint64_t *values, unsigned int limit)
1680 {
1681         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1682         unsigned int i;
1683
1684         if (ids == NULL)
1685                 return ngbe_dev_xstats_get_(dev, values, limit);
1686
1687         for (i = 0; i < limit; i++) {
1688                 uint32_t offset;
1689
1690                 if (ngbe_get_offset_by_id(ids[i], &offset)) {
1691                         PMD_INIT_LOG(WARNING, "id value %d isn't valid", i);
1692                         break;
1693                 }
1694                 values[i] = *(uint64_t *)(((char *)hw_stats) + offset);
1695         }
1696
1697         return i;
1698 }
1699
1700 static int
1701 ngbe_dev_xstats_reset(struct rte_eth_dev *dev)
1702 {
1703         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1704         struct ngbe_hw_stats *hw_stats = NGBE_DEV_STATS(dev);
1705
1706         /* HW registers are cleared on read */
1707         hw->offset_loaded = 0;
1708         ngbe_read_stats_registers(hw, hw_stats);
1709         hw->offset_loaded = 1;
1710
1711         /* Reset software totals */
1712         memset(hw_stats, 0, sizeof(*hw_stats));
1713
1714         return 0;
1715 }
1716
1717 static int
1718 ngbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
1719 {
1720         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1721         int ret;
1722
1723         ret = snprintf(fw_version, fw_size, "0x%08x", hw->eeprom_id);
1724
1725         if (ret < 0)
1726                 return -EINVAL;
1727
1728         ret += 1; /* add the size of '\0' */
1729         if (fw_size < (size_t)ret)
1730                 return ret;
1731
1732         return 0;
1733 }
1734
1735 static int
1736 ngbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1737 {
1738         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1739         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1740
1741         dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
1742         dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
1743         dev_info->min_rx_bufsize = 1024;
1744         dev_info->max_rx_pktlen = 15872;
1745         dev_info->max_mac_addrs = hw->mac.num_rar_entries;
1746         dev_info->max_hash_mac_addrs = NGBE_VMDQ_NUM_UC_MAC;
1747         dev_info->max_vfs = pci_dev->max_vfs;
1748         dev_info->rx_queue_offload_capa = ngbe_get_rx_queue_offloads(dev);
1749         dev_info->rx_offload_capa = (ngbe_get_rx_port_offloads(dev) |
1750                                      dev_info->rx_queue_offload_capa);
1751         dev_info->tx_queue_offload_capa = 0;
1752         dev_info->tx_offload_capa = ngbe_get_tx_port_offloads(dev);
1753
1754         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1755                 .rx_thresh = {
1756                         .pthresh = NGBE_DEFAULT_RX_PTHRESH,
1757                         .hthresh = NGBE_DEFAULT_RX_HTHRESH,
1758                         .wthresh = NGBE_DEFAULT_RX_WTHRESH,
1759                 },
1760                 .rx_free_thresh = NGBE_DEFAULT_RX_FREE_THRESH,
1761                 .rx_drop_en = 0,
1762                 .offloads = 0,
1763         };
1764
1765         dev_info->default_txconf = (struct rte_eth_txconf) {
1766                 .tx_thresh = {
1767                         .pthresh = NGBE_DEFAULT_TX_PTHRESH,
1768                         .hthresh = NGBE_DEFAULT_TX_HTHRESH,
1769                         .wthresh = NGBE_DEFAULT_TX_WTHRESH,
1770                 },
1771                 .tx_free_thresh = NGBE_DEFAULT_TX_FREE_THRESH,
1772                 .offloads = 0,
1773         };
1774
1775         dev_info->rx_desc_lim = rx_desc_lim;
1776         dev_info->tx_desc_lim = tx_desc_lim;
1777
1778         dev_info->hash_key_size = NGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
1779         dev_info->reta_size = RTE_ETH_RSS_RETA_SIZE_128;
1780         dev_info->flow_type_rss_offloads = NGBE_RSS_OFFLOAD_ALL;
1781
1782         dev_info->speed_capa = RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_100M |
1783                                 RTE_ETH_LINK_SPEED_10M;
1784
1785         /* Driver-preferred Rx/Tx parameters */
1786         dev_info->default_rxportconf.burst_size = 32;
1787         dev_info->default_txportconf.burst_size = 32;
1788         dev_info->default_rxportconf.nb_queues = 1;
1789         dev_info->default_txportconf.nb_queues = 1;
1790         dev_info->default_rxportconf.ring_size = 256;
1791         dev_info->default_txportconf.ring_size = 256;
1792
1793         return 0;
1794 }
1795
1796 const uint32_t *
1797 ngbe_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1798 {
1799         if (dev->rx_pkt_burst == ngbe_recv_pkts ||
1800             dev->rx_pkt_burst == ngbe_recv_pkts_sc_single_alloc ||
1801             dev->rx_pkt_burst == ngbe_recv_pkts_sc_bulk_alloc ||
1802             dev->rx_pkt_burst == ngbe_recv_pkts_bulk_alloc)
1803                 return ngbe_get_supported_ptypes();
1804
1805         return NULL;
1806 }
1807
1808 void
1809 ngbe_dev_setup_link_alarm_handler(void *param)
1810 {
1811         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1812         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1813         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1814         u32 speed;
1815         bool autoneg = false;
1816
1817         speed = hw->phy.autoneg_advertised;
1818         if (!speed)
1819                 hw->mac.get_link_capabilities(hw, &speed, &autoneg);
1820
1821         hw->mac.setup_link(hw, speed, true);
1822
1823         intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
1824 }
1825
1826 /* return 0 means link status changed, -1 means not changed */
1827 int
1828 ngbe_dev_link_update_share(struct rte_eth_dev *dev,
1829                             int wait_to_complete)
1830 {
1831         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1832         struct rte_eth_link link;
1833         u32 link_speed = NGBE_LINK_SPEED_UNKNOWN;
1834         u32 lan_speed = 0;
1835         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1836         bool link_up;
1837         int err;
1838         int wait = 1;
1839
1840         memset(&link, 0, sizeof(link));
1841         link.link_status = RTE_ETH_LINK_DOWN;
1842         link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1843         link.link_duplex = RTE_ETH_LINK_HALF_DUPLEX;
1844         link.link_autoneg = !(dev->data->dev_conf.link_speeds &
1845                         ~RTE_ETH_LINK_SPEED_AUTONEG);
1846
1847         hw->mac.get_link_status = true;
1848
1849         if (intr->flags & NGBE_FLAG_NEED_LINK_CONFIG)
1850                 return rte_eth_linkstatus_set(dev, &link);
1851
1852         /* check if it needs to wait to complete, if lsc interrupt is enabled */
1853         if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
1854                 wait = 0;
1855
1856         err = hw->mac.check_link(hw, &link_speed, &link_up, wait);
1857         if (err != 0) {
1858                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1859                 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1860                 return rte_eth_linkstatus_set(dev, &link);
1861         }
1862
1863         if (!link_up) {
1864                 if (hw->phy.media_type == ngbe_media_type_fiber &&
1865                         hw->phy.type != ngbe_phy_mvl_sfi) {
1866                         intr->flags |= NGBE_FLAG_NEED_LINK_CONFIG;
1867                         rte_eal_alarm_set(10,
1868                                 ngbe_dev_setup_link_alarm_handler, dev);
1869                 }
1870
1871                 return rte_eth_linkstatus_set(dev, &link);
1872         }
1873
1874         intr->flags &= ~NGBE_FLAG_NEED_LINK_CONFIG;
1875         link.link_status = RTE_ETH_LINK_UP;
1876         link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1877
1878         switch (link_speed) {
1879         default:
1880         case NGBE_LINK_SPEED_UNKNOWN:
1881                 link.link_speed = RTE_ETH_SPEED_NUM_NONE;
1882                 break;
1883
1884         case NGBE_LINK_SPEED_10M_FULL:
1885                 link.link_speed = RTE_ETH_SPEED_NUM_10M;
1886                 lan_speed = 0;
1887                 break;
1888
1889         case NGBE_LINK_SPEED_100M_FULL:
1890                 link.link_speed = RTE_ETH_SPEED_NUM_100M;
1891                 lan_speed = 1;
1892                 break;
1893
1894         case NGBE_LINK_SPEED_1GB_FULL:
1895                 link.link_speed = RTE_ETH_SPEED_NUM_1G;
1896                 lan_speed = 2;
1897                 break;
1898         }
1899
1900         if (hw->is_pf) {
1901                 wr32m(hw, NGBE_LAN_SPEED, NGBE_LAN_SPEED_MASK, lan_speed);
1902                 if (link_speed & (NGBE_LINK_SPEED_1GB_FULL |
1903                                 NGBE_LINK_SPEED_100M_FULL |
1904                                 NGBE_LINK_SPEED_10M_FULL)) {
1905                         wr32m(hw, NGBE_MACTXCFG, NGBE_MACTXCFG_SPEED_MASK,
1906                                 NGBE_MACTXCFG_SPEED_1G | NGBE_MACTXCFG_TE);
1907                 }
1908         }
1909
1910         return rte_eth_linkstatus_set(dev, &link);
1911 }
1912
1913 static int
1914 ngbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1915 {
1916         return ngbe_dev_link_update_share(dev, wait_to_complete);
1917 }
1918
1919 static int
1920 ngbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
1921 {
1922         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1923         uint32_t fctrl;
1924
1925         fctrl = rd32(hw, NGBE_PSRCTL);
1926         fctrl |= (NGBE_PSRCTL_UCP | NGBE_PSRCTL_MCP);
1927         wr32(hw, NGBE_PSRCTL, fctrl);
1928
1929         return 0;
1930 }
1931
1932 static int
1933 ngbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
1934 {
1935         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1936         uint32_t fctrl;
1937
1938         fctrl = rd32(hw, NGBE_PSRCTL);
1939         fctrl &= (~NGBE_PSRCTL_UCP);
1940         if (dev->data->all_multicast == 1)
1941                 fctrl |= NGBE_PSRCTL_MCP;
1942         else
1943                 fctrl &= (~NGBE_PSRCTL_MCP);
1944         wr32(hw, NGBE_PSRCTL, fctrl);
1945
1946         return 0;
1947 }
1948
1949 static int
1950 ngbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
1951 {
1952         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1953         uint32_t fctrl;
1954
1955         fctrl = rd32(hw, NGBE_PSRCTL);
1956         fctrl |= NGBE_PSRCTL_MCP;
1957         wr32(hw, NGBE_PSRCTL, fctrl);
1958
1959         return 0;
1960 }
1961
1962 static int
1963 ngbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
1964 {
1965         struct ngbe_hw *hw = ngbe_dev_hw(dev);
1966         uint32_t fctrl;
1967
1968         if (dev->data->promiscuous == 1)
1969                 return 0; /* must remain in all_multicast mode */
1970
1971         fctrl = rd32(hw, NGBE_PSRCTL);
1972         fctrl &= (~NGBE_PSRCTL_MCP);
1973         wr32(hw, NGBE_PSRCTL, fctrl);
1974
1975         return 0;
1976 }
1977
1978 /**
1979  * It clears the interrupt causes and enables the interrupt.
1980  * It will be called once only during NIC initialized.
1981  *
1982  * @param dev
1983  *  Pointer to struct rte_eth_dev.
1984  * @param on
1985  *  Enable or Disable.
1986  *
1987  * @return
1988  *  - On success, zero.
1989  *  - On failure, a negative value.
1990  */
1991 static int
1992 ngbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
1993 {
1994         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
1995
1996         ngbe_dev_link_status_print(dev);
1997         if (on != 0) {
1998                 intr->mask_misc |= NGBE_ICRMISC_PHY;
1999                 intr->mask_misc |= NGBE_ICRMISC_GPIO;
2000         } else {
2001                 intr->mask_misc &= ~NGBE_ICRMISC_PHY;
2002                 intr->mask_misc &= ~NGBE_ICRMISC_GPIO;
2003         }
2004
2005         return 0;
2006 }
2007
2008 /**
2009  * It clears the interrupt causes and enables the interrupt.
2010  * It will be called once only during NIC initialized.
2011  *
2012  * @param dev
2013  *  Pointer to struct rte_eth_dev.
2014  *
2015  * @return
2016  *  - On success, zero.
2017  *  - On failure, a negative value.
2018  */
2019 static int
2020 ngbe_dev_misc_interrupt_setup(struct rte_eth_dev *dev)
2021 {
2022         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2023         u64 mask;
2024
2025         mask = NGBE_ICR_MASK;
2026         mask &= (1ULL << NGBE_MISC_VEC_ID);
2027         intr->mask |= mask;
2028         intr->mask_misc |= NGBE_ICRMISC_GPIO;
2029
2030         return 0;
2031 }
2032
2033 /**
2034  * It clears the interrupt causes and enables the interrupt.
2035  * It will be called once only during NIC initialized.
2036  *
2037  * @param dev
2038  *  Pointer to struct rte_eth_dev.
2039  *
2040  * @return
2041  *  - On success, zero.
2042  *  - On failure, a negative value.
2043  */
2044 static int
2045 ngbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2046 {
2047         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2048         u64 mask;
2049
2050         mask = NGBE_ICR_MASK;
2051         mask &= ~((1ULL << NGBE_RX_VEC_START) - 1);
2052         intr->mask |= mask;
2053
2054         return 0;
2055 }
2056
2057 /**
2058  * It clears the interrupt causes and enables the interrupt.
2059  * It will be called once only during NIC initialized.
2060  *
2061  * @param dev
2062  *  Pointer to struct rte_eth_dev.
2063  *
2064  * @return
2065  *  - On success, zero.
2066  *  - On failure, a negative value.
2067  */
2068 static int
2069 ngbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev)
2070 {
2071         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2072
2073         intr->mask_misc |= NGBE_ICRMISC_LNKSEC;
2074
2075         return 0;
2076 }
2077
2078 /*
2079  * It reads ICR and sets flag for the link_update.
2080  *
2081  * @param dev
2082  *  Pointer to struct rte_eth_dev.
2083  *
2084  * @return
2085  *  - On success, zero.
2086  *  - On failure, a negative value.
2087  */
2088 static int
2089 ngbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
2090 {
2091         uint32_t eicr;
2092         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2093         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2094
2095         /* read-on-clear nic registers here */
2096         eicr = ((u32 *)hw->isb_mem)[NGBE_ISB_MISC];
2097         PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
2098
2099         intr->flags = 0;
2100
2101         /* set flag for async link update */
2102         if (eicr & NGBE_ICRMISC_PHY)
2103                 intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
2104
2105         if (eicr & NGBE_ICRMISC_VFMBX)
2106                 intr->flags |= NGBE_FLAG_MAILBOX;
2107
2108         if (eicr & NGBE_ICRMISC_LNKSEC)
2109                 intr->flags |= NGBE_FLAG_MACSEC;
2110
2111         if (eicr & NGBE_ICRMISC_GPIO)
2112                 intr->flags |= NGBE_FLAG_NEED_LINK_UPDATE;
2113
2114         ((u32 *)hw->isb_mem)[NGBE_ISB_MISC] = 0;
2115
2116         return 0;
2117 }
2118
2119 /**
2120  * It gets and then prints the link status.
2121  *
2122  * @param dev
2123  *  Pointer to struct rte_eth_dev.
2124  *
2125  * @return
2126  *  - On success, zero.
2127  *  - On failure, a negative value.
2128  */
2129 static void
2130 ngbe_dev_link_status_print(struct rte_eth_dev *dev)
2131 {
2132         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2133         struct rte_eth_link link;
2134
2135         rte_eth_linkstatus_get(dev, &link);
2136
2137         if (link.link_status == RTE_ETH_LINK_UP) {
2138                 PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
2139                                         (int)(dev->data->port_id),
2140                                         (unsigned int)link.link_speed,
2141                         link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX ?
2142                                         "full-duplex" : "half-duplex");
2143         } else {
2144                 PMD_INIT_LOG(INFO, " Port %d: Link Down",
2145                                 (int)(dev->data->port_id));
2146         }
2147         PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
2148                                 pci_dev->addr.domain,
2149                                 pci_dev->addr.bus,
2150                                 pci_dev->addr.devid,
2151                                 pci_dev->addr.function);
2152 }
2153
2154 /*
2155  * It executes link_update after knowing an interrupt occurred.
2156  *
2157  * @param dev
2158  *  Pointer to struct rte_eth_dev.
2159  *
2160  * @return
2161  *  - On success, zero.
2162  *  - On failure, a negative value.
2163  */
2164 static int
2165 ngbe_dev_interrupt_action(struct rte_eth_dev *dev)
2166 {
2167         struct ngbe_interrupt *intr = ngbe_dev_intr(dev);
2168
2169         PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
2170
2171         if (intr->flags & NGBE_FLAG_MAILBOX) {
2172                 ngbe_pf_mbx_process(dev);
2173                 intr->flags &= ~NGBE_FLAG_MAILBOX;
2174         }
2175
2176         if (intr->flags & NGBE_FLAG_NEED_LINK_UPDATE) {
2177                 struct rte_eth_link link;
2178
2179                 /*get the link status before link update, for predicting later*/
2180                 rte_eth_linkstatus_get(dev, &link);
2181
2182                 ngbe_dev_link_update(dev, 0);
2183                 intr->flags &= ~NGBE_FLAG_NEED_LINK_UPDATE;
2184                 ngbe_dev_link_status_print(dev);
2185                 if (dev->data->dev_link.link_speed != link.link_speed)
2186                         rte_eth_dev_callback_process(dev,
2187                                 RTE_ETH_EVENT_INTR_LSC, NULL);
2188         }
2189
2190         PMD_DRV_LOG(DEBUG, "enable intr immediately");
2191         ngbe_enable_intr(dev);
2192
2193         return 0;
2194 }
2195
2196 /**
2197  * Interrupt handler triggered by NIC  for handling
2198  * specific interrupt.
2199  *
2200  * @param param
2201  *  The address of parameter (struct rte_eth_dev *) registered before.
2202  */
2203 static void
2204 ngbe_dev_interrupt_handler(void *param)
2205 {
2206         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2207
2208         ngbe_dev_interrupt_get_status(dev);
2209         ngbe_dev_interrupt_action(dev);
2210 }
2211
2212 static int
2213 ngbe_dev_led_on(struct rte_eth_dev *dev)
2214 {
2215         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2216         return hw->mac.led_on(hw, 0) == 0 ? 0 : -ENOTSUP;
2217 }
2218
2219 static int
2220 ngbe_dev_led_off(struct rte_eth_dev *dev)
2221 {
2222         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2223         return hw->mac.led_off(hw, 0) == 0 ? 0 : -ENOTSUP;
2224 }
2225
2226 static int
2227 ngbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2228 {
2229         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2230         uint32_t mflcn_reg;
2231         uint32_t fccfg_reg;
2232         int rx_pause;
2233         int tx_pause;
2234
2235         fc_conf->pause_time = hw->fc.pause_time;
2236         fc_conf->high_water = hw->fc.high_water;
2237         fc_conf->low_water = hw->fc.low_water;
2238         fc_conf->send_xon = hw->fc.send_xon;
2239         fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
2240
2241         /*
2242          * Return rx_pause status according to actual setting of
2243          * RXFCCFG register.
2244          */
2245         mflcn_reg = rd32(hw, NGBE_RXFCCFG);
2246         if (mflcn_reg & NGBE_RXFCCFG_FC)
2247                 rx_pause = 1;
2248         else
2249                 rx_pause = 0;
2250
2251         /*
2252          * Return tx_pause status according to actual setting of
2253          * TXFCCFG register.
2254          */
2255         fccfg_reg = rd32(hw, NGBE_TXFCCFG);
2256         if (fccfg_reg & NGBE_TXFCCFG_FC)
2257                 tx_pause = 1;
2258         else
2259                 tx_pause = 0;
2260
2261         if (rx_pause && tx_pause)
2262                 fc_conf->mode = RTE_ETH_FC_FULL;
2263         else if (rx_pause)
2264                 fc_conf->mode = RTE_ETH_FC_RX_PAUSE;
2265         else if (tx_pause)
2266                 fc_conf->mode = RTE_ETH_FC_TX_PAUSE;
2267         else
2268                 fc_conf->mode = RTE_ETH_FC_NONE;
2269
2270         return 0;
2271 }
2272
2273 static int
2274 ngbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2275 {
2276         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2277         int err;
2278         uint32_t rx_buf_size;
2279         uint32_t max_high_water;
2280         enum ngbe_fc_mode rte_fcmode_2_ngbe_fcmode[] = {
2281                 ngbe_fc_none,
2282                 ngbe_fc_rx_pause,
2283                 ngbe_fc_tx_pause,
2284                 ngbe_fc_full
2285         };
2286
2287         PMD_INIT_FUNC_TRACE();
2288
2289         rx_buf_size = rd32(hw, NGBE_PBRXSIZE);
2290         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2291
2292         /*
2293          * At least reserve one Ethernet frame for watermark
2294          * high_water/low_water in kilo bytes for ngbe
2295          */
2296         max_high_water = (rx_buf_size - RTE_ETHER_MAX_LEN) >> 10;
2297         if (fc_conf->high_water > max_high_water ||
2298             fc_conf->high_water < fc_conf->low_water) {
2299                 PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
2300                 PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
2301                 return -EINVAL;
2302         }
2303
2304         hw->fc.requested_mode = rte_fcmode_2_ngbe_fcmode[fc_conf->mode];
2305         hw->fc.pause_time     = fc_conf->pause_time;
2306         hw->fc.high_water     = fc_conf->high_water;
2307         hw->fc.low_water      = fc_conf->low_water;
2308         hw->fc.send_xon       = fc_conf->send_xon;
2309         hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
2310
2311         err = hw->mac.fc_enable(hw);
2312
2313         /* Not negotiated is not an error case */
2314         if (err == 0 || err == NGBE_ERR_FC_NOT_NEGOTIATED) {
2315                 wr32m(hw, NGBE_MACRXFLT, NGBE_MACRXFLT_CTL_MASK,
2316                       (fc_conf->mac_ctrl_frame_fwd
2317                        ? NGBE_MACRXFLT_CTL_NOPS : NGBE_MACRXFLT_CTL_DROP));
2318                 ngbe_flush(hw);
2319
2320                 return 0;
2321         }
2322
2323         PMD_INIT_LOG(ERR, "ngbe_fc_enable = 0x%x", err);
2324         return -EIO;
2325 }
2326
2327 int
2328 ngbe_dev_rss_reta_update(struct rte_eth_dev *dev,
2329                           struct rte_eth_rss_reta_entry64 *reta_conf,
2330                           uint16_t reta_size)
2331 {
2332         uint8_t i, j, mask;
2333         uint32_t reta;
2334         uint16_t idx, shift;
2335         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2336         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2337
2338         PMD_INIT_FUNC_TRACE();
2339
2340         if (!hw->is_pf) {
2341                 PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
2342                         "NIC.");
2343                 return -ENOTSUP;
2344         }
2345
2346         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2347                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2348                         "(%d) doesn't match the number hardware can supported "
2349                         "(%d)", reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2350                 return -EINVAL;
2351         }
2352
2353         for (i = 0; i < reta_size; i += 4) {
2354                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2355                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2356                 mask = (uint8_t)RS64(reta_conf[idx].mask, shift, 0xF);
2357                 if (!mask)
2358                         continue;
2359
2360                 reta = rd32a(hw, NGBE_REG_RSSTBL, i >> 2);
2361                 for (j = 0; j < 4; j++) {
2362                         if (RS8(mask, j, 0x1)) {
2363                                 reta  &= ~(MS32(8 * j, 0xFF));
2364                                 reta |= LS32(reta_conf[idx].reta[shift + j],
2365                                                 8 * j, 0xFF);
2366                         }
2367                 }
2368                 wr32a(hw, NGBE_REG_RSSTBL, i >> 2, reta);
2369         }
2370         adapter->rss_reta_updated = 1;
2371
2372         return 0;
2373 }
2374
2375 int
2376 ngbe_dev_rss_reta_query(struct rte_eth_dev *dev,
2377                          struct rte_eth_rss_reta_entry64 *reta_conf,
2378                          uint16_t reta_size)
2379 {
2380         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2381         uint8_t i, j, mask;
2382         uint32_t reta;
2383         uint16_t idx, shift;
2384
2385         PMD_INIT_FUNC_TRACE();
2386
2387         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2388                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2389                         "(%d) doesn't match the number hardware can supported "
2390                         "(%d)", reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2391                 return -EINVAL;
2392         }
2393
2394         for (i = 0; i < reta_size; i += 4) {
2395                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2396                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2397                 mask = (uint8_t)RS64(reta_conf[idx].mask, shift, 0xF);
2398                 if (!mask)
2399                         continue;
2400
2401                 reta = rd32a(hw, NGBE_REG_RSSTBL, i >> 2);
2402                 for (j = 0; j < 4; j++) {
2403                         if (RS8(mask, j, 0x1))
2404                                 reta_conf[idx].reta[shift + j] =
2405                                         (uint16_t)RS32(reta, 8 * j, 0xFF);
2406                 }
2407         }
2408
2409         return 0;
2410 }
2411
2412 static int
2413 ngbe_add_rar(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
2414                                 uint32_t index, uint32_t pool)
2415 {
2416         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2417         uint32_t enable_addr = 1;
2418
2419         return ngbe_set_rar(hw, index, mac_addr->addr_bytes,
2420                              pool, enable_addr);
2421 }
2422
2423 static void
2424 ngbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
2425 {
2426         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2427
2428         ngbe_clear_rar(hw, index);
2429 }
2430
2431 static int
2432 ngbe_set_default_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr)
2433 {
2434         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2435
2436         ngbe_remove_rar(dev, 0);
2437         ngbe_add_rar(dev, addr, 0, pci_dev->max_vfs);
2438
2439         return 0;
2440 }
2441
2442 static int
2443 ngbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
2444 {
2445         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2446         uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 4;
2447         struct rte_eth_dev_data *dev_data = dev->data;
2448
2449         /* If device is started, refuse mtu that requires the support of
2450          * scattered packets when this feature has not been enabled before.
2451          */
2452         if (dev_data->dev_started && !dev_data->scattered_rx &&
2453             (frame_size + 2 * RTE_VLAN_HLEN >
2454              dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
2455                 PMD_INIT_LOG(ERR, "Stop port first.");
2456                 return -EINVAL;
2457         }
2458
2459         if (hw->mode)
2460                 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2461                         NGBE_FRAME_SIZE_MAX);
2462         else
2463                 wr32m(hw, NGBE_FRMSZ, NGBE_FRMSZ_MAX_MASK,
2464                         NGBE_FRMSZ_MAX(frame_size));
2465
2466         return 0;
2467 }
2468
2469 static uint32_t
2470 ngbe_uta_vector(struct ngbe_hw *hw, struct rte_ether_addr *uc_addr)
2471 {
2472         uint32_t vector = 0;
2473
2474         switch (hw->mac.mc_filter_type) {
2475         case 0:   /* use bits [47:36] of the address */
2476                 vector = ((uc_addr->addr_bytes[4] >> 4) |
2477                         (((uint16_t)uc_addr->addr_bytes[5]) << 4));
2478                 break;
2479         case 1:   /* use bits [46:35] of the address */
2480                 vector = ((uc_addr->addr_bytes[4] >> 3) |
2481                         (((uint16_t)uc_addr->addr_bytes[5]) << 5));
2482                 break;
2483         case 2:   /* use bits [45:34] of the address */
2484                 vector = ((uc_addr->addr_bytes[4] >> 2) |
2485                         (((uint16_t)uc_addr->addr_bytes[5]) << 6));
2486                 break;
2487         case 3:   /* use bits [43:32] of the address */
2488                 vector = ((uc_addr->addr_bytes[4]) |
2489                         (((uint16_t)uc_addr->addr_bytes[5]) << 8));
2490                 break;
2491         default:  /* Invalid mc_filter_type */
2492                 break;
2493         }
2494
2495         /* vector can only be 12-bits or boundary will be exceeded */
2496         vector &= 0xFFF;
2497         return vector;
2498 }
2499
2500 static int
2501 ngbe_uc_hash_table_set(struct rte_eth_dev *dev,
2502                         struct rte_ether_addr *mac_addr, uint8_t on)
2503 {
2504         uint32_t vector;
2505         uint32_t uta_idx;
2506         uint32_t reg_val;
2507         uint32_t uta_mask;
2508         uint32_t psrctl;
2509
2510         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2511         struct ngbe_uta_info *uta_info = NGBE_DEV_UTA_INFO(dev);
2512
2513         vector = ngbe_uta_vector(hw, mac_addr);
2514         uta_idx = (vector >> 5) & 0x7F;
2515         uta_mask = 0x1UL << (vector & 0x1F);
2516
2517         if (!!on == !!(uta_info->uta_shadow[uta_idx] & uta_mask))
2518                 return 0;
2519
2520         reg_val = rd32(hw, NGBE_UCADDRTBL(uta_idx));
2521         if (on) {
2522                 uta_info->uta_in_use++;
2523                 reg_val |= uta_mask;
2524                 uta_info->uta_shadow[uta_idx] |= uta_mask;
2525         } else {
2526                 uta_info->uta_in_use--;
2527                 reg_val &= ~uta_mask;
2528                 uta_info->uta_shadow[uta_idx] &= ~uta_mask;
2529         }
2530
2531         wr32(hw, NGBE_UCADDRTBL(uta_idx), reg_val);
2532
2533         psrctl = rd32(hw, NGBE_PSRCTL);
2534         if (uta_info->uta_in_use > 0)
2535                 psrctl |= NGBE_PSRCTL_UCHFENA;
2536         else
2537                 psrctl &= ~NGBE_PSRCTL_UCHFENA;
2538
2539         psrctl &= ~NGBE_PSRCTL_ADHF12_MASK;
2540         psrctl |= NGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
2541         wr32(hw, NGBE_PSRCTL, psrctl);
2542
2543         return 0;
2544 }
2545
2546 static int
2547 ngbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
2548 {
2549         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2550         struct ngbe_uta_info *uta_info = NGBE_DEV_UTA_INFO(dev);
2551         uint32_t psrctl;
2552         int i;
2553
2554         if (on) {
2555                 for (i = 0; i < RTE_ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2556                         uta_info->uta_shadow[i] = ~0;
2557                         wr32(hw, NGBE_UCADDRTBL(i), ~0);
2558                 }
2559         } else {
2560                 for (i = 0; i < RTE_ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
2561                         uta_info->uta_shadow[i] = 0;
2562                         wr32(hw, NGBE_UCADDRTBL(i), 0);
2563                 }
2564         }
2565
2566         psrctl = rd32(hw, NGBE_PSRCTL);
2567         if (on)
2568                 psrctl |= NGBE_PSRCTL_UCHFENA;
2569         else
2570                 psrctl &= ~NGBE_PSRCTL_UCHFENA;
2571
2572         psrctl &= ~NGBE_PSRCTL_ADHF12_MASK;
2573         psrctl |= NGBE_PSRCTL_ADHF12(hw->mac.mc_filter_type);
2574         wr32(hw, NGBE_PSRCTL, psrctl);
2575
2576         return 0;
2577 }
2578
2579 /**
2580  * Set the IVAR registers, mapping interrupt causes to vectors
2581  * @param hw
2582  *  pointer to ngbe_hw struct
2583  * @direction
2584  *  0 for Rx, 1 for Tx, -1 for other causes
2585  * @queue
2586  *  queue to map the corresponding interrupt to
2587  * @msix_vector
2588  *  the vector to map to the corresponding queue
2589  */
2590 void
2591 ngbe_set_ivar_map(struct ngbe_hw *hw, int8_t direction,
2592                    uint8_t queue, uint8_t msix_vector)
2593 {
2594         uint32_t tmp, idx;
2595
2596         if (direction == -1) {
2597                 /* other causes */
2598                 msix_vector |= NGBE_IVARMISC_VLD;
2599                 idx = 0;
2600                 tmp = rd32(hw, NGBE_IVARMISC);
2601                 tmp &= ~(0xFF << idx);
2602                 tmp |= (msix_vector << idx);
2603                 wr32(hw, NGBE_IVARMISC, tmp);
2604         } else {
2605                 /* rx or tx causes */
2606                 /* Workaround for ICR lost */
2607                 idx = ((16 * (queue & 1)) + (8 * direction));
2608                 tmp = rd32(hw, NGBE_IVAR(queue >> 1));
2609                 tmp &= ~(0xFF << idx);
2610                 tmp |= (msix_vector << idx);
2611                 wr32(hw, NGBE_IVAR(queue >> 1), tmp);
2612         }
2613 }
2614
2615 /**
2616  * Sets up the hardware to properly generate MSI-X interrupts
2617  * @hw
2618  *  board private structure
2619  */
2620 static void
2621 ngbe_configure_msix(struct rte_eth_dev *dev)
2622 {
2623         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2624         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
2625         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2626         uint32_t queue_id, base = NGBE_MISC_VEC_ID;
2627         uint32_t vec = NGBE_MISC_VEC_ID;
2628         uint32_t gpie;
2629
2630         /*
2631          * Won't configure MSI-X register if no mapping is done
2632          * between intr vector and event fd
2633          * but if MSI-X has been enabled already, need to configure
2634          * auto clean, auto mask and throttling.
2635          */
2636         gpie = rd32(hw, NGBE_GPIE);
2637         if (!rte_intr_dp_is_en(intr_handle) &&
2638             !(gpie & NGBE_GPIE_MSIX))
2639                 return;
2640
2641         if (rte_intr_allow_others(intr_handle)) {
2642                 base = NGBE_RX_VEC_START;
2643                 vec = base;
2644         }
2645
2646         /* setup GPIE for MSI-X mode */
2647         gpie = rd32(hw, NGBE_GPIE);
2648         gpie |= NGBE_GPIE_MSIX;
2649         wr32(hw, NGBE_GPIE, gpie);
2650
2651         /* Populate the IVAR table and set the ITR values to the
2652          * corresponding register.
2653          */
2654         if (rte_intr_dp_is_en(intr_handle)) {
2655                 for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
2656                         queue_id++) {
2657                         /* by default, 1:1 mapping */
2658                         ngbe_set_ivar_map(hw, 0, queue_id, vec);
2659                         rte_intr_vec_list_index_set(intr_handle,
2660                                                            queue_id, vec);
2661                         if (vec < base + rte_intr_nb_efd_get(intr_handle)
2662                             - 1)
2663                                 vec++;
2664                 }
2665
2666                 ngbe_set_ivar_map(hw, -1, 1, NGBE_MISC_VEC_ID);
2667         }
2668         wr32(hw, NGBE_ITR(NGBE_MISC_VEC_ID),
2669                         NGBE_ITR_IVAL_1G(NGBE_QUEUE_ITR_INTERVAL_DEFAULT)
2670                         | NGBE_ITR_WRDSA);
2671 }
2672
2673 static u8 *
2674 ngbe_dev_addr_list_itr(__rte_unused struct ngbe_hw *hw,
2675                         u8 **mc_addr_ptr, u32 *vmdq)
2676 {
2677         u8 *mc_addr;
2678
2679         *vmdq = 0;
2680         mc_addr = *mc_addr_ptr;
2681         *mc_addr_ptr = (mc_addr + sizeof(struct rte_ether_addr));
2682         return mc_addr;
2683 }
2684
2685 int
2686 ngbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
2687                           struct rte_ether_addr *mc_addr_set,
2688                           uint32_t nb_mc_addr)
2689 {
2690         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2691         u8 *mc_addr_list;
2692
2693         mc_addr_list = (u8 *)mc_addr_set;
2694         return hw->mac.update_mc_addr_list(hw, mc_addr_list, nb_mc_addr,
2695                                          ngbe_dev_addr_list_itr, TRUE);
2696 }
2697
2698 static uint64_t
2699 ngbe_read_systime_cyclecounter(struct rte_eth_dev *dev)
2700 {
2701         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2702         uint64_t systime_cycles;
2703
2704         systime_cycles = (uint64_t)rd32(hw, NGBE_TSTIMEL);
2705         systime_cycles |= (uint64_t)rd32(hw, NGBE_TSTIMEH) << 32;
2706
2707         return systime_cycles;
2708 }
2709
2710 static uint64_t
2711 ngbe_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
2712 {
2713         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2714         uint64_t rx_tstamp_cycles;
2715
2716         /* TSRXSTMPL stores ns and TSRXSTMPH stores seconds. */
2717         rx_tstamp_cycles = (uint64_t)rd32(hw, NGBE_TSRXSTMPL);
2718         rx_tstamp_cycles |= (uint64_t)rd32(hw, NGBE_TSRXSTMPH) << 32;
2719
2720         return rx_tstamp_cycles;
2721 }
2722
2723 static uint64_t
2724 ngbe_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
2725 {
2726         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2727         uint64_t tx_tstamp_cycles;
2728
2729         /* TSTXSTMPL stores ns and TSTXSTMPH stores seconds. */
2730         tx_tstamp_cycles = (uint64_t)rd32(hw, NGBE_TSTXSTMPL);
2731         tx_tstamp_cycles |= (uint64_t)rd32(hw, NGBE_TSTXSTMPH) << 32;
2732
2733         return tx_tstamp_cycles;
2734 }
2735
2736 static void
2737 ngbe_start_timecounters(struct rte_eth_dev *dev)
2738 {
2739         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2740         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2741         uint32_t incval = 0;
2742         uint32_t shift = 0;
2743
2744         incval = NGBE_INCVAL_1GB;
2745         shift = NGBE_INCVAL_SHIFT_1GB;
2746
2747         wr32(hw, NGBE_TSTIMEINC, NGBE_TSTIMEINC_IV(incval));
2748
2749         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
2750         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
2751         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
2752
2753         adapter->systime_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2754         adapter->systime_tc.cc_shift = shift;
2755         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
2756
2757         adapter->rx_tstamp_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2758         adapter->rx_tstamp_tc.cc_shift = shift;
2759         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
2760
2761         adapter->tx_tstamp_tc.cc_mask = NGBE_CYCLECOUNTER_MASK;
2762         adapter->tx_tstamp_tc.cc_shift = shift;
2763         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
2764 }
2765
2766 static int
2767 ngbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
2768 {
2769         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2770
2771         adapter->systime_tc.nsec += delta;
2772         adapter->rx_tstamp_tc.nsec += delta;
2773         adapter->tx_tstamp_tc.nsec += delta;
2774
2775         return 0;
2776 }
2777
2778 static int
2779 ngbe_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
2780 {
2781         uint64_t ns;
2782         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2783
2784         ns = rte_timespec_to_ns(ts);
2785         /* Set the timecounters to a new value. */
2786         adapter->systime_tc.nsec = ns;
2787         adapter->rx_tstamp_tc.nsec = ns;
2788         adapter->tx_tstamp_tc.nsec = ns;
2789
2790         return 0;
2791 }
2792
2793 static int
2794 ngbe_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
2795 {
2796         uint64_t ns, systime_cycles;
2797         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2798
2799         systime_cycles = ngbe_read_systime_cyclecounter(dev);
2800         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
2801         *ts = rte_ns_to_timespec(ns);
2802
2803         return 0;
2804 }
2805
2806 static int
2807 ngbe_timesync_enable(struct rte_eth_dev *dev)
2808 {
2809         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2810         uint32_t tsync_ctl;
2811
2812         /* Stop the timesync system time. */
2813         wr32(hw, NGBE_TSTIMEINC, 0x0);
2814         /* Reset the timesync system time value. */
2815         wr32(hw, NGBE_TSTIMEL, 0x0);
2816         wr32(hw, NGBE_TSTIMEH, 0x0);
2817
2818         ngbe_start_timecounters(dev);
2819
2820         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
2821         wr32(hw, NGBE_ETFLT(NGBE_ETF_ID_1588),
2822                 RTE_ETHER_TYPE_1588 | NGBE_ETFLT_ENA | NGBE_ETFLT_1588);
2823
2824         /* Enable timestamping of received PTP packets. */
2825         tsync_ctl = rd32(hw, NGBE_TSRXCTL);
2826         tsync_ctl |= NGBE_TSRXCTL_ENA;
2827         wr32(hw, NGBE_TSRXCTL, tsync_ctl);
2828
2829         /* Enable timestamping of transmitted PTP packets. */
2830         tsync_ctl = rd32(hw, NGBE_TSTXCTL);
2831         tsync_ctl |= NGBE_TSTXCTL_ENA;
2832         wr32(hw, NGBE_TSTXCTL, tsync_ctl);
2833
2834         ngbe_flush(hw);
2835
2836         return 0;
2837 }
2838
2839 static int
2840 ngbe_timesync_disable(struct rte_eth_dev *dev)
2841 {
2842         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2843         uint32_t tsync_ctl;
2844
2845         /* Disable timestamping of transmitted PTP packets. */
2846         tsync_ctl = rd32(hw, NGBE_TSTXCTL);
2847         tsync_ctl &= ~NGBE_TSTXCTL_ENA;
2848         wr32(hw, NGBE_TSTXCTL, tsync_ctl);
2849
2850         /* Disable timestamping of received PTP packets. */
2851         tsync_ctl = rd32(hw, NGBE_TSRXCTL);
2852         tsync_ctl &= ~NGBE_TSRXCTL_ENA;
2853         wr32(hw, NGBE_TSRXCTL, tsync_ctl);
2854
2855         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
2856         wr32(hw, NGBE_ETFLT(NGBE_ETF_ID_1588), 0);
2857
2858         /* Stop incrementing the System Time registers. */
2859         wr32(hw, NGBE_TSTIMEINC, 0);
2860
2861         return 0;
2862 }
2863
2864 static int
2865 ngbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
2866                                  struct timespec *timestamp,
2867                                  uint32_t flags __rte_unused)
2868 {
2869         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2870         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2871         uint32_t tsync_rxctl;
2872         uint64_t rx_tstamp_cycles;
2873         uint64_t ns;
2874
2875         tsync_rxctl = rd32(hw, NGBE_TSRXCTL);
2876         if ((tsync_rxctl & NGBE_TSRXCTL_VLD) == 0)
2877                 return -EINVAL;
2878
2879         rx_tstamp_cycles = ngbe_read_rx_tstamp_cyclecounter(dev);
2880         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
2881         *timestamp = rte_ns_to_timespec(ns);
2882
2883         return  0;
2884 }
2885
2886 static int
2887 ngbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
2888                                  struct timespec *timestamp)
2889 {
2890         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2891         struct ngbe_adapter *adapter = ngbe_dev_adapter(dev);
2892         uint32_t tsync_txctl;
2893         uint64_t tx_tstamp_cycles;
2894         uint64_t ns;
2895
2896         tsync_txctl = rd32(hw, NGBE_TSTXCTL);
2897         if ((tsync_txctl & NGBE_TSTXCTL_VLD) == 0)
2898                 return -EINVAL;
2899
2900         tx_tstamp_cycles = ngbe_read_tx_tstamp_cyclecounter(dev);
2901         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
2902         *timestamp = rte_ns_to_timespec(ns);
2903
2904         return 0;
2905 }
2906
2907 static int
2908 ngbe_get_reg_length(struct rte_eth_dev *dev __rte_unused)
2909 {
2910         int count = 0;
2911         int g_ind = 0;
2912         const struct reg_info *reg_group;
2913         const struct reg_info **reg_set = ngbe_regs_others;
2914
2915         while ((reg_group = reg_set[g_ind++]))
2916                 count += ngbe_regs_group_count(reg_group);
2917
2918         return count;
2919 }
2920
2921 static int
2922 ngbe_get_regs(struct rte_eth_dev *dev,
2923               struct rte_dev_reg_info *regs)
2924 {
2925         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2926         uint32_t *data = regs->data;
2927         int g_ind = 0;
2928         int count = 0;
2929         const struct reg_info *reg_group;
2930         const struct reg_info **reg_set = ngbe_regs_others;
2931
2932         if (data == NULL) {
2933                 regs->length = ngbe_get_reg_length(dev);
2934                 regs->width = sizeof(uint32_t);
2935                 return 0;
2936         }
2937
2938         /* Support only full register dump */
2939         if (regs->length == 0 ||
2940             regs->length == (uint32_t)ngbe_get_reg_length(dev)) {
2941                 regs->version = hw->mac.type << 24 |
2942                                 hw->revision_id << 16 |
2943                                 hw->device_id;
2944                 while ((reg_group = reg_set[g_ind++]))
2945                         count += ngbe_read_regs_group(dev, &data[count],
2946                                                       reg_group);
2947                 return 0;
2948         }
2949
2950         return -ENOTSUP;
2951 }
2952
2953 static int
2954 ngbe_get_eeprom_length(struct rte_eth_dev *dev)
2955 {
2956         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2957
2958         /* Return unit is byte count */
2959         return hw->rom.word_size * 2;
2960 }
2961
2962 static int
2963 ngbe_get_eeprom(struct rte_eth_dev *dev,
2964                 struct rte_dev_eeprom_info *in_eeprom)
2965 {
2966         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2967         struct ngbe_rom_info *eeprom = &hw->rom;
2968         uint16_t *data = in_eeprom->data;
2969         int first, length;
2970
2971         first = in_eeprom->offset >> 1;
2972         length = in_eeprom->length >> 1;
2973         if (first > hw->rom.word_size ||
2974             ((first + length) > hw->rom.word_size))
2975                 return -EINVAL;
2976
2977         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
2978
2979         return eeprom->readw_buffer(hw, first, length, data);
2980 }
2981
2982 static int
2983 ngbe_set_eeprom(struct rte_eth_dev *dev,
2984                 struct rte_dev_eeprom_info *in_eeprom)
2985 {
2986         struct ngbe_hw *hw = ngbe_dev_hw(dev);
2987         struct ngbe_rom_info *eeprom = &hw->rom;
2988         uint16_t *data = in_eeprom->data;
2989         int first, length;
2990
2991         first = in_eeprom->offset >> 1;
2992         length = in_eeprom->length >> 1;
2993         if (first > hw->rom.word_size ||
2994             ((first + length) > hw->rom.word_size))
2995                 return -EINVAL;
2996
2997         in_eeprom->magic = hw->vendor_id | (hw->device_id << 16);
2998
2999         return eeprom->writew_buffer(hw,  first, length, data);
3000 }
3001
3002 static const struct eth_dev_ops ngbe_eth_dev_ops = {
3003         .dev_configure              = ngbe_dev_configure,
3004         .dev_infos_get              = ngbe_dev_info_get,
3005         .dev_start                  = ngbe_dev_start,
3006         .dev_stop                   = ngbe_dev_stop,
3007         .dev_close                  = ngbe_dev_close,
3008         .dev_reset                  = ngbe_dev_reset,
3009         .promiscuous_enable         = ngbe_dev_promiscuous_enable,
3010         .promiscuous_disable        = ngbe_dev_promiscuous_disable,
3011         .allmulticast_enable        = ngbe_dev_allmulticast_enable,
3012         .allmulticast_disable       = ngbe_dev_allmulticast_disable,
3013         .link_update                = ngbe_dev_link_update,
3014         .stats_get                  = ngbe_dev_stats_get,
3015         .xstats_get                 = ngbe_dev_xstats_get,
3016         .xstats_get_by_id           = ngbe_dev_xstats_get_by_id,
3017         .stats_reset                = ngbe_dev_stats_reset,
3018         .xstats_reset               = ngbe_dev_xstats_reset,
3019         .xstats_get_names           = ngbe_dev_xstats_get_names,
3020         .xstats_get_names_by_id     = ngbe_dev_xstats_get_names_by_id,
3021         .fw_version_get             = ngbe_fw_version_get,
3022         .dev_supported_ptypes_get   = ngbe_dev_supported_ptypes_get,
3023         .mtu_set                    = ngbe_dev_mtu_set,
3024         .vlan_filter_set            = ngbe_vlan_filter_set,
3025         .vlan_tpid_set              = ngbe_vlan_tpid_set,
3026         .vlan_offload_set           = ngbe_vlan_offload_set,
3027         .vlan_strip_queue_set       = ngbe_vlan_strip_queue_set,
3028         .rx_queue_start             = ngbe_dev_rx_queue_start,
3029         .rx_queue_stop              = ngbe_dev_rx_queue_stop,
3030         .tx_queue_start             = ngbe_dev_tx_queue_start,
3031         .tx_queue_stop              = ngbe_dev_tx_queue_stop,
3032         .rx_queue_setup             = ngbe_dev_rx_queue_setup,
3033         .rx_queue_release           = ngbe_dev_rx_queue_release,
3034         .tx_queue_setup             = ngbe_dev_tx_queue_setup,
3035         .tx_queue_release           = ngbe_dev_tx_queue_release,
3036         .dev_led_on                 = ngbe_dev_led_on,
3037         .dev_led_off                = ngbe_dev_led_off,
3038         .flow_ctrl_get              = ngbe_flow_ctrl_get,
3039         .flow_ctrl_set              = ngbe_flow_ctrl_set,
3040         .mac_addr_add               = ngbe_add_rar,
3041         .mac_addr_remove            = ngbe_remove_rar,
3042         .mac_addr_set               = ngbe_set_default_mac_addr,
3043         .uc_hash_table_set          = ngbe_uc_hash_table_set,
3044         .uc_all_hash_table_set      = ngbe_uc_all_hash_table_set,
3045         .reta_update                = ngbe_dev_rss_reta_update,
3046         .reta_query                 = ngbe_dev_rss_reta_query,
3047         .rss_hash_update            = ngbe_dev_rss_hash_update,
3048         .rss_hash_conf_get          = ngbe_dev_rss_hash_conf_get,
3049         .set_mc_addr_list           = ngbe_dev_set_mc_addr_list,
3050         .rxq_info_get               = ngbe_rxq_info_get,
3051         .txq_info_get               = ngbe_txq_info_get,
3052         .rx_burst_mode_get          = ngbe_rx_burst_mode_get,
3053         .tx_burst_mode_get          = ngbe_tx_burst_mode_get,
3054         .timesync_enable            = ngbe_timesync_enable,
3055         .timesync_disable           = ngbe_timesync_disable,
3056         .timesync_read_rx_timestamp = ngbe_timesync_read_rx_timestamp,
3057         .timesync_read_tx_timestamp = ngbe_timesync_read_tx_timestamp,
3058         .get_reg                    = ngbe_get_regs,
3059         .get_eeprom_length          = ngbe_get_eeprom_length,
3060         .get_eeprom                 = ngbe_get_eeprom,
3061         .set_eeprom                 = ngbe_set_eeprom,
3062         .timesync_adjust_time       = ngbe_timesync_adjust_time,
3063         .timesync_read_time         = ngbe_timesync_read_time,
3064         .timesync_write_time        = ngbe_timesync_write_time,
3065         .tx_done_cleanup            = ngbe_dev_tx_done_cleanup,
3066 };
3067
3068 RTE_PMD_REGISTER_PCI(net_ngbe, rte_ngbe_pmd);
3069 RTE_PMD_REGISTER_PCI_TABLE(net_ngbe, pci_id_ngbe_map);
3070 RTE_PMD_REGISTER_KMOD_DEP(net_ngbe, "* igb_uio | uio_pci_generic | vfio-pci");
3071
3072 RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_init, init, NOTICE);
3073 RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_driver, driver, NOTICE);
3074
3075 #ifdef RTE_ETHDEV_DEBUG_RX
3076         RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_rx, rx, DEBUG);
3077 #endif
3078 #ifdef RTE_ETHDEV_DEBUG_TX
3079         RTE_LOG_REGISTER_SUFFIX(ngbe_logtype_tx, tx, DEBUG);
3080 #endif