drivers/net/igc/igc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7
   8 #include <rte_string_fns.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_malloc.h>
  14 #include <rte_alarm.h>
  15
  16 #include "igc_logs.h"
  17 #include "igc_txrx.h"
  18
  19 #define IGC_INTEL_VENDOR_ID             0x8086
  20
  21 /*
  22  * The overhead from MTU to max frame size.
  23  * Considering VLAN so tag needs to be counted.
  24  */
  25 #define IGC_ETH_OVERHEAD                (RTE_ETHER_HDR_LEN + \
  26                                         RTE_ETHER_CRC_LEN + VLAN_TAG_SIZE)
  27
  28 #define IGC_FC_PAUSE_TIME               0x0680
  29 #define IGC_LINK_UPDATE_CHECK_TIMEOUT   90  /* 9s */
  30 #define IGC_LINK_UPDATE_CHECK_INTERVAL  100 /* ms */
  31
  32 #define IGC_MISC_VEC_ID                 RTE_INTR_VEC_ZERO_OFFSET
  33 #define IGC_RX_VEC_START                RTE_INTR_VEC_RXTX_OFFSET
  34 #define IGC_MSIX_OTHER_INTR_VEC         0   /* MSI-X other interrupt vector */
  35 #define IGC_FLAG_NEED_LINK_UPDATE       (1u << 0)       /* need update link */
  36
  37 #define IGC_DEFAULT_RX_FREE_THRESH      32
  38
  39 #define IGC_DEFAULT_RX_PTHRESH          8
  40 #define IGC_DEFAULT_RX_HTHRESH          8
  41 #define IGC_DEFAULT_RX_WTHRESH          4
  42
  43 #define IGC_DEFAULT_TX_PTHRESH          8
  44 #define IGC_DEFAULT_TX_HTHRESH          1
  45 #define IGC_DEFAULT_TX_WTHRESH          16
  46
  47 /* MSI-X other interrupt vector */
  48 #define IGC_MSIX_OTHER_INTR_VEC         0
  49
  50 /* External VLAN Enable bit mask */
  51 #define IGC_CTRL_EXT_EXT_VLAN           (1u << 26)
  52
  53 /* Per Queue Good Packets Received Count */
  54 #define IGC_PQGPRC(idx)         (0x10010 + 0x100 * (idx))
  55 /* Per Queue Good Octets Received Count */
  56 #define IGC_PQGORC(idx)         (0x10018 + 0x100 * (idx))
  57 /* Per Queue Good Octets Transmitted Count */
  58 #define IGC_PQGOTC(idx)         (0x10034 + 0x100 * (idx))
  59 /* Per Queue Multicast Packets Received Count */
  60 #define IGC_PQMPRC(idx)         (0x10038 + 0x100 * (idx))
  61 /* Transmit Queue Drop Packet Count */
  62 #define IGC_TQDPC(idx)          (0xe030 + 0x40 * (idx))
  63
  64 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
  65 #define U32_0_IN_U64            0       /* lower bytes of u64 */
  66 #define U32_1_IN_U64            1       /* higher bytes of u64 */
  67 #else
  68 #define U32_0_IN_U64            1
  69 #define U32_1_IN_U64            0
  70 #endif
  71
  72 #define IGC_ALARM_INTERVAL      8000000u
  73 /* us, about 13.6s some per-queue registers will wrap around back to 0. */
  74
  75 static const struct rte_eth_desc_lim rx_desc_lim = {
  76         .nb_max = IGC_MAX_RXD,
  77         .nb_min = IGC_MIN_RXD,
  78         .nb_align = IGC_RXD_ALIGN,
  79 };
  80
  81 static const struct rte_eth_desc_lim tx_desc_lim = {
  82         .nb_max = IGC_MAX_TXD,
  83         .nb_min = IGC_MIN_TXD,
  84         .nb_align = IGC_TXD_ALIGN,
  85         .nb_seg_max = IGC_TX_MAX_SEG,
  86         .nb_mtu_seg_max = IGC_TX_MAX_MTU_SEG,
  87 };
  88
  89 static const struct rte_pci_id pci_id_igc_map[] = {
  90         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_LM) },
  91         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_V)  },
  92         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_I)  },
  93         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_K)  },
  94         { .vendor_id = 0, /* sentinel */ },
  95 };
  96
  97 /* store statistics names and its offset in stats structure */
  98 struct rte_igc_xstats_name_off {
  99         char name[RTE_ETH_XSTATS_NAME_SIZE];
 100         unsigned int offset;
 101 };
 102
 103 static const struct rte_igc_xstats_name_off rte_igc_stats_strings[] = {
 104         {"rx_crc_errors", offsetof(struct igc_hw_stats, crcerrs)},
 105         {"rx_align_errors", offsetof(struct igc_hw_stats, algnerrc)},
 106         {"rx_errors", offsetof(struct igc_hw_stats, rxerrc)},
 107         {"rx_missed_packets", offsetof(struct igc_hw_stats, mpc)},
 108         {"tx_single_collision_packets", offsetof(struct igc_hw_stats, scc)},
 109         {"tx_multiple_collision_packets", offsetof(struct igc_hw_stats, mcc)},
 110         {"tx_excessive_collision_packets", offsetof(struct igc_hw_stats,
 111                 ecol)},
 112         {"tx_late_collisions", offsetof(struct igc_hw_stats, latecol)},
 113         {"tx_total_collisions", offsetof(struct igc_hw_stats, colc)},
 114         {"tx_deferred_packets", offsetof(struct igc_hw_stats, dc)},
 115         {"tx_no_carrier_sense_packets", offsetof(struct igc_hw_stats, tncrs)},
 116         {"tx_discarded_packets", offsetof(struct igc_hw_stats, htdpmc)},
 117         {"rx_length_errors", offsetof(struct igc_hw_stats, rlec)},
 118         {"rx_xon_packets", offsetof(struct igc_hw_stats, xonrxc)},
 119         {"tx_xon_packets", offsetof(struct igc_hw_stats, xontxc)},
 120         {"rx_xoff_packets", offsetof(struct igc_hw_stats, xoffrxc)},
 121         {"tx_xoff_packets", offsetof(struct igc_hw_stats, xofftxc)},
 122         {"rx_flow_control_unsupported_packets", offsetof(struct igc_hw_stats,
 123                 fcruc)},
 124         {"rx_size_64_packets", offsetof(struct igc_hw_stats, prc64)},
 125         {"rx_size_65_to_127_packets", offsetof(struct igc_hw_stats, prc127)},
 126         {"rx_size_128_to_255_packets", offsetof(struct igc_hw_stats, prc255)},
 127         {"rx_size_256_to_511_packets", offsetof(struct igc_hw_stats, prc511)},
 128         {"rx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 129                 prc1023)},
 130         {"rx_size_1024_to_max_packets", offsetof(struct igc_hw_stats,
 131                 prc1522)},
 132         {"rx_broadcast_packets", offsetof(struct igc_hw_stats, bprc)},
 133         {"rx_multicast_packets", offsetof(struct igc_hw_stats, mprc)},
 134         {"rx_undersize_errors", offsetof(struct igc_hw_stats, ruc)},
 135         {"rx_fragment_errors", offsetof(struct igc_hw_stats, rfc)},
 136         {"rx_oversize_errors", offsetof(struct igc_hw_stats, roc)},
 137         {"rx_jabber_errors", offsetof(struct igc_hw_stats, rjc)},
 138         {"rx_no_buffers", offsetof(struct igc_hw_stats, rnbc)},
 139         {"rx_management_packets", offsetof(struct igc_hw_stats, mgprc)},
 140         {"rx_management_dropped", offsetof(struct igc_hw_stats, mgpdc)},
 141         {"tx_management_packets", offsetof(struct igc_hw_stats, mgptc)},
 142         {"rx_total_packets", offsetof(struct igc_hw_stats, tpr)},
 143         {"tx_total_packets", offsetof(struct igc_hw_stats, tpt)},
 144         {"rx_total_bytes", offsetof(struct igc_hw_stats, tor)},
 145         {"tx_total_bytes", offsetof(struct igc_hw_stats, tot)},
 146         {"tx_size_64_packets", offsetof(struct igc_hw_stats, ptc64)},
 147         {"tx_size_65_to_127_packets", offsetof(struct igc_hw_stats, ptc127)},
 148         {"tx_size_128_to_255_packets", offsetof(struct igc_hw_stats, ptc255)},
 149         {"tx_size_256_to_511_packets", offsetof(struct igc_hw_stats, ptc511)},
 150         {"tx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 151                 ptc1023)},
 152         {"tx_size_1023_to_max_packets", offsetof(struct igc_hw_stats,
 153                 ptc1522)},
 154         {"tx_multicast_packets", offsetof(struct igc_hw_stats, mptc)},
 155         {"tx_broadcast_packets", offsetof(struct igc_hw_stats, bptc)},
 156         {"tx_tso_packets", offsetof(struct igc_hw_stats, tsctc)},
 157         {"rx_sent_to_host_packets", offsetof(struct igc_hw_stats, rpthc)},
 158         {"tx_sent_by_host_packets", offsetof(struct igc_hw_stats, hgptc)},
 159         {"interrupt_assert_count", offsetof(struct igc_hw_stats, iac)},
 160         {"rx_descriptor_lower_threshold",
 161                 offsetof(struct igc_hw_stats, icrxdmtc)},
 162 };
 163
 164 #define IGC_NB_XSTATS (sizeof(rte_igc_stats_strings) / \
 165                 sizeof(rte_igc_stats_strings[0]))
 166
 167 static int eth_igc_configure(struct rte_eth_dev *dev);
 168 static int eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
 169 static void eth_igc_stop(struct rte_eth_dev *dev);
 170 static int eth_igc_start(struct rte_eth_dev *dev);
 171 static int eth_igc_set_link_up(struct rte_eth_dev *dev);
 172 static int eth_igc_set_link_down(struct rte_eth_dev *dev);
 173 static void eth_igc_close(struct rte_eth_dev *dev);
 174 static int eth_igc_reset(struct rte_eth_dev *dev);
 175 static int eth_igc_promiscuous_enable(struct rte_eth_dev *dev);
 176 static int eth_igc_promiscuous_disable(struct rte_eth_dev *dev);
 177 static int eth_igc_fw_version_get(struct rte_eth_dev *dev,
 178                                 char *fw_version, size_t fw_size);
 179 static int eth_igc_infos_get(struct rte_eth_dev *dev,
 180                         struct rte_eth_dev_info *dev_info);
 181 static int eth_igc_led_on(struct rte_eth_dev *dev);
 182 static int eth_igc_led_off(struct rte_eth_dev *dev);
 183 static const uint32_t *eth_igc_supported_ptypes_get(struct rte_eth_dev *dev);
 184 static int eth_igc_rar_set(struct rte_eth_dev *dev,
 185                 struct rte_ether_addr *mac_addr, uint32_t index, uint32_t pool);
 186 static void eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 187 static int eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
 188                         struct rte_ether_addr *addr);
 189 static int eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
 190                          struct rte_ether_addr *mc_addr_set,
 191                          uint32_t nb_mc_addr);
 192 static int eth_igc_allmulticast_enable(struct rte_eth_dev *dev);
 193 static int eth_igc_allmulticast_disable(struct rte_eth_dev *dev);
 194 static int eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 195 static int eth_igc_stats_get(struct rte_eth_dev *dev,
 196                         struct rte_eth_stats *rte_stats);
 197 static int eth_igc_xstats_get(struct rte_eth_dev *dev,
 198                         struct rte_eth_xstat *xstats, unsigned int n);
 199 static int eth_igc_xstats_get_by_id(struct rte_eth_dev *dev,
 200                                 const uint64_t *ids,
 201                                 uint64_t *values, unsigned int n);
 202 static int eth_igc_xstats_get_names(struct rte_eth_dev *dev,
 203                                 struct rte_eth_xstat_name *xstats_names,
 204                                 unsigned int size);
 205 static int eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 206                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
 207                 unsigned int limit);
 208 static int eth_igc_xstats_reset(struct rte_eth_dev *dev);
 209 static int
 210 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
 211         uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx);
 212
 213 static const struct eth_dev_ops eth_igc_ops = {
 214         .dev_configure          = eth_igc_configure,
 215         .link_update            = eth_igc_link_update,
 216         .dev_stop               = eth_igc_stop,
 217         .dev_start              = eth_igc_start,
 218         .dev_close              = eth_igc_close,
 219         .dev_reset              = eth_igc_reset,
 220         .dev_set_link_up        = eth_igc_set_link_up,
 221         .dev_set_link_down      = eth_igc_set_link_down,
 222         .promiscuous_enable     = eth_igc_promiscuous_enable,
 223         .promiscuous_disable    = eth_igc_promiscuous_disable,
 224         .allmulticast_enable    = eth_igc_allmulticast_enable,
 225         .allmulticast_disable   = eth_igc_allmulticast_disable,
 226         .fw_version_get         = eth_igc_fw_version_get,
 227         .dev_infos_get          = eth_igc_infos_get,
 228         .dev_led_on             = eth_igc_led_on,
 229         .dev_led_off            = eth_igc_led_off,
 230         .dev_supported_ptypes_get = eth_igc_supported_ptypes_get,
 231         .mtu_set                = eth_igc_mtu_set,
 232         .mac_addr_add           = eth_igc_rar_set,
 233         .mac_addr_remove        = eth_igc_rar_clear,
 234         .mac_addr_set           = eth_igc_default_mac_addr_set,
 235         .set_mc_addr_list       = eth_igc_set_mc_addr_list,
 236
 237         .rx_queue_setup         = eth_igc_rx_queue_setup,
 238         .rx_queue_release       = eth_igc_rx_queue_release,
 239         .rx_queue_count         = eth_igc_rx_queue_count,
 240         .rx_descriptor_done     = eth_igc_rx_descriptor_done,
 241         .rx_descriptor_status   = eth_igc_rx_descriptor_status,
 242         .tx_descriptor_status   = eth_igc_tx_descriptor_status,
 243         .tx_queue_setup         = eth_igc_tx_queue_setup,
 244         .tx_queue_release       = eth_igc_tx_queue_release,
 245         .tx_done_cleanup        = eth_igc_tx_done_cleanup,
 246         .rxq_info_get           = eth_igc_rxq_info_get,
 247         .txq_info_get           = eth_igc_txq_info_get,
 248         .stats_get              = eth_igc_stats_get,
 249         .xstats_get             = eth_igc_xstats_get,
 250         .xstats_get_by_id       = eth_igc_xstats_get_by_id,
 251         .xstats_get_names_by_id = eth_igc_xstats_get_names_by_id,
 252         .xstats_get_names       = eth_igc_xstats_get_names,
 253         .stats_reset            = eth_igc_xstats_reset,
 254         .xstats_reset           = eth_igc_xstats_reset,
 255         .queue_stats_mapping_set = eth_igc_queue_stats_mapping_set,
 256 };
 257
 258 /*
 259  * multiple queue mode checking
 260  */
 261 static int
 262 igc_check_mq_mode(struct rte_eth_dev *dev)
 263 {
 264         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 265         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
 266
 267         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 268                 PMD_INIT_LOG(ERR, "SRIOV is not supported.");
 269                 return -EINVAL;
 270         }
 271
 272         if (rx_mq_mode != ETH_MQ_RX_NONE &&
 273                 rx_mq_mode != ETH_MQ_RX_RSS) {
 274                 /* RSS together with VMDq not supported*/
 275                 PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
 276                                 rx_mq_mode);
 277                 return -EINVAL;
 278         }
 279
 280         /* To no break software that set invalid mode, only display
 281          * warning if invalid mode is used.
 282          */
 283         if (tx_mq_mode != ETH_MQ_TX_NONE)
 284                 PMD_INIT_LOG(WARNING,
 285                         "TX mode %d is not supported. Due to meaningless in this driver, just ignore",
 286                         tx_mq_mode);
 287
 288         return 0;
 289 }
 290
 291 static int
 292 eth_igc_configure(struct rte_eth_dev *dev)
 293 {
 294         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 295         int ret;
 296
 297         PMD_INIT_FUNC_TRACE();
 298
 299         ret  = igc_check_mq_mode(dev);
 300         if (ret != 0)
 301                 return ret;
 302
 303         intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 304         return 0;
 305 }
 306
 307 static int
 308 eth_igc_set_link_up(struct rte_eth_dev *dev)
 309 {
 310         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 311
 312         if (hw->phy.media_type == igc_media_type_copper)
 313                 igc_power_up_phy(hw);
 314         else
 315                 igc_power_up_fiber_serdes_link(hw);
 316         return 0;
 317 }
 318
 319 static int
 320 eth_igc_set_link_down(struct rte_eth_dev *dev)
 321 {
 322         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 323
 324         if (hw->phy.media_type == igc_media_type_copper)
 325                 igc_power_down_phy(hw);
 326         else
 327                 igc_shutdown_fiber_serdes_link(hw);
 328         return 0;
 329 }
 330
 331 /*
 332  * disable other interrupt
 333  */
 334 static void
 335 igc_intr_other_disable(struct rte_eth_dev *dev)
 336 {
 337         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 338         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 339         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 340
 341         if (rte_intr_allow_others(intr_handle) &&
 342                 dev->data->dev_conf.intr_conf.lsc) {
 343                 IGC_WRITE_REG(hw, IGC_EIMC, 1u << IGC_MSIX_OTHER_INTR_VEC);
 344         }
 345
 346         IGC_WRITE_REG(hw, IGC_IMC, ~0);
 347         IGC_WRITE_FLUSH(hw);
 348 }
 349
 350 /*
 351  * enable other interrupt
 352  */
 353 static inline void
 354 igc_intr_other_enable(struct rte_eth_dev *dev)
 355 {
 356         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 357         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 358         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 359         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 360
 361         if (rte_intr_allow_others(intr_handle) &&
 362                 dev->data->dev_conf.intr_conf.lsc) {
 363                 IGC_WRITE_REG(hw, IGC_EIMS, 1u << IGC_MSIX_OTHER_INTR_VEC);
 364         }
 365
 366         IGC_WRITE_REG(hw, IGC_IMS, intr->mask);
 367         IGC_WRITE_FLUSH(hw);
 368 }
 369
 370 /*
 371  * It reads ICR and gets interrupt causes, check it and set a bit flag
 372  * to update link status.
 373  */
 374 static void
 375 eth_igc_interrupt_get_status(struct rte_eth_dev *dev)
 376 {
 377         uint32_t icr;
 378         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 379         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 380
 381         /* read-on-clear nic registers here */
 382         icr = IGC_READ_REG(hw, IGC_ICR);
 383
 384         intr->flags = 0;
 385         if (icr & IGC_ICR_LSC)
 386                 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 387 }
 388
 389 /* return 0 means link status changed, -1 means not changed */
 390 static int
 391 eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 392 {
 393         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 394         struct rte_eth_link link;
 395         int link_check, count;
 396
 397         link_check = 0;
 398         hw->mac.get_link_status = 1;
 399
 400         /* possible wait-to-complete in up to 9 seconds */
 401         for (count = 0; count < IGC_LINK_UPDATE_CHECK_TIMEOUT; count++) {
 402                 /* Read the real link status */
 403                 switch (hw->phy.media_type) {
 404                 case igc_media_type_copper:
 405                         /* Do the work to read phy */
 406                         igc_check_for_link(hw);
 407                         link_check = !hw->mac.get_link_status;
 408                         break;
 409
 410                 case igc_media_type_fiber:
 411                         igc_check_for_link(hw);
 412                         link_check = (IGC_READ_REG(hw, IGC_STATUS) &
 413                                       IGC_STATUS_LU);
 414                         break;
 415
 416                 case igc_media_type_internal_serdes:
 417                         igc_check_for_link(hw);
 418                         link_check = hw->mac.serdes_has_link;
 419                         break;
 420
 421                 default:
 422                         break;
 423                 }
 424                 if (link_check || wait_to_complete == 0)
 425                         break;
 426                 rte_delay_ms(IGC_LINK_UPDATE_CHECK_INTERVAL);
 427         }
 428         memset(&link, 0, sizeof(link));
 429
 430         /* Now we check if a transition has happened */
 431         if (link_check) {
 432                 uint16_t duplex, speed;
 433                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
 434                 link.link_duplex = (duplex == FULL_DUPLEX) ?
 435                                 ETH_LINK_FULL_DUPLEX :
 436                                 ETH_LINK_HALF_DUPLEX;
 437                 link.link_speed = speed;
 438                 link.link_status = ETH_LINK_UP;
 439                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 440                                 ETH_LINK_SPEED_FIXED);
 441
 442                 if (speed == SPEED_2500) {
 443                         uint32_t tipg = IGC_READ_REG(hw, IGC_TIPG);
 444                         if ((tipg & IGC_TIPG_IPGT_MASK) != 0x0b) {
 445                                 tipg &= ~IGC_TIPG_IPGT_MASK;
 446                                 tipg |= 0x0b;
 447                                 IGC_WRITE_REG(hw, IGC_TIPG, tipg);
 448                         }
 449                 }
 450         } else {
 451                 link.link_speed = 0;
 452                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
 453                 link.link_status = ETH_LINK_DOWN;
 454                 link.link_autoneg = ETH_LINK_FIXED;
 455         }
 456
 457         return rte_eth_linkstatus_set(dev, &link);
 458 }
 459
 460 /*
 461  * It executes link_update after knowing an interrupt is present.
 462  */
 463 static void
 464 eth_igc_interrupt_action(struct rte_eth_dev *dev)
 465 {
 466         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 467         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 468         struct rte_eth_link link;
 469         int ret;
 470
 471         if (intr->flags & IGC_FLAG_NEED_LINK_UPDATE) {
 472                 intr->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
 473
 474                 /* set get_link_status to check register later */
 475                 ret = eth_igc_link_update(dev, 0);
 476
 477                 /* check if link has changed */
 478                 if (ret < 0)
 479                         return;
 480
 481                 rte_eth_linkstatus_get(dev, &link);
 482                 if (link.link_status)
 483                         PMD_DRV_LOG(INFO,
 484                                 " Port %d: Link Up - speed %u Mbps - %s",
 485                                 dev->data->port_id,
 486                                 (unsigned int)link.link_speed,
 487                                 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
 488                                 "full-duplex" : "half-duplex");
 489                 else
 490                         PMD_DRV_LOG(INFO, " Port %d: Link Down",
 491                                 dev->data->port_id);
 492
 493                 PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
 494                                 pci_dev->addr.domain,
 495                                 pci_dev->addr.bus,
 496                                 pci_dev->addr.devid,
 497                                 pci_dev->addr.function);
 498                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
 499                                 NULL);
 500         }
 501 }
 502
 503 /*
 504  * Interrupt handler which shall be registered at first.
 505  *
 506  * @handle
 507  *  Pointer to interrupt handle.
 508  * @param
 509  *  The address of parameter (struct rte_eth_dev *) registered before.
 510  */
 511 static void
 512 eth_igc_interrupt_handler(void *param)
 513 {
 514         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 515
 516         eth_igc_interrupt_get_status(dev);
 517         eth_igc_interrupt_action(dev);
 518 }
 519
 520 static void igc_read_queue_stats_register(struct rte_eth_dev *dev);
 521
 522 /*
 523  * Update the queue status every IGC_ALARM_INTERVAL time.
 524  * @param
 525  *  The address of parameter (struct rte_eth_dev *) registered before.
 526  */
 527 static void
 528 igc_update_queue_stats_handler(void *param)
 529 {
 530         struct rte_eth_dev *dev = param;
 531         igc_read_queue_stats_register(dev);
 532         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
 533                         igc_update_queue_stats_handler, dev);
 534 }
 535
 536 /*
 537  * rx,tx enable/disable
 538  */
 539 static void
 540 eth_igc_rxtx_control(struct rte_eth_dev *dev, bool enable)
 541 {
 542         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 543         uint32_t tctl, rctl;
 544
 545         tctl = IGC_READ_REG(hw, IGC_TCTL);
 546         rctl = IGC_READ_REG(hw, IGC_RCTL);
 547
 548         if (enable) {
 549                 /* enable Tx/Rx */
 550                 tctl |= IGC_TCTL_EN;
 551                 rctl |= IGC_RCTL_EN;
 552         } else {
 553                 /* disable Tx/Rx */
 554                 tctl &= ~IGC_TCTL_EN;
 555                 rctl &= ~IGC_RCTL_EN;
 556         }
 557         IGC_WRITE_REG(hw, IGC_TCTL, tctl);
 558         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
 559         IGC_WRITE_FLUSH(hw);
 560 }
 561
 562 /*
 563  *  This routine disables all traffic on the adapter by issuing a
 564  *  global reset on the MAC.
 565  */
 566 static void
 567 eth_igc_stop(struct rte_eth_dev *dev)
 568 {
 569         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 570         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 571         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 572         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 573         struct rte_eth_link link;
 574
 575         adapter->stopped = 1;
 576
 577         /* disable receive and transmit */
 578         eth_igc_rxtx_control(dev, false);
 579
 580         /* disable all MSI-X interrupts */
 581         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 582         IGC_WRITE_FLUSH(hw);
 583
 584         /* clear all MSI-X interrupts */
 585         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 586
 587         igc_intr_other_disable(dev);
 588
 589         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
 590
 591         /* disable intr eventfd mapping */
 592         rte_intr_disable(intr_handle);
 593
 594         igc_reset_hw(hw);
 595
 596         /* disable all wake up */
 597         IGC_WRITE_REG(hw, IGC_WUC, 0);
 598
 599         /* Set bit for Go Link disconnect */
 600         igc_read_reg_check_set_bits(hw, IGC_82580_PHY_POWER_MGMT,
 601                         IGC_82580_PM_GO_LINKD);
 602
 603         /* Power down the phy. Needed to make the link go Down */
 604         eth_igc_set_link_down(dev);
 605
 606         igc_dev_clear_queues(dev);
 607
 608         /* clear the recorded link status */
 609         memset(&link, 0, sizeof(link));
 610         rte_eth_linkstatus_set(dev, &link);
 611
 612         if (!rte_intr_allow_others(intr_handle))
 613                 /* resume to the default handler */
 614                 rte_intr_callback_register(intr_handle,
 615                                            eth_igc_interrupt_handler,
 616                                            (void *)dev);
 617
 618         /* Clean datapath event and queue/vec mapping */
 619         rte_intr_efd_disable(intr_handle);
 620 }
 621
 622 /* Sets up the hardware to generate MSI-X interrupts properly
 623  * @hw
 624  *  board private structure
 625  */
 626 static void
 627 igc_configure_msix_intr(struct rte_eth_dev *dev)
 628 {
 629         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 630         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 631         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 632
 633         uint32_t intr_mask;
 634
 635         /* won't configure msix register if no mapping is done
 636          * between intr vector and event fd
 637          */
 638         if (!rte_intr_dp_is_en(intr_handle) ||
 639                 !dev->data->dev_conf.intr_conf.lsc)
 640                 return;
 641
 642         /* turn on MSI-X capability first */
 643         IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE |
 644                                 IGC_GPIE_PBA | IGC_GPIE_EIAME |
 645                                 IGC_GPIE_NSICR);
 646
 647         intr_mask = (1u << IGC_MSIX_OTHER_INTR_VEC);
 648
 649         /* enable msix auto-clear */
 650         igc_read_reg_check_set_bits(hw, IGC_EIAC, intr_mask);
 651
 652         /* set other cause interrupt vector */
 653         igc_read_reg_check_set_bits(hw, IGC_IVAR_MISC,
 654                 (uint32_t)(IGC_MSIX_OTHER_INTR_VEC | IGC_IVAR_VALID) << 8);
 655
 656         /* enable auto-mask */
 657         igc_read_reg_check_set_bits(hw, IGC_EIAM, intr_mask);
 658
 659         IGC_WRITE_FLUSH(hw);
 660 }
 661
 662 /**
 663  * It enables the interrupt mask and then enable the interrupt.
 664  *
 665  * @dev
 666  *  Pointer to struct rte_eth_dev.
 667  * @on
 668  *  Enable or Disable
 669  */
 670 static void
 671 igc_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
 672 {
 673         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 674
 675         if (on)
 676                 intr->mask |= IGC_ICR_LSC;
 677         else
 678                 intr->mask &= ~IGC_ICR_LSC;
 679 }
 680
 681 /*
 682  *  Get hardware rx-buffer size.
 683  */
 684 static inline int
 685 igc_get_rx_buffer_size(struct igc_hw *hw)
 686 {
 687         return (IGC_READ_REG(hw, IGC_RXPBS) & 0x3f) << 10;
 688 }
 689
 690 /*
 691  * igc_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
 692  * For ASF and Pass Through versions of f/w this means
 693  * that the driver is loaded.
 694  */
 695 static void
 696 igc_hw_control_acquire(struct igc_hw *hw)
 697 {
 698         uint32_t ctrl_ext;
 699
 700         /* Let firmware know the driver has taken over */
 701         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 702         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 703 }
 704
 705 /*
 706  * igc_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
 707  * For ASF and Pass Through versions of f/w this means that the
 708  * driver is no longer loaded.
 709  */
 710 static void
 711 igc_hw_control_release(struct igc_hw *hw)
 712 {
 713         uint32_t ctrl_ext;
 714
 715         /* Let firmware taken over control of h/w */
 716         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 717         IGC_WRITE_REG(hw, IGC_CTRL_EXT,
 718                         ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 719 }
 720
 721 static int
 722 igc_hardware_init(struct igc_hw *hw)
 723 {
 724         uint32_t rx_buf_size;
 725         int diag;
 726
 727         /* Let the firmware know the OS is in control */
 728         igc_hw_control_acquire(hw);
 729
 730         /* Issue a global reset */
 731         igc_reset_hw(hw);
 732
 733         /* disable all wake up */
 734         IGC_WRITE_REG(hw, IGC_WUC, 0);
 735
 736         /*
 737          * Hardware flow control
 738          * - High water mark should allow for at least two standard size (1518)
 739          *   frames to be received after sending an XOFF.
 740          * - Low water mark works best when it is very near the high water mark.
 741          *   This allows the receiver to restart by sending XON when it has
 742          *   drained a bit. Here we use an arbitrary value of 1500 which will
 743          *   restart after one full frame is pulled from the buffer. There
 744          *   could be several smaller frames in the buffer and if so they will
 745          *   not trigger the XON until their total number reduces the buffer
 746          *   by 1500.
 747          */
 748         rx_buf_size = igc_get_rx_buffer_size(hw);
 749         hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2);
 750         hw->fc.low_water = hw->fc.high_water - 1500;
 751         hw->fc.pause_time = IGC_FC_PAUSE_TIME;
 752         hw->fc.send_xon = 1;
 753         hw->fc.requested_mode = igc_fc_full;
 754
 755         diag = igc_init_hw(hw);
 756         if (diag < 0)
 757                 return diag;
 758
 759         igc_get_phy_info(hw);
 760         igc_check_for_link(hw);
 761
 762         return 0;
 763 }
 764
 765 static int
 766 eth_igc_start(struct rte_eth_dev *dev)
 767 {
 768         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 769         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 770         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 771         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 772         uint32_t *speeds;
 773         int ret;
 774
 775         PMD_INIT_FUNC_TRACE();
 776
 777         /* disable all MSI-X interrupts */
 778         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 779         IGC_WRITE_FLUSH(hw);
 780
 781         /* clear all MSI-X interrupts */
 782         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 783
 784         /* disable uio/vfio intr/eventfd mapping */
 785         if (!adapter->stopped)
 786                 rte_intr_disable(intr_handle);
 787
 788         /* Power up the phy. Needed to make the link go Up */
 789         eth_igc_set_link_up(dev);
 790
 791         /* Put the address into the Receive Address Array */
 792         igc_rar_set(hw, hw->mac.addr, 0);
 793
 794         /* Initialize the hardware */
 795         if (igc_hardware_init(hw)) {
 796                 PMD_DRV_LOG(ERR, "Unable to initialize the hardware");
 797                 return -EIO;
 798         }
 799         adapter->stopped = 0;
 800
 801         /* confiugre msix for rx interrupt */
 802         igc_configure_msix_intr(dev);
 803
 804         igc_tx_init(dev);
 805
 806         /* This can fail when allocating mbufs for descriptor rings */
 807         ret = igc_rx_init(dev);
 808         if (ret) {
 809                 PMD_DRV_LOG(ERR, "Unable to initialize RX hardware");
 810                 igc_dev_clear_queues(dev);
 811                 return ret;
 812         }
 813
 814         igc_clear_hw_cntrs_base_generic(hw);
 815
 816         /* Setup link speed and duplex */
 817         speeds = &dev->data->dev_conf.link_speeds;
 818         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
 819                 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
 820                 hw->mac.autoneg = 1;
 821         } else {
 822                 int num_speeds = 0;
 823                 bool autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
 824
 825                 /* Reset */
 826                 hw->phy.autoneg_advertised = 0;
 827
 828                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
 829                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
 830                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
 831                                 ETH_LINK_SPEED_FIXED)) {
 832                         num_speeds = -1;
 833                         goto error_invalid_config;
 834                 }
 835                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
 836                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
 837                         num_speeds++;
 838                 }
 839                 if (*speeds & ETH_LINK_SPEED_10M) {
 840                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
 841                         num_speeds++;
 842                 }
 843                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
 844                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
 845                         num_speeds++;
 846                 }
 847                 if (*speeds & ETH_LINK_SPEED_100M) {
 848                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
 849                         num_speeds++;
 850                 }
 851                 if (*speeds & ETH_LINK_SPEED_1G) {
 852                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
 853                         num_speeds++;
 854                 }
 855                 if (*speeds & ETH_LINK_SPEED_2_5G) {
 856                         hw->phy.autoneg_advertised |= ADVERTISE_2500_FULL;
 857                         num_speeds++;
 858                 }
 859                 if (num_speeds == 0 || (!autoneg && num_speeds > 1))
 860                         goto error_invalid_config;
 861
 862                 /* Set/reset the mac.autoneg based on the link speed,
 863                  * fixed or not
 864                  */
 865                 if (!autoneg) {
 866                         hw->mac.autoneg = 0;
 867                         hw->mac.forced_speed_duplex =
 868                                         hw->phy.autoneg_advertised;
 869                 } else {
 870                         hw->mac.autoneg = 1;
 871                 }
 872         }
 873
 874         igc_setup_link(hw);
 875
 876         if (rte_intr_allow_others(intr_handle)) {
 877                 /* check if lsc interrupt is enabled */
 878                 if (dev->data->dev_conf.intr_conf.lsc)
 879                         igc_lsc_interrupt_setup(dev, 1);
 880                 else
 881                         igc_lsc_interrupt_setup(dev, 0);
 882         } else {
 883                 rte_intr_callback_unregister(intr_handle,
 884                                              eth_igc_interrupt_handler,
 885                                              (void *)dev);
 886                 if (dev->data->dev_conf.intr_conf.lsc)
 887                         PMD_DRV_LOG(INFO,
 888                                 "LSC won't enable because of no intr multiplex");
 889         }
 890
 891         /* enable uio/vfio intr/eventfd mapping */
 892         rte_intr_enable(intr_handle);
 893
 894         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
 895                         igc_update_queue_stats_handler, dev);
 896
 897         /* resume enabled intr since hw reset */
 898         igc_intr_other_enable(dev);
 899
 900         eth_igc_rxtx_control(dev, true);
 901         eth_igc_link_update(dev, 0);
 902
 903         return 0;
 904
 905 error_invalid_config:
 906         PMD_DRV_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
 907                      dev->data->dev_conf.link_speeds, dev->data->port_id);
 908         igc_dev_clear_queues(dev);
 909         return -EINVAL;
 910 }
 911
 912 static int
 913 igc_reset_swfw_lock(struct igc_hw *hw)
 914 {
 915         int ret_val;
 916
 917         /*
 918          * Do mac ops initialization manually here, since we will need
 919          * some function pointers set by this call.
 920          */
 921         ret_val = igc_init_mac_params(hw);
 922         if (ret_val)
 923                 return ret_val;
 924
 925         /*
 926          * SMBI lock should not fail in this early stage. If this is the case,
 927          * it is due to an improper exit of the application.
 928          * So force the release of the faulty lock.
 929          */
 930         if (igc_get_hw_semaphore_generic(hw) < 0)
 931                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 932
 933         igc_put_hw_semaphore_generic(hw);
 934
 935         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 936                 uint16_t mask;
 937
 938                 /*
 939                  * Phy lock should not fail in this early stage.
 940                  * If this is the case, it is due to an improper exit of the
 941                  * application. So force the release of the faulty lock.
 942                  */
 943                 mask = IGC_SWFW_PHY0_SM;
 944                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 945                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 946                                     hw->bus.func);
 947                 }
 948                 hw->mac.ops.release_swfw_sync(hw, mask);
 949
 950                 /*
 951                  * This one is more tricky since it is common to all ports; but
 952                  * swfw_sync retries last long enough (1s) to be almost sure
 953                  * that if lock can not be taken it is due to an improper lock
 954                  * of the semaphore.
 955                  */
 956                 mask = IGC_SWFW_EEP_SM;
 957                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0)
 958                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 959
 960                 hw->mac.ops.release_swfw_sync(hw, mask);
 961         }
 962
 963         return IGC_SUCCESS;
 964 }
 965
 966 /*
 967  * free all rx/tx queues.
 968  */
 969 static void
 970 igc_dev_free_queues(struct rte_eth_dev *dev)
 971 {
 972         uint16_t i;
 973
 974         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 975                 eth_igc_rx_queue_release(dev->data->rx_queues[i]);
 976                 dev->data->rx_queues[i] = NULL;
 977         }
 978         dev->data->nb_rx_queues = 0;
 979
 980         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 981                 eth_igc_tx_queue_release(dev->data->tx_queues[i]);
 982                 dev->data->tx_queues[i] = NULL;
 983         }
 984         dev->data->nb_tx_queues = 0;
 985 }
 986
 987 static void
 988 eth_igc_close(struct rte_eth_dev *dev)
 989 {
 990         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 991         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 992         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 993         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 994         int retry = 0;
 995
 996         PMD_INIT_FUNC_TRACE();
 997
 998         if (!adapter->stopped)
 999                 eth_igc_stop(dev);
1000
1001         igc_intr_other_disable(dev);
1002         do {
1003                 int ret = rte_intr_callback_unregister(intr_handle,
1004                                 eth_igc_interrupt_handler, dev);
1005                 if (ret >= 0 || ret == -ENOENT || ret == -EINVAL)
1006                         break;
1007
1008                 PMD_DRV_LOG(ERR, "intr callback unregister failed: %d", ret);
1009                 DELAY(200 * 1000); /* delay 200ms */
1010         } while (retry++ < 5);
1011
1012         igc_phy_hw_reset(hw);
1013         igc_hw_control_release(hw);
1014         igc_dev_free_queues(dev);
1015
1016         /* Reset any pending lock */
1017         igc_reset_swfw_lock(hw);
1018 }
1019
1020 static void
1021 igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
1022 {
1023         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1024
1025         hw->vendor_id = pci_dev->id.vendor_id;
1026         hw->device_id = pci_dev->id.device_id;
1027         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1028         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1029 }
1030
1031 static int
1032 eth_igc_dev_init(struct rte_eth_dev *dev)
1033 {
1034         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1035         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1036         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1037         int i, error = 0;
1038
1039         PMD_INIT_FUNC_TRACE();
1040         dev->dev_ops = &eth_igc_ops;
1041
1042         /*
1043          * for secondary processes, we don't initialize any further as primary
1044          * has already done this work. Only check we don't need a different
1045          * RX function.
1046          */
1047         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1048                 return 0;
1049
1050         rte_eth_copy_pci_info(dev, pci_dev);
1051
1052         hw->back = pci_dev;
1053         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1054
1055         igc_identify_hardware(dev, pci_dev);
1056         if (igc_setup_init_funcs(hw, false) != IGC_SUCCESS) {
1057                 error = -EIO;
1058                 goto err_late;
1059         }
1060
1061         igc_get_bus_info(hw);
1062
1063         /* Reset any pending lock */
1064         if (igc_reset_swfw_lock(hw) != IGC_SUCCESS) {
1065                 error = -EIO;
1066                 goto err_late;
1067         }
1068
1069         /* Finish initialization */
1070         if (igc_setup_init_funcs(hw, true) != IGC_SUCCESS) {
1071                 error = -EIO;
1072                 goto err_late;
1073         }
1074
1075         hw->mac.autoneg = 1;
1076         hw->phy.autoneg_wait_to_complete = 0;
1077         hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
1078
1079         /* Copper options */
1080         if (hw->phy.media_type == igc_media_type_copper) {
1081                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1082                 hw->phy.disable_polarity_correction = 0;
1083                 hw->phy.ms_type = igc_ms_hw_default;
1084         }
1085
1086         /*
1087          * Start from a known state, this is important in reading the nvm
1088          * and mac from that.
1089          */
1090         igc_reset_hw(hw);
1091
1092         /* Make sure we have a good EEPROM before we read from it */
1093         if (igc_validate_nvm_checksum(hw) < 0) {
1094                 /*
1095                  * Some PCI-E parts fail the first check due to
1096                  * the link being in sleep state, call it again,
1097                  * if it fails a second time its a real issue.
1098                  */
1099                 if (igc_validate_nvm_checksum(hw) < 0) {
1100                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
1101                         error = -EIO;
1102                         goto err_late;
1103                 }
1104         }
1105
1106         /* Read the permanent MAC address out of the EEPROM */
1107         if (igc_read_mac_addr(hw) != 0) {
1108                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
1109                 error = -EIO;
1110                 goto err_late;
1111         }
1112
1113         /* Allocate memory for storing MAC addresses */
1114         dev->data->mac_addrs = rte_zmalloc("igc",
1115                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
1116         if (dev->data->mac_addrs == NULL) {
1117                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes for storing MAC",
1118                                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1119                 error = -ENOMEM;
1120                 goto err_late;
1121         }
1122
1123         /* Copy the permanent MAC address */
1124         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
1125                         &dev->data->mac_addrs[0]);
1126
1127         /* Now initialize the hardware */
1128         if (igc_hardware_init(hw) != 0) {
1129                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
1130                 rte_free(dev->data->mac_addrs);
1131                 dev->data->mac_addrs = NULL;
1132                 error = -ENODEV;
1133                 goto err_late;
1134         }
1135
1136         /* Pass the information to the rte_eth_dev_close() that it should also
1137          * release the private port resources.
1138          */
1139         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1140
1141         hw->mac.get_link_status = 1;
1142         igc->stopped = 0;
1143
1144         /* Indicate SOL/IDER usage */
1145         if (igc_check_reset_block(hw) < 0)
1146                 PMD_INIT_LOG(ERR,
1147                         "PHY reset is blocked due to SOL/IDER session.");
1148
1149         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
1150                         dev->data->port_id, pci_dev->id.vendor_id,
1151                         pci_dev->id.device_id);
1152
1153         rte_intr_callback_register(&pci_dev->intr_handle,
1154                         eth_igc_interrupt_handler, (void *)dev);
1155
1156         /* enable uio/vfio intr/eventfd mapping */
1157         rte_intr_enable(&pci_dev->intr_handle);
1158
1159         /* enable support intr */
1160         igc_intr_other_enable(dev);
1161
1162         /* initiate queue status */
1163         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1164                 igc->txq_stats_map[i] = -1;
1165                 igc->rxq_stats_map[i] = -1;
1166         }
1167         return 0;
1168
1169 err_late:
1170         igc_hw_control_release(hw);
1171         return error;
1172 }
1173
1174 static int
1175 eth_igc_dev_uninit(__rte_unused struct rte_eth_dev *eth_dev)
1176 {
1177         PMD_INIT_FUNC_TRACE();
1178
1179         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1180                 return 0;
1181
1182         eth_igc_close(eth_dev);
1183         return 0;
1184 }
1185
1186 static int
1187 eth_igc_reset(struct rte_eth_dev *dev)
1188 {
1189         int ret;
1190
1191         PMD_INIT_FUNC_TRACE();
1192
1193         ret = eth_igc_dev_uninit(dev);
1194         if (ret)
1195                 return ret;
1196
1197         return eth_igc_dev_init(dev);
1198 }
1199
1200 static int
1201 eth_igc_promiscuous_enable(struct rte_eth_dev *dev)
1202 {
1203         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1204         uint32_t rctl;
1205
1206         rctl = IGC_READ_REG(hw, IGC_RCTL);
1207         rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1208         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1209         return 0;
1210 }
1211
1212 static int
1213 eth_igc_promiscuous_disable(struct rte_eth_dev *dev)
1214 {
1215         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1216         uint32_t rctl;
1217
1218         rctl = IGC_READ_REG(hw, IGC_RCTL);
1219         rctl &= (~IGC_RCTL_UPE);
1220         if (dev->data->all_multicast == 1)
1221                 rctl |= IGC_RCTL_MPE;
1222         else
1223                 rctl &= (~IGC_RCTL_MPE);
1224         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1225         return 0;
1226 }
1227
1228 static int
1229 eth_igc_allmulticast_enable(struct rte_eth_dev *dev)
1230 {
1231         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1232         uint32_t rctl;
1233
1234         rctl = IGC_READ_REG(hw, IGC_RCTL);
1235         rctl |= IGC_RCTL_MPE;
1236         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1237         return 0;
1238 }
1239
1240 static int
1241 eth_igc_allmulticast_disable(struct rte_eth_dev *dev)
1242 {
1243         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1244         uint32_t rctl;
1245
1246         if (dev->data->promiscuous == 1)
1247                 return 0;       /* must remain in all_multicast mode */
1248
1249         rctl = IGC_READ_REG(hw, IGC_RCTL);
1250         rctl &= (~IGC_RCTL_MPE);
1251         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1252         return 0;
1253 }
1254
1255 static int
1256 eth_igc_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
1257                        size_t fw_size)
1258 {
1259         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1260         struct igc_fw_version fw;
1261         int ret;
1262
1263         igc_get_fw_version(hw, &fw);
1264
1265         /* if option rom is valid, display its version too */
1266         if (fw.or_valid) {
1267                 ret = snprintf(fw_version, fw_size,
1268                          "%d.%d, 0x%08x, %d.%d.%d",
1269                          fw.eep_major, fw.eep_minor, fw.etrack_id,
1270                          fw.or_major, fw.or_build, fw.or_patch);
1271         /* no option rom */
1272         } else {
1273                 if (fw.etrack_id != 0X0000) {
1274                         ret = snprintf(fw_version, fw_size,
1275                                  "%d.%d, 0x%08x",
1276                                  fw.eep_major, fw.eep_minor,
1277                                  fw.etrack_id);
1278                 } else {
1279                         ret = snprintf(fw_version, fw_size,
1280                                  "%d.%d.%d",
1281                                  fw.eep_major, fw.eep_minor,
1282                                  fw.eep_build);
1283                 }
1284         }
1285
1286         ret += 1; /* add the size of '\0' */
1287         if (fw_size < (u32)ret)
1288                 return ret;
1289         else
1290                 return 0;
1291 }
1292
1293 static int
1294 eth_igc_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1295 {
1296         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1297
1298         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1299         dev_info->max_rx_pktlen = MAX_RX_JUMBO_FRAME_SIZE;
1300         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1301         dev_info->rx_offload_capa = IGC_RX_OFFLOAD_ALL;
1302         dev_info->tx_offload_capa = IGC_TX_OFFLOAD_ALL;
1303
1304         dev_info->max_rx_queues = IGC_QUEUE_PAIRS_NUM;
1305         dev_info->max_tx_queues = IGC_QUEUE_PAIRS_NUM;
1306         dev_info->max_vmdq_pools = 0;
1307
1308         dev_info->hash_key_size = IGC_HKEY_MAX_INDEX * sizeof(uint32_t);
1309         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1310         dev_info->flow_type_rss_offloads = IGC_RSS_OFFLOAD_ALL;
1311
1312         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1313                 .rx_thresh = {
1314                         .pthresh = IGC_DEFAULT_RX_PTHRESH,
1315                         .hthresh = IGC_DEFAULT_RX_HTHRESH,
1316                         .wthresh = IGC_DEFAULT_RX_WTHRESH,
1317                 },
1318                 .rx_free_thresh = IGC_DEFAULT_RX_FREE_THRESH,
1319                 .rx_drop_en = 0,
1320                 .offloads = 0,
1321         };
1322
1323         dev_info->default_txconf = (struct rte_eth_txconf) {
1324                 .tx_thresh = {
1325                         .pthresh = IGC_DEFAULT_TX_PTHRESH,
1326                         .hthresh = IGC_DEFAULT_TX_HTHRESH,
1327                         .wthresh = IGC_DEFAULT_TX_WTHRESH,
1328                 },
1329                 .offloads = 0,
1330         };
1331
1332         dev_info->rx_desc_lim = rx_desc_lim;
1333         dev_info->tx_desc_lim = tx_desc_lim;
1334
1335         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1336                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1337                         ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G;
1338
1339         dev_info->max_mtu = dev_info->max_rx_pktlen - IGC_ETH_OVERHEAD;
1340         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
1341         return 0;
1342 }
1343
1344 static int
1345 eth_igc_led_on(struct rte_eth_dev *dev)
1346 {
1347         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1348
1349         return igc_led_on(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1350 }
1351
1352 static int
1353 eth_igc_led_off(struct rte_eth_dev *dev)
1354 {
1355         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1356
1357         return igc_led_off(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1358 }
1359
1360 static const uint32_t *
1361 eth_igc_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev)
1362 {
1363         static const uint32_t ptypes[] = {
1364                 /* refers to rx_desc_pkt_info_to_pkt_type() */
1365                 RTE_PTYPE_L2_ETHER,
1366                 RTE_PTYPE_L3_IPV4,
1367                 RTE_PTYPE_L3_IPV4_EXT,
1368                 RTE_PTYPE_L3_IPV6,
1369                 RTE_PTYPE_L3_IPV6_EXT,
1370                 RTE_PTYPE_L4_TCP,
1371                 RTE_PTYPE_L4_UDP,
1372                 RTE_PTYPE_L4_SCTP,
1373                 RTE_PTYPE_TUNNEL_IP,
1374                 RTE_PTYPE_INNER_L3_IPV6,
1375                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1376                 RTE_PTYPE_INNER_L4_TCP,
1377                 RTE_PTYPE_INNER_L4_UDP,
1378                 RTE_PTYPE_UNKNOWN
1379         };
1380
1381         return ptypes;
1382 }
1383
1384 static int
1385 eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1386 {
1387         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1388         uint32_t frame_size = mtu + IGC_ETH_OVERHEAD;
1389         uint32_t rctl;
1390
1391         /* if extend vlan has been enabled */
1392         if (IGC_READ_REG(hw, IGC_CTRL_EXT) & IGC_CTRL_EXT_EXT_VLAN)
1393                 frame_size += VLAN_TAG_SIZE;
1394
1395         /* check that mtu is within the allowed range */
1396         if (mtu < RTE_ETHER_MIN_MTU ||
1397                 frame_size > MAX_RX_JUMBO_FRAME_SIZE)
1398                 return -EINVAL;
1399
1400         /*
1401          * refuse mtu that requires the support of scattered packets when
1402          * this feature has not been enabled before.
1403          */
1404         if (!dev->data->scattered_rx &&
1405             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
1406                 return -EINVAL;
1407
1408         rctl = IGC_READ_REG(hw, IGC_RCTL);
1409
1410         /* switch to jumbo mode if needed */
1411         if (mtu > RTE_ETHER_MTU) {
1412                 dev->data->dev_conf.rxmode.offloads |=
1413                         DEV_RX_OFFLOAD_JUMBO_FRAME;
1414                 rctl |= IGC_RCTL_LPE;
1415         } else {
1416                 dev->data->dev_conf.rxmode.offloads &=
1417                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1418                 rctl &= ~IGC_RCTL_LPE;
1419         }
1420         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1421
1422         /* update max frame size */
1423         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1424
1425         IGC_WRITE_REG(hw, IGC_RLPML,
1426                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1427
1428         return 0;
1429 }
1430
1431 static int
1432 eth_igc_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1433                 uint32_t index, uint32_t pool)
1434 {
1435         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1436
1437         igc_rar_set(hw, mac_addr->addr_bytes, index);
1438         RTE_SET_USED(pool);
1439         return 0;
1440 }
1441
1442 static void
1443 eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1444 {
1445         uint8_t addr[RTE_ETHER_ADDR_LEN];
1446         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1447
1448         memset(addr, 0, sizeof(addr));
1449         igc_rar_set(hw, addr, index);
1450 }
1451
1452 static int
1453 eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
1454                         struct rte_ether_addr *addr)
1455 {
1456         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1457         igc_rar_set(hw, addr->addr_bytes, 0);
1458         return 0;
1459 }
1460
1461 static int
1462 eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
1463                          struct rte_ether_addr *mc_addr_set,
1464                          uint32_t nb_mc_addr)
1465 {
1466         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1467         igc_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
1468         return 0;
1469 }
1470
1471 /*
1472  * Read hardware registers
1473  */
1474 static void
1475 igc_read_stats_registers(struct igc_hw *hw, struct igc_hw_stats *stats)
1476 {
1477         int pause_frames;
1478
1479         uint64_t old_gprc  = stats->gprc;
1480         uint64_t old_gptc  = stats->gptc;
1481         uint64_t old_tpr   = stats->tpr;
1482         uint64_t old_tpt   = stats->tpt;
1483         uint64_t old_rpthc = stats->rpthc;
1484         uint64_t old_hgptc = stats->hgptc;
1485
1486         stats->crcerrs += IGC_READ_REG(hw, IGC_CRCERRS);
1487         stats->algnerrc += IGC_READ_REG(hw, IGC_ALGNERRC);
1488         stats->rxerrc += IGC_READ_REG(hw, IGC_RXERRC);
1489         stats->mpc += IGC_READ_REG(hw, IGC_MPC);
1490         stats->scc += IGC_READ_REG(hw, IGC_SCC);
1491         stats->ecol += IGC_READ_REG(hw, IGC_ECOL);
1492
1493         stats->mcc += IGC_READ_REG(hw, IGC_MCC);
1494         stats->latecol += IGC_READ_REG(hw, IGC_LATECOL);
1495         stats->colc += IGC_READ_REG(hw, IGC_COLC);
1496
1497         stats->dc += IGC_READ_REG(hw, IGC_DC);
1498         stats->tncrs += IGC_READ_REG(hw, IGC_TNCRS);
1499         stats->htdpmc += IGC_READ_REG(hw, IGC_HTDPMC);
1500         stats->rlec += IGC_READ_REG(hw, IGC_RLEC);
1501         stats->xonrxc += IGC_READ_REG(hw, IGC_XONRXC);
1502         stats->xontxc += IGC_READ_REG(hw, IGC_XONTXC);
1503
1504         /*
1505          * For watchdog management we need to know if we have been
1506          * paused during the last interval, so capture that here.
1507          */
1508         pause_frames = IGC_READ_REG(hw, IGC_XOFFRXC);
1509         stats->xoffrxc += pause_frames;
1510         stats->xofftxc += IGC_READ_REG(hw, IGC_XOFFTXC);
1511         stats->fcruc += IGC_READ_REG(hw, IGC_FCRUC);
1512         stats->prc64 += IGC_READ_REG(hw, IGC_PRC64);
1513         stats->prc127 += IGC_READ_REG(hw, IGC_PRC127);
1514         stats->prc255 += IGC_READ_REG(hw, IGC_PRC255);
1515         stats->prc511 += IGC_READ_REG(hw, IGC_PRC511);
1516         stats->prc1023 += IGC_READ_REG(hw, IGC_PRC1023);
1517         stats->prc1522 += IGC_READ_REG(hw, IGC_PRC1522);
1518         stats->gprc += IGC_READ_REG(hw, IGC_GPRC);
1519         stats->bprc += IGC_READ_REG(hw, IGC_BPRC);
1520         stats->mprc += IGC_READ_REG(hw, IGC_MPRC);
1521         stats->gptc += IGC_READ_REG(hw, IGC_GPTC);
1522
1523         /* For the 64-bit byte counters the low dword must be read first. */
1524         /* Both registers clear on the read of the high dword */
1525
1526         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1527         stats->gorc += IGC_READ_REG(hw, IGC_GORCL);
1528         stats->gorc += ((uint64_t)IGC_READ_REG(hw, IGC_GORCH) << 32);
1529         stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN;
1530         stats->gotc += IGC_READ_REG(hw, IGC_GOTCL);
1531         stats->gotc += ((uint64_t)IGC_READ_REG(hw, IGC_GOTCH) << 32);
1532         stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN;
1533
1534         stats->rnbc += IGC_READ_REG(hw, IGC_RNBC);
1535         stats->ruc += IGC_READ_REG(hw, IGC_RUC);
1536         stats->rfc += IGC_READ_REG(hw, IGC_RFC);
1537         stats->roc += IGC_READ_REG(hw, IGC_ROC);
1538         stats->rjc += IGC_READ_REG(hw, IGC_RJC);
1539
1540         stats->mgprc += IGC_READ_REG(hw, IGC_MGTPRC);
1541         stats->mgpdc += IGC_READ_REG(hw, IGC_MGTPDC);
1542         stats->mgptc += IGC_READ_REG(hw, IGC_MGTPTC);
1543         stats->b2ospc += IGC_READ_REG(hw, IGC_B2OSPC);
1544         stats->b2ogprc += IGC_READ_REG(hw, IGC_B2OGPRC);
1545         stats->o2bgptc += IGC_READ_REG(hw, IGC_O2BGPTC);
1546         stats->o2bspc += IGC_READ_REG(hw, IGC_O2BSPC);
1547
1548         stats->tpr += IGC_READ_REG(hw, IGC_TPR);
1549         stats->tpt += IGC_READ_REG(hw, IGC_TPT);
1550
1551         stats->tor += IGC_READ_REG(hw, IGC_TORL);
1552         stats->tor += ((uint64_t)IGC_READ_REG(hw, IGC_TORH) << 32);
1553         stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN;
1554         stats->tot += IGC_READ_REG(hw, IGC_TOTL);
1555         stats->tot += ((uint64_t)IGC_READ_REG(hw, IGC_TOTH) << 32);
1556         stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN;
1557
1558         stats->ptc64 += IGC_READ_REG(hw, IGC_PTC64);
1559         stats->ptc127 += IGC_READ_REG(hw, IGC_PTC127);
1560         stats->ptc255 += IGC_READ_REG(hw, IGC_PTC255);
1561         stats->ptc511 += IGC_READ_REG(hw, IGC_PTC511);
1562         stats->ptc1023 += IGC_READ_REG(hw, IGC_PTC1023);
1563         stats->ptc1522 += IGC_READ_REG(hw, IGC_PTC1522);
1564         stats->mptc += IGC_READ_REG(hw, IGC_MPTC);
1565         stats->bptc += IGC_READ_REG(hw, IGC_BPTC);
1566         stats->tsctc += IGC_READ_REG(hw, IGC_TSCTC);
1567
1568         stats->iac += IGC_READ_REG(hw, IGC_IAC);
1569         stats->rpthc += IGC_READ_REG(hw, IGC_RPTHC);
1570         stats->hgptc += IGC_READ_REG(hw, IGC_HGPTC);
1571         stats->icrxdmtc += IGC_READ_REG(hw, IGC_ICRXDMTC);
1572
1573         /* Host to Card Statistics */
1574         stats->hgorc += IGC_READ_REG(hw, IGC_HGORCL);
1575         stats->hgorc += ((uint64_t)IGC_READ_REG(hw, IGC_HGORCH) << 32);
1576         stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN;
1577         stats->hgotc += IGC_READ_REG(hw, IGC_HGOTCL);
1578         stats->hgotc += ((uint64_t)IGC_READ_REG(hw, IGC_HGOTCH) << 32);
1579         stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN;
1580         stats->lenerrs += IGC_READ_REG(hw, IGC_LENERRS);
1581 }
1582
1583 /*
1584  * Write 0 to all queue status registers
1585  */
1586 static void
1587 igc_reset_queue_stats_register(struct igc_hw *hw)
1588 {
1589         int i;
1590
1591         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1592                 IGC_WRITE_REG(hw, IGC_PQGPRC(i), 0);
1593                 IGC_WRITE_REG(hw, IGC_PQGPTC(i), 0);
1594                 IGC_WRITE_REG(hw, IGC_PQGORC(i), 0);
1595                 IGC_WRITE_REG(hw, IGC_PQGOTC(i), 0);
1596                 IGC_WRITE_REG(hw, IGC_PQMPRC(i), 0);
1597                 IGC_WRITE_REG(hw, IGC_RQDPC(i), 0);
1598                 IGC_WRITE_REG(hw, IGC_TQDPC(i), 0);
1599         }
1600 }
1601
1602 /*
1603  * Read all hardware queue status registers
1604  */
1605 static void
1606 igc_read_queue_stats_register(struct rte_eth_dev *dev)
1607 {
1608         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1609         struct igc_hw_queue_stats *queue_stats =
1610                                 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1611         int i;
1612
1613         /*
1614          * This register is not cleared on read. Furthermore, the register wraps
1615          * around back to 0x00000000 on the next increment when reaching a value
1616          * of 0xFFFFFFFF and then continues normal count operation.
1617          */
1618         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1619                 union {
1620                         u64 ddword;
1621                         u32 dword[2];
1622                 } value;
1623                 u32 tmp;
1624
1625                 /*
1626                  * Read the register first, if the value is smaller than that
1627                  * previous read, that mean the register has been overflowed,
1628                  * then we add the high 4 bytes by 1 and replace the low 4
1629                  * bytes by the new value.
1630                  */
1631                 tmp = IGC_READ_REG(hw, IGC_PQGPRC(i));
1632                 value.ddword = queue_stats->pqgprc[i];
1633                 if (value.dword[U32_0_IN_U64] > tmp)
1634                         value.dword[U32_1_IN_U64]++;
1635                 value.dword[U32_0_IN_U64] = tmp;
1636                 queue_stats->pqgprc[i] = value.ddword;
1637
1638                 tmp = IGC_READ_REG(hw, IGC_PQGPTC(i));
1639                 value.ddword = queue_stats->pqgptc[i];
1640                 if (value.dword[U32_0_IN_U64] > tmp)
1641                         value.dword[U32_1_IN_U64]++;
1642                 value.dword[U32_0_IN_U64] = tmp;
1643                 queue_stats->pqgptc[i] = value.ddword;
1644
1645                 tmp = IGC_READ_REG(hw, IGC_PQGORC(i));
1646                 value.ddword = queue_stats->pqgorc[i];
1647                 if (value.dword[U32_0_IN_U64] > tmp)
1648                         value.dword[U32_1_IN_U64]++;
1649                 value.dword[U32_0_IN_U64] = tmp;
1650                 queue_stats->pqgorc[i] = value.ddword;
1651
1652                 tmp = IGC_READ_REG(hw, IGC_PQGOTC(i));
1653                 value.ddword = queue_stats->pqgotc[i];
1654                 if (value.dword[U32_0_IN_U64] > tmp)
1655                         value.dword[U32_1_IN_U64]++;
1656                 value.dword[U32_0_IN_U64] = tmp;
1657                 queue_stats->pqgotc[i] = value.ddword;
1658
1659                 tmp = IGC_READ_REG(hw, IGC_PQMPRC(i));
1660                 value.ddword = queue_stats->pqmprc[i];
1661                 if (value.dword[U32_0_IN_U64] > tmp)
1662                         value.dword[U32_1_IN_U64]++;
1663                 value.dword[U32_0_IN_U64] = tmp;
1664                 queue_stats->pqmprc[i] = value.ddword;
1665
1666                 tmp = IGC_READ_REG(hw, IGC_RQDPC(i));
1667                 value.ddword = queue_stats->rqdpc[i];
1668                 if (value.dword[U32_0_IN_U64] > tmp)
1669                         value.dword[U32_1_IN_U64]++;
1670                 value.dword[U32_0_IN_U64] = tmp;
1671                 queue_stats->rqdpc[i] = value.ddword;
1672
1673                 tmp = IGC_READ_REG(hw, IGC_TQDPC(i));
1674                 value.ddword = queue_stats->tqdpc[i];
1675                 if (value.dword[U32_0_IN_U64] > tmp)
1676                         value.dword[U32_1_IN_U64]++;
1677                 value.dword[U32_0_IN_U64] = tmp;
1678                 queue_stats->tqdpc[i] = value.ddword;
1679         }
1680 }
1681
1682 static int
1683 eth_igc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1684 {
1685         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1686         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1687         struct igc_hw_stats *stats = IGC_DEV_PRIVATE_STATS(dev);
1688         struct igc_hw_queue_stats *queue_stats =
1689                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1690         int i;
1691
1692         /*
1693          * Cancel status handler since it will read the queue status registers
1694          */
1695         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1696
1697         /* Read status register */
1698         igc_read_queue_stats_register(dev);
1699         igc_read_stats_registers(hw, stats);
1700
1701         if (rte_stats == NULL) {
1702                 /* Restart queue status handler */
1703                 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1704                                 igc_update_queue_stats_handler, dev);
1705                 return -EINVAL;
1706         }
1707
1708         /* Rx Errors */
1709         rte_stats->imissed = stats->mpc;
1710         rte_stats->ierrors = stats->crcerrs +
1711                         stats->rlec + stats->ruc + stats->roc +
1712                         stats->rxerrc + stats->algnerrc;
1713
1714         /* Tx Errors */
1715         rte_stats->oerrors = stats->ecol + stats->latecol;
1716
1717         rte_stats->ipackets = stats->gprc;
1718         rte_stats->opackets = stats->gptc;
1719         rte_stats->ibytes   = stats->gorc;
1720         rte_stats->obytes   = stats->gotc;
1721
1722         /* Get per-queue statuses */
1723         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1724                 /* GET TX queue statuses */
1725                 int map_id = igc->txq_stats_map[i];
1726                 if (map_id >= 0) {
1727                         rte_stats->q_opackets[map_id] += queue_stats->pqgptc[i];
1728                         rte_stats->q_obytes[map_id] += queue_stats->pqgotc[i];
1729                 }
1730                 /* Get RX queue statuses */
1731                 map_id = igc->rxq_stats_map[i];
1732                 if (map_id >= 0) {
1733                         rte_stats->q_ipackets[map_id] += queue_stats->pqgprc[i];
1734                         rte_stats->q_ibytes[map_id] += queue_stats->pqgorc[i];
1735                         rte_stats->q_errors[map_id] += queue_stats->rqdpc[i];
1736                 }
1737         }
1738
1739         /* Restart queue status handler */
1740         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1741                         igc_update_queue_stats_handler, dev);
1742         return 0;
1743 }
1744
1745 static int
1746 eth_igc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1747                    unsigned int n)
1748 {
1749         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1750         struct igc_hw_stats *hw_stats =
1751                         IGC_DEV_PRIVATE_STATS(dev);
1752         unsigned int i;
1753
1754         igc_read_stats_registers(hw, hw_stats);
1755
1756         if (n < IGC_NB_XSTATS)
1757                 return IGC_NB_XSTATS;
1758
1759         /* If this is a reset xstats is NULL, and we have cleared the
1760          * registers by reading them.
1761          */
1762         if (!xstats)
1763                 return 0;
1764
1765         /* Extended stats */
1766         for (i = 0; i < IGC_NB_XSTATS; i++) {
1767                 xstats[i].id = i;
1768                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1769                         rte_igc_stats_strings[i].offset);
1770         }
1771
1772         return IGC_NB_XSTATS;
1773 }
1774
1775 static int
1776 eth_igc_xstats_reset(struct rte_eth_dev *dev)
1777 {
1778         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1779         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1780         struct igc_hw_queue_stats *queue_stats =
1781                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1782
1783         /* Cancel queue status handler for avoid conflict */
1784         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1785
1786         /* HW registers are cleared on read */
1787         igc_reset_queue_stats_register(hw);
1788         igc_read_stats_registers(hw, hw_stats);
1789
1790         /* Reset software totals */
1791         memset(hw_stats, 0, sizeof(*hw_stats));
1792         memset(queue_stats, 0, sizeof(*queue_stats));
1793
1794         /* Restart the queue status handler */
1795         rte_eal_alarm_set(IGC_ALARM_INTERVAL, igc_update_queue_stats_handler,
1796                         dev);
1797
1798         return 0;
1799 }
1800
1801 static int
1802 eth_igc_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1803         struct rte_eth_xstat_name *xstats_names, unsigned int size)
1804 {
1805         unsigned int i;
1806
1807         if (xstats_names == NULL)
1808                 return IGC_NB_XSTATS;
1809
1810         if (size < IGC_NB_XSTATS) {
1811                 PMD_DRV_LOG(ERR, "not enough buffers!");
1812                 return IGC_NB_XSTATS;
1813         }
1814
1815         for (i = 0; i < IGC_NB_XSTATS; i++)
1816                 strlcpy(xstats_names[i].name, rte_igc_stats_strings[i].name,
1817                         sizeof(xstats_names[i].name));
1818
1819         return IGC_NB_XSTATS;
1820 }
1821
1822 static int
1823 eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
1824                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
1825                 unsigned int limit)
1826 {
1827         unsigned int i;
1828
1829         if (!ids)
1830                 return eth_igc_xstats_get_names(dev, xstats_names, limit);
1831
1832         for (i = 0; i < limit; i++) {
1833                 if (ids[i] >= IGC_NB_XSTATS) {
1834                         PMD_DRV_LOG(ERR, "id value isn't valid");
1835                         return -EINVAL;
1836                 }
1837                 strlcpy(xstats_names[i].name,
1838                         rte_igc_stats_strings[ids[i]].name,
1839                         sizeof(xstats_names[i].name));
1840         }
1841         return limit;
1842 }
1843
1844 static int
1845 eth_igc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1846                 uint64_t *values, unsigned int n)
1847 {
1848         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1849         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1850         unsigned int i;
1851
1852         igc_read_stats_registers(hw, hw_stats);
1853
1854         if (!ids) {
1855                 if (n < IGC_NB_XSTATS)
1856                         return IGC_NB_XSTATS;
1857
1858                 /* If this is a reset xstats is NULL, and we have cleared the
1859                  * registers by reading them.
1860                  */
1861                 if (!values)
1862                         return 0;
1863
1864                 /* Extended stats */
1865                 for (i = 0; i < IGC_NB_XSTATS; i++)
1866                         values[i] = *(uint64_t *)(((char *)hw_stats) +
1867                                         rte_igc_stats_strings[i].offset);
1868
1869                 return IGC_NB_XSTATS;
1870
1871         } else {
1872                 for (i = 0; i < n; i++) {
1873                         if (ids[i] >= IGC_NB_XSTATS) {
1874                                 PMD_DRV_LOG(ERR, "id value isn't valid");
1875                                 return -EINVAL;
1876                         }
1877                         values[i] = *(uint64_t *)(((char *)hw_stats) +
1878                                         rte_igc_stats_strings[ids[i]].offset);
1879                 }
1880                 return n;
1881         }
1882 }
1883
1884 static int
1885 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
1886                 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx)
1887 {
1888         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1889
1890         /* check queue id is valid */
1891         if (queue_id >= IGC_QUEUE_PAIRS_NUM) {
1892                 PMD_DRV_LOG(ERR, "queue id(%u) error, max is %u",
1893                         queue_id, IGC_QUEUE_PAIRS_NUM - 1);
1894                 return -EINVAL;
1895         }
1896
1897         /* store the mapping status id */
1898         if (is_rx)
1899                 igc->rxq_stats_map[queue_id] = stat_idx;
1900         else
1901                 igc->txq_stats_map[queue_id] = stat_idx;
1902
1903         return 0;
1904 }
1905
1906 static int
1907 eth_igc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1908         struct rte_pci_device *pci_dev)
1909 {
1910         PMD_INIT_FUNC_TRACE();
1911         return rte_eth_dev_pci_generic_probe(pci_dev,
1912                 sizeof(struct igc_adapter), eth_igc_dev_init);
1913 }
1914
1915 static int
1916 eth_igc_pci_remove(struct rte_pci_device *pci_dev)
1917 {
1918         PMD_INIT_FUNC_TRACE();
1919         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igc_dev_uninit);
1920 }
1921
1922 static struct rte_pci_driver rte_igc_pmd = {
1923         .id_table = pci_id_igc_map,
1924         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1925         .probe = eth_igc_pci_probe,
1926         .remove = eth_igc_pci_remove,
1927 };
1928
1929 RTE_PMD_REGISTER_PCI(net_igc, rte_igc_pmd);
1930 RTE_PMD_REGISTER_PCI_TABLE(net_igc, pci_id_igc_map);
1931 RTE_PMD_REGISTER_KMOD_DEP(net_igc, "* igb_uio | uio_pci_generic | vfio-pci");