drivers/net/igc/igc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7
   8 #include <rte_string_fns.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_malloc.h>
  14 #include <rte_alarm.h>
  15
  16 #include "igc_logs.h"
  17 #include "igc_txrx.h"
  18 #include "igc_filter.h"
  19 #include "igc_flow.h"
  20
  21 #define IGC_INTEL_VENDOR_ID             0x8086
  22
  23 /*
  24  * The overhead from MTU to max frame size.
  25  * Considering VLAN so tag needs to be counted.
  26  */
  27 #define IGC_ETH_OVERHEAD                (RTE_ETHER_HDR_LEN + \
  28                                         RTE_ETHER_CRC_LEN + VLAN_TAG_SIZE)
  29
  30 #define IGC_FC_PAUSE_TIME               0x0680
  31 #define IGC_LINK_UPDATE_CHECK_TIMEOUT   90  /* 9s */
  32 #define IGC_LINK_UPDATE_CHECK_INTERVAL  100 /* ms */
  33
  34 #define IGC_MISC_VEC_ID                 RTE_INTR_VEC_ZERO_OFFSET
  35 #define IGC_RX_VEC_START                RTE_INTR_VEC_RXTX_OFFSET
  36 #define IGC_MSIX_OTHER_INTR_VEC         0   /* MSI-X other interrupt vector */
  37 #define IGC_FLAG_NEED_LINK_UPDATE       (1u << 0)       /* need update link */
  38
  39 #define IGC_DEFAULT_RX_FREE_THRESH      32
  40
  41 #define IGC_DEFAULT_RX_PTHRESH          8
  42 #define IGC_DEFAULT_RX_HTHRESH          8
  43 #define IGC_DEFAULT_RX_WTHRESH          4
  44
  45 #define IGC_DEFAULT_TX_PTHRESH          8
  46 #define IGC_DEFAULT_TX_HTHRESH          1
  47 #define IGC_DEFAULT_TX_WTHRESH          16
  48
  49 /* MSI-X other interrupt vector */
  50 #define IGC_MSIX_OTHER_INTR_VEC         0
  51
  52 /* External VLAN Enable bit mask */
  53 #define IGC_CTRL_EXT_EXT_VLAN           (1u << 26)
  54
  55 /* Speed select */
  56 #define IGC_CTRL_SPEED_MASK             (7u << 8)
  57 #define IGC_CTRL_SPEED_2500             (6u << 8)
  58
  59 /* External VLAN Ether Type bit mask and shift */
  60 #define IGC_VET_EXT                     0xFFFF0000
  61 #define IGC_VET_EXT_SHIFT               16
  62
  63 /* Force EEE Auto-negotiation */
  64 #define IGC_EEER_EEE_FRC_AN             (1u << 28)
  65
  66 /* Per Queue Good Packets Received Count */
  67 #define IGC_PQGPRC(idx)         (0x10010 + 0x100 * (idx))
  68 /* Per Queue Good Octets Received Count */
  69 #define IGC_PQGORC(idx)         (0x10018 + 0x100 * (idx))
  70 /* Per Queue Good Octets Transmitted Count */
  71 #define IGC_PQGOTC(idx)         (0x10034 + 0x100 * (idx))
  72 /* Per Queue Multicast Packets Received Count */
  73 #define IGC_PQMPRC(idx)         (0x10038 + 0x100 * (idx))
  74 /* Transmit Queue Drop Packet Count */
  75 #define IGC_TQDPC(idx)          (0xe030 + 0x40 * (idx))
  76
  77 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
  78 #define U32_0_IN_U64            0       /* lower bytes of u64 */
  79 #define U32_1_IN_U64            1       /* higher bytes of u64 */
  80 #else
  81 #define U32_0_IN_U64            1
  82 #define U32_1_IN_U64            0
  83 #endif
  84
  85 #define IGC_ALARM_INTERVAL      8000000u
  86 /* us, about 13.6s some per-queue registers will wrap around back to 0. */
  87
  88 static const struct rte_eth_desc_lim rx_desc_lim = {
  89         .nb_max = IGC_MAX_RXD,
  90         .nb_min = IGC_MIN_RXD,
  91         .nb_align = IGC_RXD_ALIGN,
  92 };
  93
  94 static const struct rte_eth_desc_lim tx_desc_lim = {
  95         .nb_max = IGC_MAX_TXD,
  96         .nb_min = IGC_MIN_TXD,
  97         .nb_align = IGC_TXD_ALIGN,
  98         .nb_seg_max = IGC_TX_MAX_SEG,
  99         .nb_mtu_seg_max = IGC_TX_MAX_MTU_SEG,
 100 };
 101
 102 static const struct rte_pci_id pci_id_igc_map[] = {
 103         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_LM) },
 104         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_V)  },
 105         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_I)  },
 106         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_K)  },
 107         { .vendor_id = 0, /* sentinel */ },
 108 };
 109
 110 /* store statistics names and its offset in stats structure */
 111 struct rte_igc_xstats_name_off {
 112         char name[RTE_ETH_XSTATS_NAME_SIZE];
 113         unsigned int offset;
 114 };
 115
 116 static const struct rte_igc_xstats_name_off rte_igc_stats_strings[] = {
 117         {"rx_crc_errors", offsetof(struct igc_hw_stats, crcerrs)},
 118         {"rx_align_errors", offsetof(struct igc_hw_stats, algnerrc)},
 119         {"rx_errors", offsetof(struct igc_hw_stats, rxerrc)},
 120         {"rx_missed_packets", offsetof(struct igc_hw_stats, mpc)},
 121         {"tx_single_collision_packets", offsetof(struct igc_hw_stats, scc)},
 122         {"tx_multiple_collision_packets", offsetof(struct igc_hw_stats, mcc)},
 123         {"tx_excessive_collision_packets", offsetof(struct igc_hw_stats,
 124                 ecol)},
 125         {"tx_late_collisions", offsetof(struct igc_hw_stats, latecol)},
 126         {"tx_total_collisions", offsetof(struct igc_hw_stats, colc)},
 127         {"tx_deferred_packets", offsetof(struct igc_hw_stats, dc)},
 128         {"tx_no_carrier_sense_packets", offsetof(struct igc_hw_stats, tncrs)},
 129         {"tx_discarded_packets", offsetof(struct igc_hw_stats, htdpmc)},
 130         {"rx_length_errors", offsetof(struct igc_hw_stats, rlec)},
 131         {"rx_xon_packets", offsetof(struct igc_hw_stats, xonrxc)},
 132         {"tx_xon_packets", offsetof(struct igc_hw_stats, xontxc)},
 133         {"rx_xoff_packets", offsetof(struct igc_hw_stats, xoffrxc)},
 134         {"tx_xoff_packets", offsetof(struct igc_hw_stats, xofftxc)},
 135         {"rx_flow_control_unsupported_packets", offsetof(struct igc_hw_stats,
 136                 fcruc)},
 137         {"rx_size_64_packets", offsetof(struct igc_hw_stats, prc64)},
 138         {"rx_size_65_to_127_packets", offsetof(struct igc_hw_stats, prc127)},
 139         {"rx_size_128_to_255_packets", offsetof(struct igc_hw_stats, prc255)},
 140         {"rx_size_256_to_511_packets", offsetof(struct igc_hw_stats, prc511)},
 141         {"rx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 142                 prc1023)},
 143         {"rx_size_1024_to_max_packets", offsetof(struct igc_hw_stats,
 144                 prc1522)},
 145         {"rx_broadcast_packets", offsetof(struct igc_hw_stats, bprc)},
 146         {"rx_multicast_packets", offsetof(struct igc_hw_stats, mprc)},
 147         {"rx_undersize_errors", offsetof(struct igc_hw_stats, ruc)},
 148         {"rx_fragment_errors", offsetof(struct igc_hw_stats, rfc)},
 149         {"rx_oversize_errors", offsetof(struct igc_hw_stats, roc)},
 150         {"rx_jabber_errors", offsetof(struct igc_hw_stats, rjc)},
 151         {"rx_no_buffers", offsetof(struct igc_hw_stats, rnbc)},
 152         {"rx_management_packets", offsetof(struct igc_hw_stats, mgprc)},
 153         {"rx_management_dropped", offsetof(struct igc_hw_stats, mgpdc)},
 154         {"tx_management_packets", offsetof(struct igc_hw_stats, mgptc)},
 155         {"rx_total_packets", offsetof(struct igc_hw_stats, tpr)},
 156         {"tx_total_packets", offsetof(struct igc_hw_stats, tpt)},
 157         {"rx_total_bytes", offsetof(struct igc_hw_stats, tor)},
 158         {"tx_total_bytes", offsetof(struct igc_hw_stats, tot)},
 159         {"tx_size_64_packets", offsetof(struct igc_hw_stats, ptc64)},
 160         {"tx_size_65_to_127_packets", offsetof(struct igc_hw_stats, ptc127)},
 161         {"tx_size_128_to_255_packets", offsetof(struct igc_hw_stats, ptc255)},
 162         {"tx_size_256_to_511_packets", offsetof(struct igc_hw_stats, ptc511)},
 163         {"tx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 164                 ptc1023)},
 165         {"tx_size_1023_to_max_packets", offsetof(struct igc_hw_stats,
 166                 ptc1522)},
 167         {"tx_multicast_packets", offsetof(struct igc_hw_stats, mptc)},
 168         {"tx_broadcast_packets", offsetof(struct igc_hw_stats, bptc)},
 169         {"tx_tso_packets", offsetof(struct igc_hw_stats, tsctc)},
 170         {"rx_sent_to_host_packets", offsetof(struct igc_hw_stats, rpthc)},
 171         {"tx_sent_by_host_packets", offsetof(struct igc_hw_stats, hgptc)},
 172         {"interrupt_assert_count", offsetof(struct igc_hw_stats, iac)},
 173         {"rx_descriptor_lower_threshold",
 174                 offsetof(struct igc_hw_stats, icrxdmtc)},
 175 };
 176
 177 #define IGC_NB_XSTATS (sizeof(rte_igc_stats_strings) / \
 178                 sizeof(rte_igc_stats_strings[0]))
 179
 180 static int eth_igc_configure(struct rte_eth_dev *dev);
 181 static int eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
 182 static void eth_igc_stop(struct rte_eth_dev *dev);
 183 static int eth_igc_start(struct rte_eth_dev *dev);
 184 static int eth_igc_set_link_up(struct rte_eth_dev *dev);
 185 static int eth_igc_set_link_down(struct rte_eth_dev *dev);
 186 static void eth_igc_close(struct rte_eth_dev *dev);
 187 static int eth_igc_reset(struct rte_eth_dev *dev);
 188 static int eth_igc_promiscuous_enable(struct rte_eth_dev *dev);
 189 static int eth_igc_promiscuous_disable(struct rte_eth_dev *dev);
 190 static int eth_igc_fw_version_get(struct rte_eth_dev *dev,
 191                                 char *fw_version, size_t fw_size);
 192 static int eth_igc_infos_get(struct rte_eth_dev *dev,
 193                         struct rte_eth_dev_info *dev_info);
 194 static int eth_igc_led_on(struct rte_eth_dev *dev);
 195 static int eth_igc_led_off(struct rte_eth_dev *dev);
 196 static const uint32_t *eth_igc_supported_ptypes_get(struct rte_eth_dev *dev);
 197 static int eth_igc_rar_set(struct rte_eth_dev *dev,
 198                 struct rte_ether_addr *mac_addr, uint32_t index, uint32_t pool);
 199 static void eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 200 static int eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
 201                         struct rte_ether_addr *addr);
 202 static int eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
 203                          struct rte_ether_addr *mc_addr_set,
 204                          uint32_t nb_mc_addr);
 205 static int eth_igc_allmulticast_enable(struct rte_eth_dev *dev);
 206 static int eth_igc_allmulticast_disable(struct rte_eth_dev *dev);
 207 static int eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 208 static int eth_igc_stats_get(struct rte_eth_dev *dev,
 209                         struct rte_eth_stats *rte_stats);
 210 static int eth_igc_xstats_get(struct rte_eth_dev *dev,
 211                         struct rte_eth_xstat *xstats, unsigned int n);
 212 static int eth_igc_xstats_get_by_id(struct rte_eth_dev *dev,
 213                                 const uint64_t *ids,
 214                                 uint64_t *values, unsigned int n);
 215 static int eth_igc_xstats_get_names(struct rte_eth_dev *dev,
 216                                 struct rte_eth_xstat_name *xstats_names,
 217                                 unsigned int size);
 218 static int eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 219                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
 220                 unsigned int limit);
 221 static int eth_igc_xstats_reset(struct rte_eth_dev *dev);
 222 static int
 223 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
 224         uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx);
 225 static int
 226 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 227 static int
 228 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 229 static int
 230 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 231 static int
 232 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 233 static int eth_igc_rss_reta_update(struct rte_eth_dev *dev,
 234                         struct rte_eth_rss_reta_entry64 *reta_conf,
 235                         uint16_t reta_size);
 236 static int eth_igc_rss_reta_query(struct rte_eth_dev *dev,
 237                        struct rte_eth_rss_reta_entry64 *reta_conf,
 238                        uint16_t reta_size);
 239 static int eth_igc_rss_hash_update(struct rte_eth_dev *dev,
 240                         struct rte_eth_rss_conf *rss_conf);
 241 static int eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
 242                         struct rte_eth_rss_conf *rss_conf);
 243 static int
 244 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 245 static int eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 246 static int eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
 247                       enum rte_vlan_type vlan_type, uint16_t tpid);
 248
 249 static const struct eth_dev_ops eth_igc_ops = {
 250         .dev_configure          = eth_igc_configure,
 251         .link_update            = eth_igc_link_update,
 252         .dev_stop               = eth_igc_stop,
 253         .dev_start              = eth_igc_start,
 254         .dev_close              = eth_igc_close,
 255         .dev_reset              = eth_igc_reset,
 256         .dev_set_link_up        = eth_igc_set_link_up,
 257         .dev_set_link_down      = eth_igc_set_link_down,
 258         .promiscuous_enable     = eth_igc_promiscuous_enable,
 259         .promiscuous_disable    = eth_igc_promiscuous_disable,
 260         .allmulticast_enable    = eth_igc_allmulticast_enable,
 261         .allmulticast_disable   = eth_igc_allmulticast_disable,
 262         .fw_version_get         = eth_igc_fw_version_get,
 263         .dev_infos_get          = eth_igc_infos_get,
 264         .dev_led_on             = eth_igc_led_on,
 265         .dev_led_off            = eth_igc_led_off,
 266         .dev_supported_ptypes_get = eth_igc_supported_ptypes_get,
 267         .mtu_set                = eth_igc_mtu_set,
 268         .mac_addr_add           = eth_igc_rar_set,
 269         .mac_addr_remove        = eth_igc_rar_clear,
 270         .mac_addr_set           = eth_igc_default_mac_addr_set,
 271         .set_mc_addr_list       = eth_igc_set_mc_addr_list,
 272
 273         .rx_queue_setup         = eth_igc_rx_queue_setup,
 274         .rx_queue_release       = eth_igc_rx_queue_release,
 275         .tx_queue_setup         = eth_igc_tx_queue_setup,
 276         .tx_queue_release       = eth_igc_tx_queue_release,
 277         .tx_done_cleanup        = eth_igc_tx_done_cleanup,
 278         .rxq_info_get           = eth_igc_rxq_info_get,
 279         .txq_info_get           = eth_igc_txq_info_get,
 280         .stats_get              = eth_igc_stats_get,
 281         .xstats_get             = eth_igc_xstats_get,
 282         .xstats_get_by_id       = eth_igc_xstats_get_by_id,
 283         .xstats_get_names_by_id = eth_igc_xstats_get_names_by_id,
 284         .xstats_get_names       = eth_igc_xstats_get_names,
 285         .stats_reset            = eth_igc_xstats_reset,
 286         .xstats_reset           = eth_igc_xstats_reset,
 287         .queue_stats_mapping_set = eth_igc_queue_stats_mapping_set,
 288         .rx_queue_intr_enable   = eth_igc_rx_queue_intr_enable,
 289         .rx_queue_intr_disable  = eth_igc_rx_queue_intr_disable,
 290         .flow_ctrl_get          = eth_igc_flow_ctrl_get,
 291         .flow_ctrl_set          = eth_igc_flow_ctrl_set,
 292         .reta_update            = eth_igc_rss_reta_update,
 293         .reta_query             = eth_igc_rss_reta_query,
 294         .rss_hash_update        = eth_igc_rss_hash_update,
 295         .rss_hash_conf_get      = eth_igc_rss_hash_conf_get,
 296         .vlan_filter_set        = eth_igc_vlan_filter_set,
 297         .vlan_offload_set       = eth_igc_vlan_offload_set,
 298         .vlan_tpid_set          = eth_igc_vlan_tpid_set,
 299         .vlan_strip_queue_set   = eth_igc_vlan_strip_queue_set,
 300         .filter_ctrl            = eth_igc_filter_ctrl,
 301 };
 302
 303 /*
 304  * multiple queue mode checking
 305  */
 306 static int
 307 igc_check_mq_mode(struct rte_eth_dev *dev)
 308 {
 309         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 310         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
 311
 312         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 313                 PMD_INIT_LOG(ERR, "SRIOV is not supported.");
 314                 return -EINVAL;
 315         }
 316
 317         if (rx_mq_mode != ETH_MQ_RX_NONE &&
 318                 rx_mq_mode != ETH_MQ_RX_RSS) {
 319                 /* RSS together with VMDq not supported*/
 320                 PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
 321                                 rx_mq_mode);
 322                 return -EINVAL;
 323         }
 324
 325         /* To no break software that set invalid mode, only display
 326          * warning if invalid mode is used.
 327          */
 328         if (tx_mq_mode != ETH_MQ_TX_NONE)
 329                 PMD_INIT_LOG(WARNING,
 330                         "TX mode %d is not supported. Due to meaningless in this driver, just ignore",
 331                         tx_mq_mode);
 332
 333         return 0;
 334 }
 335
 336 static int
 337 eth_igc_configure(struct rte_eth_dev *dev)
 338 {
 339         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 340         int ret;
 341
 342         PMD_INIT_FUNC_TRACE();
 343
 344         ret  = igc_check_mq_mode(dev);
 345         if (ret != 0)
 346                 return ret;
 347
 348         intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 349         return 0;
 350 }
 351
 352 static int
 353 eth_igc_set_link_up(struct rte_eth_dev *dev)
 354 {
 355         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 356
 357         if (hw->phy.media_type == igc_media_type_copper)
 358                 igc_power_up_phy(hw);
 359         else
 360                 igc_power_up_fiber_serdes_link(hw);
 361         return 0;
 362 }
 363
 364 static int
 365 eth_igc_set_link_down(struct rte_eth_dev *dev)
 366 {
 367         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 368
 369         if (hw->phy.media_type == igc_media_type_copper)
 370                 igc_power_down_phy(hw);
 371         else
 372                 igc_shutdown_fiber_serdes_link(hw);
 373         return 0;
 374 }
 375
 376 /*
 377  * disable other interrupt
 378  */
 379 static void
 380 igc_intr_other_disable(struct rte_eth_dev *dev)
 381 {
 382         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 383         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 384         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 385
 386         if (rte_intr_allow_others(intr_handle) &&
 387                 dev->data->dev_conf.intr_conf.lsc) {
 388                 IGC_WRITE_REG(hw, IGC_EIMC, 1u << IGC_MSIX_OTHER_INTR_VEC);
 389         }
 390
 391         IGC_WRITE_REG(hw, IGC_IMC, ~0);
 392         IGC_WRITE_FLUSH(hw);
 393 }
 394
 395 /*
 396  * enable other interrupt
 397  */
 398 static inline void
 399 igc_intr_other_enable(struct rte_eth_dev *dev)
 400 {
 401         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 402         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 403         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 404         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 405
 406         if (rte_intr_allow_others(intr_handle) &&
 407                 dev->data->dev_conf.intr_conf.lsc) {
 408                 IGC_WRITE_REG(hw, IGC_EIMS, 1u << IGC_MSIX_OTHER_INTR_VEC);
 409         }
 410
 411         IGC_WRITE_REG(hw, IGC_IMS, intr->mask);
 412         IGC_WRITE_FLUSH(hw);
 413 }
 414
 415 /*
 416  * It reads ICR and gets interrupt causes, check it and set a bit flag
 417  * to update link status.
 418  */
 419 static void
 420 eth_igc_interrupt_get_status(struct rte_eth_dev *dev)
 421 {
 422         uint32_t icr;
 423         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 424         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 425
 426         /* read-on-clear nic registers here */
 427         icr = IGC_READ_REG(hw, IGC_ICR);
 428
 429         intr->flags = 0;
 430         if (icr & IGC_ICR_LSC)
 431                 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 432 }
 433
 434 /* return 0 means link status changed, -1 means not changed */
 435 static int
 436 eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 437 {
 438         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 439         struct rte_eth_link link;
 440         int link_check, count;
 441
 442         link_check = 0;
 443         hw->mac.get_link_status = 1;
 444
 445         /* possible wait-to-complete in up to 9 seconds */
 446         for (count = 0; count < IGC_LINK_UPDATE_CHECK_TIMEOUT; count++) {
 447                 /* Read the real link status */
 448                 switch (hw->phy.media_type) {
 449                 case igc_media_type_copper:
 450                         /* Do the work to read phy */
 451                         igc_check_for_link(hw);
 452                         link_check = !hw->mac.get_link_status;
 453                         break;
 454
 455                 case igc_media_type_fiber:
 456                         igc_check_for_link(hw);
 457                         link_check = (IGC_READ_REG(hw, IGC_STATUS) &
 458                                       IGC_STATUS_LU);
 459                         break;
 460
 461                 case igc_media_type_internal_serdes:
 462                         igc_check_for_link(hw);
 463                         link_check = hw->mac.serdes_has_link;
 464                         break;
 465
 466                 default:
 467                         break;
 468                 }
 469                 if (link_check || wait_to_complete == 0)
 470                         break;
 471                 rte_delay_ms(IGC_LINK_UPDATE_CHECK_INTERVAL);
 472         }
 473         memset(&link, 0, sizeof(link));
 474
 475         /* Now we check if a transition has happened */
 476         if (link_check) {
 477                 uint16_t duplex, speed;
 478                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
 479                 link.link_duplex = (duplex == FULL_DUPLEX) ?
 480                                 ETH_LINK_FULL_DUPLEX :
 481                                 ETH_LINK_HALF_DUPLEX;
 482                 link.link_speed = speed;
 483                 link.link_status = ETH_LINK_UP;
 484                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 485                                 ETH_LINK_SPEED_FIXED);
 486
 487                 if (speed == SPEED_2500) {
 488                         uint32_t tipg = IGC_READ_REG(hw, IGC_TIPG);
 489                         if ((tipg & IGC_TIPG_IPGT_MASK) != 0x0b) {
 490                                 tipg &= ~IGC_TIPG_IPGT_MASK;
 491                                 tipg |= 0x0b;
 492                                 IGC_WRITE_REG(hw, IGC_TIPG, tipg);
 493                         }
 494                 }
 495         } else {
 496                 link.link_speed = 0;
 497                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
 498                 link.link_status = ETH_LINK_DOWN;
 499                 link.link_autoneg = ETH_LINK_FIXED;
 500         }
 501
 502         return rte_eth_linkstatus_set(dev, &link);
 503 }
 504
 505 /*
 506  * It executes link_update after knowing an interrupt is present.
 507  */
 508 static void
 509 eth_igc_interrupt_action(struct rte_eth_dev *dev)
 510 {
 511         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 512         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 513         struct rte_eth_link link;
 514         int ret;
 515
 516         if (intr->flags & IGC_FLAG_NEED_LINK_UPDATE) {
 517                 intr->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
 518
 519                 /* set get_link_status to check register later */
 520                 ret = eth_igc_link_update(dev, 0);
 521
 522                 /* check if link has changed */
 523                 if (ret < 0)
 524                         return;
 525
 526                 rte_eth_linkstatus_get(dev, &link);
 527                 if (link.link_status)
 528                         PMD_DRV_LOG(INFO,
 529                                 " Port %d: Link Up - speed %u Mbps - %s",
 530                                 dev->data->port_id,
 531                                 (unsigned int)link.link_speed,
 532                                 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
 533                                 "full-duplex" : "half-duplex");
 534                 else
 535                         PMD_DRV_LOG(INFO, " Port %d: Link Down",
 536                                 dev->data->port_id);
 537
 538                 PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
 539                                 pci_dev->addr.domain,
 540                                 pci_dev->addr.bus,
 541                                 pci_dev->addr.devid,
 542                                 pci_dev->addr.function);
 543                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
 544                                 NULL);
 545         }
 546 }
 547
 548 /*
 549  * Interrupt handler which shall be registered at first.
 550  *
 551  * @handle
 552  *  Pointer to interrupt handle.
 553  * @param
 554  *  The address of parameter (struct rte_eth_dev *) registered before.
 555  */
 556 static void
 557 eth_igc_interrupt_handler(void *param)
 558 {
 559         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 560
 561         eth_igc_interrupt_get_status(dev);
 562         eth_igc_interrupt_action(dev);
 563 }
 564
 565 static void igc_read_queue_stats_register(struct rte_eth_dev *dev);
 566
 567 /*
 568  * Update the queue status every IGC_ALARM_INTERVAL time.
 569  * @param
 570  *  The address of parameter (struct rte_eth_dev *) registered before.
 571  */
 572 static void
 573 igc_update_queue_stats_handler(void *param)
 574 {
 575         struct rte_eth_dev *dev = param;
 576         igc_read_queue_stats_register(dev);
 577         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
 578                         igc_update_queue_stats_handler, dev);
 579 }
 580
 581 /*
 582  * rx,tx enable/disable
 583  */
 584 static void
 585 eth_igc_rxtx_control(struct rte_eth_dev *dev, bool enable)
 586 {
 587         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 588         uint32_t tctl, rctl;
 589
 590         tctl = IGC_READ_REG(hw, IGC_TCTL);
 591         rctl = IGC_READ_REG(hw, IGC_RCTL);
 592
 593         if (enable) {
 594                 /* enable Tx/Rx */
 595                 tctl |= IGC_TCTL_EN;
 596                 rctl |= IGC_RCTL_EN;
 597         } else {
 598                 /* disable Tx/Rx */
 599                 tctl &= ~IGC_TCTL_EN;
 600                 rctl &= ~IGC_RCTL_EN;
 601         }
 602         IGC_WRITE_REG(hw, IGC_TCTL, tctl);
 603         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
 604         IGC_WRITE_FLUSH(hw);
 605 }
 606
 607 /*
 608  *  This routine disables all traffic on the adapter by issuing a
 609  *  global reset on the MAC.
 610  */
 611 static void
 612 eth_igc_stop(struct rte_eth_dev *dev)
 613 {
 614         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 615         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 616         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 617         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 618         struct rte_eth_link link;
 619
 620         adapter->stopped = 1;
 621
 622         /* disable receive and transmit */
 623         eth_igc_rxtx_control(dev, false);
 624
 625         /* disable all MSI-X interrupts */
 626         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 627         IGC_WRITE_FLUSH(hw);
 628
 629         /* clear all MSI-X interrupts */
 630         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 631
 632         igc_intr_other_disable(dev);
 633
 634         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
 635
 636         /* disable intr eventfd mapping */
 637         rte_intr_disable(intr_handle);
 638
 639         igc_reset_hw(hw);
 640
 641         /* disable all wake up */
 642         IGC_WRITE_REG(hw, IGC_WUC, 0);
 643
 644         /* disable checking EEE operation in MAC loopback mode */
 645         igc_read_reg_check_clear_bits(hw, IGC_EEER, IGC_EEER_EEE_FRC_AN);
 646
 647         /* Set bit for Go Link disconnect */
 648         igc_read_reg_check_set_bits(hw, IGC_82580_PHY_POWER_MGMT,
 649                         IGC_82580_PM_GO_LINKD);
 650
 651         /* Power down the phy. Needed to make the link go Down */
 652         eth_igc_set_link_down(dev);
 653
 654         igc_dev_clear_queues(dev);
 655
 656         /* clear the recorded link status */
 657         memset(&link, 0, sizeof(link));
 658         rte_eth_linkstatus_set(dev, &link);
 659
 660         if (!rte_intr_allow_others(intr_handle))
 661                 /* resume to the default handler */
 662                 rte_intr_callback_register(intr_handle,
 663                                            eth_igc_interrupt_handler,
 664                                            (void *)dev);
 665
 666         /* Clean datapath event and queue/vec mapping */
 667         rte_intr_efd_disable(intr_handle);
 668         if (intr_handle->intr_vec != NULL) {
 669                 rte_free(intr_handle->intr_vec);
 670                 intr_handle->intr_vec = NULL;
 671         }
 672 }
 673
 674 /*
 675  * write interrupt vector allocation register
 676  * @hw
 677  *  board private structure
 678  * @queue_index
 679  *  queue index, valid 0,1,2,3
 680  * @tx
 681  *  tx:1, rx:0
 682  * @msix_vector
 683  *  msix-vector, valid 0,1,2,3,4
 684  */
 685 static void
 686 igc_write_ivar(struct igc_hw *hw, uint8_t queue_index,
 687                 bool tx, uint8_t msix_vector)
 688 {
 689         uint8_t offset = 0;
 690         uint8_t reg_index = queue_index >> 1;
 691         uint32_t val;
 692
 693         /*
 694          * IVAR(0)
 695          * bit31...24   bit23...16      bit15...8       bit7...0
 696          * TX1          RX1             TX0             RX0
 697          *
 698          * IVAR(1)
 699          * bit31...24   bit23...16      bit15...8       bit7...0
 700          * TX3          RX3             TX2             RX2
 701          */
 702
 703         if (tx)
 704                 offset = 8;
 705
 706         if (queue_index & 1)
 707                 offset += 16;
 708
 709         val = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, reg_index);
 710
 711         /* clear bits */
 712         val &= ~((uint32_t)0xFF << offset);
 713
 714         /* write vector and valid bit */
 715         val |= (uint32_t)(msix_vector | IGC_IVAR_VALID) << offset;
 716
 717         IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, reg_index, val);
 718 }
 719
 720 /* Sets up the hardware to generate MSI-X interrupts properly
 721  * @hw
 722  *  board private structure
 723  */
 724 static void
 725 igc_configure_msix_intr(struct rte_eth_dev *dev)
 726 {
 727         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 728         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 729         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 730
 731         uint32_t intr_mask;
 732         uint32_t vec = IGC_MISC_VEC_ID;
 733         uint32_t base = IGC_MISC_VEC_ID;
 734         uint32_t misc_shift = 0;
 735         int i;
 736
 737         /* won't configure msix register if no mapping is done
 738          * between intr vector and event fd
 739          */
 740         if (!rte_intr_dp_is_en(intr_handle))
 741                 return;
 742
 743         if (rte_intr_allow_others(intr_handle)) {
 744                 base = IGC_RX_VEC_START;
 745                 vec = base;
 746                 misc_shift = 1;
 747         }
 748
 749         /* turn on MSI-X capability first */
 750         IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE |
 751                                 IGC_GPIE_PBA | IGC_GPIE_EIAME |
 752                                 IGC_GPIE_NSICR);
 753         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
 754                 misc_shift;
 755
 756         if (dev->data->dev_conf.intr_conf.lsc)
 757                 intr_mask |= (1u << IGC_MSIX_OTHER_INTR_VEC);
 758
 759         /* enable msix auto-clear */
 760         igc_read_reg_check_set_bits(hw, IGC_EIAC, intr_mask);
 761
 762         /* set other cause interrupt vector */
 763         igc_read_reg_check_set_bits(hw, IGC_IVAR_MISC,
 764                 (uint32_t)(IGC_MSIX_OTHER_INTR_VEC | IGC_IVAR_VALID) << 8);
 765
 766         /* enable auto-mask */
 767         igc_read_reg_check_set_bits(hw, IGC_EIAM, intr_mask);
 768
 769         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 770                 igc_write_ivar(hw, i, 0, vec);
 771                 intr_handle->intr_vec[i] = vec;
 772                 if (vec < base + intr_handle->nb_efd - 1)
 773                         vec++;
 774         }
 775
 776         IGC_WRITE_FLUSH(hw);
 777 }
 778
 779 /**
 780  * It enables the interrupt mask and then enable the interrupt.
 781  *
 782  * @dev
 783  *  Pointer to struct rte_eth_dev.
 784  * @on
 785  *  Enable or Disable
 786  */
 787 static void
 788 igc_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
 789 {
 790         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 791
 792         if (on)
 793                 intr->mask |= IGC_ICR_LSC;
 794         else
 795                 intr->mask &= ~IGC_ICR_LSC;
 796 }
 797
 798 /*
 799  * It enables the interrupt.
 800  * It will be called once only during nic initialized.
 801  */
 802 static void
 803 igc_rxq_interrupt_setup(struct rte_eth_dev *dev)
 804 {
 805         uint32_t mask;
 806         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 807         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 808         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 809         int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0;
 810
 811         /* won't configure msix register if no mapping is done
 812          * between intr vector and event fd
 813          */
 814         if (!rte_intr_dp_is_en(intr_handle))
 815                 return;
 816
 817         mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << misc_shift;
 818         IGC_WRITE_REG(hw, IGC_EIMS, mask);
 819 }
 820
 821 /*
 822  *  Get hardware rx-buffer size.
 823  */
 824 static inline int
 825 igc_get_rx_buffer_size(struct igc_hw *hw)
 826 {
 827         return (IGC_READ_REG(hw, IGC_RXPBS) & 0x3f) << 10;
 828 }
 829
 830 /*
 831  * igc_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
 832  * For ASF and Pass Through versions of f/w this means
 833  * that the driver is loaded.
 834  */
 835 static void
 836 igc_hw_control_acquire(struct igc_hw *hw)
 837 {
 838         uint32_t ctrl_ext;
 839
 840         /* Let firmware know the driver has taken over */
 841         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 842         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 843 }
 844
 845 /*
 846  * igc_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
 847  * For ASF and Pass Through versions of f/w this means that the
 848  * driver is no longer loaded.
 849  */
 850 static void
 851 igc_hw_control_release(struct igc_hw *hw)
 852 {
 853         uint32_t ctrl_ext;
 854
 855         /* Let firmware taken over control of h/w */
 856         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 857         IGC_WRITE_REG(hw, IGC_CTRL_EXT,
 858                         ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 859 }
 860
 861 static int
 862 igc_hardware_init(struct igc_hw *hw)
 863 {
 864         uint32_t rx_buf_size;
 865         int diag;
 866
 867         /* Let the firmware know the OS is in control */
 868         igc_hw_control_acquire(hw);
 869
 870         /* Issue a global reset */
 871         igc_reset_hw(hw);
 872
 873         /* disable all wake up */
 874         IGC_WRITE_REG(hw, IGC_WUC, 0);
 875
 876         /*
 877          * Hardware flow control
 878          * - High water mark should allow for at least two standard size (1518)
 879          *   frames to be received after sending an XOFF.
 880          * - Low water mark works best when it is very near the high water mark.
 881          *   This allows the receiver to restart by sending XON when it has
 882          *   drained a bit. Here we use an arbitrary value of 1500 which will
 883          *   restart after one full frame is pulled from the buffer. There
 884          *   could be several smaller frames in the buffer and if so they will
 885          *   not trigger the XON until their total number reduces the buffer
 886          *   by 1500.
 887          */
 888         rx_buf_size = igc_get_rx_buffer_size(hw);
 889         hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2);
 890         hw->fc.low_water = hw->fc.high_water - 1500;
 891         hw->fc.pause_time = IGC_FC_PAUSE_TIME;
 892         hw->fc.send_xon = 1;
 893         hw->fc.requested_mode = igc_fc_full;
 894
 895         diag = igc_init_hw(hw);
 896         if (diag < 0)
 897                 return diag;
 898
 899         igc_get_phy_info(hw);
 900         igc_check_for_link(hw);
 901
 902         return 0;
 903 }
 904
 905 static int
 906 eth_igc_start(struct rte_eth_dev *dev)
 907 {
 908         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 909         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 910         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 911         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 912         uint32_t *speeds;
 913         int ret;
 914
 915         PMD_INIT_FUNC_TRACE();
 916
 917         /* disable all MSI-X interrupts */
 918         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 919         IGC_WRITE_FLUSH(hw);
 920
 921         /* clear all MSI-X interrupts */
 922         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 923
 924         /* disable uio/vfio intr/eventfd mapping */
 925         if (!adapter->stopped)
 926                 rte_intr_disable(intr_handle);
 927
 928         /* Power up the phy. Needed to make the link go Up */
 929         eth_igc_set_link_up(dev);
 930
 931         /* Put the address into the Receive Address Array */
 932         igc_rar_set(hw, hw->mac.addr, 0);
 933
 934         /* Initialize the hardware */
 935         if (igc_hardware_init(hw)) {
 936                 PMD_DRV_LOG(ERR, "Unable to initialize the hardware");
 937                 return -EIO;
 938         }
 939         adapter->stopped = 0;
 940
 941         /* check and configure queue intr-vector mapping */
 942         if (rte_intr_cap_multiple(intr_handle) &&
 943                 dev->data->dev_conf.intr_conf.rxq) {
 944                 uint32_t intr_vector = dev->data->nb_rx_queues;
 945                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 946                         return -1;
 947         }
 948
 949         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 950                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
 951                         dev->data->nb_rx_queues * sizeof(int), 0);
 952                 if (intr_handle->intr_vec == NULL) {
 953                         PMD_DRV_LOG(ERR,
 954                                 "Failed to allocate %d rx_queues intr_vec",
 955                                 dev->data->nb_rx_queues);
 956                         return -ENOMEM;
 957                 }
 958         }
 959
 960         /* configure msix for rx interrupt */
 961         igc_configure_msix_intr(dev);
 962
 963         igc_tx_init(dev);
 964
 965         /* This can fail when allocating mbufs for descriptor rings */
 966         ret = igc_rx_init(dev);
 967         if (ret) {
 968                 PMD_DRV_LOG(ERR, "Unable to initialize RX hardware");
 969                 igc_dev_clear_queues(dev);
 970                 return ret;
 971         }
 972
 973         igc_clear_hw_cntrs_base_generic(hw);
 974
 975         /* VLAN Offload Settings */
 976         eth_igc_vlan_offload_set(dev,
 977                 ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
 978                 ETH_VLAN_EXTEND_MASK);
 979
 980         /* Setup link speed and duplex */
 981         speeds = &dev->data->dev_conf.link_speeds;
 982         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
 983                 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
 984                 hw->mac.autoneg = 1;
 985         } else {
 986                 int num_speeds = 0;
 987                 bool autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
 988
 989                 /* Reset */
 990                 hw->phy.autoneg_advertised = 0;
 991
 992                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
 993                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
 994                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
 995                                 ETH_LINK_SPEED_FIXED)) {
 996                         num_speeds = -1;
 997                         goto error_invalid_config;
 998                 }
 999                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
1000                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
1001                         num_speeds++;
1002                 }
1003                 if (*speeds & ETH_LINK_SPEED_10M) {
1004                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
1005                         num_speeds++;
1006                 }
1007                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
1008                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1009                         num_speeds++;
1010                 }
1011                 if (*speeds & ETH_LINK_SPEED_100M) {
1012                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1013                         num_speeds++;
1014                 }
1015                 if (*speeds & ETH_LINK_SPEED_1G) {
1016                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1017                         num_speeds++;
1018                 }
1019                 if (*speeds & ETH_LINK_SPEED_2_5G) {
1020                         hw->phy.autoneg_advertised |= ADVERTISE_2500_FULL;
1021                         num_speeds++;
1022                 }
1023                 if (num_speeds == 0 || (!autoneg && num_speeds > 1))
1024                         goto error_invalid_config;
1025
1026                 /* Set/reset the mac.autoneg based on the link speed,
1027                  * fixed or not
1028                  */
1029                 if (!autoneg) {
1030                         hw->mac.autoneg = 0;
1031                         hw->mac.forced_speed_duplex =
1032                                         hw->phy.autoneg_advertised;
1033                 } else {
1034                         hw->mac.autoneg = 1;
1035                 }
1036         }
1037
1038         igc_setup_link(hw);
1039
1040         if (rte_intr_allow_others(intr_handle)) {
1041                 /* check if lsc interrupt is enabled */
1042                 if (dev->data->dev_conf.intr_conf.lsc)
1043                         igc_lsc_interrupt_setup(dev, 1);
1044                 else
1045                         igc_lsc_interrupt_setup(dev, 0);
1046         } else {
1047                 rte_intr_callback_unregister(intr_handle,
1048                                              eth_igc_interrupt_handler,
1049                                              (void *)dev);
1050                 if (dev->data->dev_conf.intr_conf.lsc)
1051                         PMD_DRV_LOG(INFO,
1052                                 "LSC won't enable because of no intr multiplex");
1053         }
1054
1055         /* enable uio/vfio intr/eventfd mapping */
1056         rte_intr_enable(intr_handle);
1057
1058         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1059                         igc_update_queue_stats_handler, dev);
1060
1061         /* check if rxq interrupt is enabled */
1062         if (dev->data->dev_conf.intr_conf.rxq &&
1063                         rte_intr_dp_is_en(intr_handle))
1064                 igc_rxq_interrupt_setup(dev);
1065
1066         /* resume enabled intr since hw reset */
1067         igc_intr_other_enable(dev);
1068
1069         eth_igc_rxtx_control(dev, true);
1070         eth_igc_link_update(dev, 0);
1071
1072         /* configure MAC-loopback mode */
1073         if (dev->data->dev_conf.lpbk_mode == 1) {
1074                 uint32_t reg_val;
1075
1076                 reg_val = IGC_READ_REG(hw, IGC_CTRL);
1077                 reg_val &= ~IGC_CTRL_SPEED_MASK;
1078                 reg_val |= IGC_CTRL_SLU | IGC_CTRL_FRCSPD |
1079                         IGC_CTRL_FRCDPX | IGC_CTRL_FD | IGC_CTRL_SPEED_2500;
1080                 IGC_WRITE_REG(hw, IGC_CTRL, reg_val);
1081
1082                 igc_read_reg_check_set_bits(hw, IGC_EEER, IGC_EEER_EEE_FRC_AN);
1083         }
1084
1085         return 0;
1086
1087 error_invalid_config:
1088         PMD_DRV_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1089                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1090         igc_dev_clear_queues(dev);
1091         return -EINVAL;
1092 }
1093
1094 static int
1095 igc_reset_swfw_lock(struct igc_hw *hw)
1096 {
1097         int ret_val;
1098
1099         /*
1100          * Do mac ops initialization manually here, since we will need
1101          * some function pointers set by this call.
1102          */
1103         ret_val = igc_init_mac_params(hw);
1104         if (ret_val)
1105                 return ret_val;
1106
1107         /*
1108          * SMBI lock should not fail in this early stage. If this is the case,
1109          * it is due to an improper exit of the application.
1110          * So force the release of the faulty lock.
1111          */
1112         if (igc_get_hw_semaphore_generic(hw) < 0)
1113                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
1114
1115         igc_put_hw_semaphore_generic(hw);
1116
1117         if (hw->mac.ops.acquire_swfw_sync != NULL) {
1118                 uint16_t mask;
1119
1120                 /*
1121                  * Phy lock should not fail in this early stage.
1122                  * If this is the case, it is due to an improper exit of the
1123                  * application. So force the release of the faulty lock.
1124                  */
1125                 mask = IGC_SWFW_PHY0_SM;
1126                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
1127                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
1128                                     hw->bus.func);
1129                 }
1130                 hw->mac.ops.release_swfw_sync(hw, mask);
1131
1132                 /*
1133                  * This one is more tricky since it is common to all ports; but
1134                  * swfw_sync retries last long enough (1s) to be almost sure
1135                  * that if lock can not be taken it is due to an improper lock
1136                  * of the semaphore.
1137                  */
1138                 mask = IGC_SWFW_EEP_SM;
1139                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0)
1140                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
1141
1142                 hw->mac.ops.release_swfw_sync(hw, mask);
1143         }
1144
1145         return IGC_SUCCESS;
1146 }
1147
1148 /*
1149  * free all rx/tx queues.
1150  */
1151 static void
1152 igc_dev_free_queues(struct rte_eth_dev *dev)
1153 {
1154         uint16_t i;
1155
1156         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1157                 eth_igc_rx_queue_release(dev->data->rx_queues[i]);
1158                 dev->data->rx_queues[i] = NULL;
1159         }
1160         dev->data->nb_rx_queues = 0;
1161
1162         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1163                 eth_igc_tx_queue_release(dev->data->tx_queues[i]);
1164                 dev->data->tx_queues[i] = NULL;
1165         }
1166         dev->data->nb_tx_queues = 0;
1167 }
1168
1169 static void
1170 eth_igc_close(struct rte_eth_dev *dev)
1171 {
1172         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1173         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1174         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1175         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
1176         int retry = 0;
1177
1178         PMD_INIT_FUNC_TRACE();
1179
1180         if (!adapter->stopped)
1181                 eth_igc_stop(dev);
1182
1183         igc_flow_flush(dev, NULL);
1184         igc_clear_all_filter(dev);
1185
1186         igc_intr_other_disable(dev);
1187         do {
1188                 int ret = rte_intr_callback_unregister(intr_handle,
1189                                 eth_igc_interrupt_handler, dev);
1190                 if (ret >= 0 || ret == -ENOENT || ret == -EINVAL)
1191                         break;
1192
1193                 PMD_DRV_LOG(ERR, "intr callback unregister failed: %d", ret);
1194                 DELAY(200 * 1000); /* delay 200ms */
1195         } while (retry++ < 5);
1196
1197         igc_phy_hw_reset(hw);
1198         igc_hw_control_release(hw);
1199         igc_dev_free_queues(dev);
1200
1201         /* Reset any pending lock */
1202         igc_reset_swfw_lock(hw);
1203 }
1204
1205 static void
1206 igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
1207 {
1208         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1209
1210         hw->vendor_id = pci_dev->id.vendor_id;
1211         hw->device_id = pci_dev->id.device_id;
1212         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1213         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1214 }
1215
1216 static int
1217 eth_igc_dev_init(struct rte_eth_dev *dev)
1218 {
1219         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1220         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1221         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1222         int i, error = 0;
1223
1224         PMD_INIT_FUNC_TRACE();
1225         dev->dev_ops = &eth_igc_ops;
1226         dev->rx_descriptor_done = eth_igc_rx_descriptor_done;
1227         dev->rx_queue_count = eth_igc_rx_queue_count;
1228         dev->rx_descriptor_status = eth_igc_rx_descriptor_status;
1229         dev->tx_descriptor_status = eth_igc_tx_descriptor_status;
1230
1231         /*
1232          * for secondary processes, we don't initialize any further as primary
1233          * has already done this work. Only check we don't need a different
1234          * RX function.
1235          */
1236         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1237                 return 0;
1238
1239         rte_eth_copy_pci_info(dev, pci_dev);
1240
1241         hw->back = pci_dev;
1242         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1243
1244         igc_identify_hardware(dev, pci_dev);
1245         if (igc_setup_init_funcs(hw, false) != IGC_SUCCESS) {
1246                 error = -EIO;
1247                 goto err_late;
1248         }
1249
1250         igc_get_bus_info(hw);
1251
1252         /* Reset any pending lock */
1253         if (igc_reset_swfw_lock(hw) != IGC_SUCCESS) {
1254                 error = -EIO;
1255                 goto err_late;
1256         }
1257
1258         /* Finish initialization */
1259         if (igc_setup_init_funcs(hw, true) != IGC_SUCCESS) {
1260                 error = -EIO;
1261                 goto err_late;
1262         }
1263
1264         hw->mac.autoneg = 1;
1265         hw->phy.autoneg_wait_to_complete = 0;
1266         hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
1267
1268         /* Copper options */
1269         if (hw->phy.media_type == igc_media_type_copper) {
1270                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1271                 hw->phy.disable_polarity_correction = 0;
1272                 hw->phy.ms_type = igc_ms_hw_default;
1273         }
1274
1275         /*
1276          * Start from a known state, this is important in reading the nvm
1277          * and mac from that.
1278          */
1279         igc_reset_hw(hw);
1280
1281         /* Make sure we have a good EEPROM before we read from it */
1282         if (igc_validate_nvm_checksum(hw) < 0) {
1283                 /*
1284                  * Some PCI-E parts fail the first check due to
1285                  * the link being in sleep state, call it again,
1286                  * if it fails a second time its a real issue.
1287                  */
1288                 if (igc_validate_nvm_checksum(hw) < 0) {
1289                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
1290                         error = -EIO;
1291                         goto err_late;
1292                 }
1293         }
1294
1295         /* Read the permanent MAC address out of the EEPROM */
1296         if (igc_read_mac_addr(hw) != 0) {
1297                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
1298                 error = -EIO;
1299                 goto err_late;
1300         }
1301
1302         /* Allocate memory for storing MAC addresses */
1303         dev->data->mac_addrs = rte_zmalloc("igc",
1304                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
1305         if (dev->data->mac_addrs == NULL) {
1306                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes for storing MAC",
1307                                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1308                 error = -ENOMEM;
1309                 goto err_late;
1310         }
1311
1312         /* Copy the permanent MAC address */
1313         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
1314                         &dev->data->mac_addrs[0]);
1315
1316         /* Now initialize the hardware */
1317         if (igc_hardware_init(hw) != 0) {
1318                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
1319                 rte_free(dev->data->mac_addrs);
1320                 dev->data->mac_addrs = NULL;
1321                 error = -ENODEV;
1322                 goto err_late;
1323         }
1324
1325         /* Pass the information to the rte_eth_dev_close() that it should also
1326          * release the private port resources.
1327          */
1328         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1329
1330         hw->mac.get_link_status = 1;
1331         igc->stopped = 0;
1332
1333         /* Indicate SOL/IDER usage */
1334         if (igc_check_reset_block(hw) < 0)
1335                 PMD_INIT_LOG(ERR,
1336                         "PHY reset is blocked due to SOL/IDER session.");
1337
1338         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
1339                         dev->data->port_id, pci_dev->id.vendor_id,
1340                         pci_dev->id.device_id);
1341
1342         rte_intr_callback_register(&pci_dev->intr_handle,
1343                         eth_igc_interrupt_handler, (void *)dev);
1344
1345         /* enable uio/vfio intr/eventfd mapping */
1346         rte_intr_enable(&pci_dev->intr_handle);
1347
1348         /* enable support intr */
1349         igc_intr_other_enable(dev);
1350
1351         /* initiate queue status */
1352         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1353                 igc->txq_stats_map[i] = -1;
1354                 igc->rxq_stats_map[i] = -1;
1355         }
1356
1357         igc_flow_init(dev);
1358         igc_clear_all_filter(dev);
1359         return 0;
1360
1361 err_late:
1362         igc_hw_control_release(hw);
1363         return error;
1364 }
1365
1366 static int
1367 eth_igc_dev_uninit(__rte_unused struct rte_eth_dev *eth_dev)
1368 {
1369         PMD_INIT_FUNC_TRACE();
1370
1371         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1372                 return 0;
1373
1374         eth_igc_close(eth_dev);
1375         return 0;
1376 }
1377
1378 static int
1379 eth_igc_reset(struct rte_eth_dev *dev)
1380 {
1381         int ret;
1382
1383         PMD_INIT_FUNC_TRACE();
1384
1385         ret = eth_igc_dev_uninit(dev);
1386         if (ret)
1387                 return ret;
1388
1389         return eth_igc_dev_init(dev);
1390 }
1391
1392 static int
1393 eth_igc_promiscuous_enable(struct rte_eth_dev *dev)
1394 {
1395         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1396         uint32_t rctl;
1397
1398         rctl = IGC_READ_REG(hw, IGC_RCTL);
1399         rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1400         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1401         return 0;
1402 }
1403
1404 static int
1405 eth_igc_promiscuous_disable(struct rte_eth_dev *dev)
1406 {
1407         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1408         uint32_t rctl;
1409
1410         rctl = IGC_READ_REG(hw, IGC_RCTL);
1411         rctl &= (~IGC_RCTL_UPE);
1412         if (dev->data->all_multicast == 1)
1413                 rctl |= IGC_RCTL_MPE;
1414         else
1415                 rctl &= (~IGC_RCTL_MPE);
1416         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1417         return 0;
1418 }
1419
1420 static int
1421 eth_igc_allmulticast_enable(struct rte_eth_dev *dev)
1422 {
1423         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1424         uint32_t rctl;
1425
1426         rctl = IGC_READ_REG(hw, IGC_RCTL);
1427         rctl |= IGC_RCTL_MPE;
1428         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1429         return 0;
1430 }
1431
1432 static int
1433 eth_igc_allmulticast_disable(struct rte_eth_dev *dev)
1434 {
1435         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1436         uint32_t rctl;
1437
1438         if (dev->data->promiscuous == 1)
1439                 return 0;       /* must remain in all_multicast mode */
1440
1441         rctl = IGC_READ_REG(hw, IGC_RCTL);
1442         rctl &= (~IGC_RCTL_MPE);
1443         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1444         return 0;
1445 }
1446
1447 static int
1448 eth_igc_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
1449                        size_t fw_size)
1450 {
1451         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1452         struct igc_fw_version fw;
1453         int ret;
1454
1455         igc_get_fw_version(hw, &fw);
1456
1457         /* if option rom is valid, display its version too */
1458         if (fw.or_valid) {
1459                 ret = snprintf(fw_version, fw_size,
1460                          "%d.%d, 0x%08x, %d.%d.%d",
1461                          fw.eep_major, fw.eep_minor, fw.etrack_id,
1462                          fw.or_major, fw.or_build, fw.or_patch);
1463         /* no option rom */
1464         } else {
1465                 if (fw.etrack_id != 0X0000) {
1466                         ret = snprintf(fw_version, fw_size,
1467                                  "%d.%d, 0x%08x",
1468                                  fw.eep_major, fw.eep_minor,
1469                                  fw.etrack_id);
1470                 } else {
1471                         ret = snprintf(fw_version, fw_size,
1472                                  "%d.%d.%d",
1473                                  fw.eep_major, fw.eep_minor,
1474                                  fw.eep_build);
1475                 }
1476         }
1477
1478         ret += 1; /* add the size of '\0' */
1479         if (fw_size < (u32)ret)
1480                 return ret;
1481         else
1482                 return 0;
1483 }
1484
1485 static int
1486 eth_igc_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1487 {
1488         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1489
1490         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1491         dev_info->max_rx_pktlen = MAX_RX_JUMBO_FRAME_SIZE;
1492         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1493         dev_info->rx_offload_capa = IGC_RX_OFFLOAD_ALL;
1494         dev_info->tx_offload_capa = IGC_TX_OFFLOAD_ALL;
1495         dev_info->rx_queue_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1496
1497         dev_info->max_rx_queues = IGC_QUEUE_PAIRS_NUM;
1498         dev_info->max_tx_queues = IGC_QUEUE_PAIRS_NUM;
1499         dev_info->max_vmdq_pools = 0;
1500
1501         dev_info->hash_key_size = IGC_HKEY_MAX_INDEX * sizeof(uint32_t);
1502         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1503         dev_info->flow_type_rss_offloads = IGC_RSS_OFFLOAD_ALL;
1504
1505         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1506                 .rx_thresh = {
1507                         .pthresh = IGC_DEFAULT_RX_PTHRESH,
1508                         .hthresh = IGC_DEFAULT_RX_HTHRESH,
1509                         .wthresh = IGC_DEFAULT_RX_WTHRESH,
1510                 },
1511                 .rx_free_thresh = IGC_DEFAULT_RX_FREE_THRESH,
1512                 .rx_drop_en = 0,
1513                 .offloads = 0,
1514         };
1515
1516         dev_info->default_txconf = (struct rte_eth_txconf) {
1517                 .tx_thresh = {
1518                         .pthresh = IGC_DEFAULT_TX_PTHRESH,
1519                         .hthresh = IGC_DEFAULT_TX_HTHRESH,
1520                         .wthresh = IGC_DEFAULT_TX_WTHRESH,
1521                 },
1522                 .offloads = 0,
1523         };
1524
1525         dev_info->rx_desc_lim = rx_desc_lim;
1526         dev_info->tx_desc_lim = tx_desc_lim;
1527
1528         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1529                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1530                         ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G;
1531
1532         dev_info->max_mtu = dev_info->max_rx_pktlen - IGC_ETH_OVERHEAD;
1533         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
1534         return 0;
1535 }
1536
1537 static int
1538 eth_igc_led_on(struct rte_eth_dev *dev)
1539 {
1540         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1541
1542         return igc_led_on(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1543 }
1544
1545 static int
1546 eth_igc_led_off(struct rte_eth_dev *dev)
1547 {
1548         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1549
1550         return igc_led_off(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1551 }
1552
1553 static const uint32_t *
1554 eth_igc_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev)
1555 {
1556         static const uint32_t ptypes[] = {
1557                 /* refers to rx_desc_pkt_info_to_pkt_type() */
1558                 RTE_PTYPE_L2_ETHER,
1559                 RTE_PTYPE_L3_IPV4,
1560                 RTE_PTYPE_L3_IPV4_EXT,
1561                 RTE_PTYPE_L3_IPV6,
1562                 RTE_PTYPE_L3_IPV6_EXT,
1563                 RTE_PTYPE_L4_TCP,
1564                 RTE_PTYPE_L4_UDP,
1565                 RTE_PTYPE_L4_SCTP,
1566                 RTE_PTYPE_TUNNEL_IP,
1567                 RTE_PTYPE_INNER_L3_IPV6,
1568                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1569                 RTE_PTYPE_INNER_L4_TCP,
1570                 RTE_PTYPE_INNER_L4_UDP,
1571                 RTE_PTYPE_UNKNOWN
1572         };
1573
1574         return ptypes;
1575 }
1576
1577 static int
1578 eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1579 {
1580         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1581         uint32_t frame_size = mtu + IGC_ETH_OVERHEAD;
1582         uint32_t rctl;
1583
1584         /* if extend vlan has been enabled */
1585         if (IGC_READ_REG(hw, IGC_CTRL_EXT) & IGC_CTRL_EXT_EXT_VLAN)
1586                 frame_size += VLAN_TAG_SIZE;
1587
1588         /* check that mtu is within the allowed range */
1589         if (mtu < RTE_ETHER_MIN_MTU ||
1590                 frame_size > MAX_RX_JUMBO_FRAME_SIZE)
1591                 return -EINVAL;
1592
1593         /*
1594          * refuse mtu that requires the support of scattered packets when
1595          * this feature has not been enabled before.
1596          */
1597         if (!dev->data->scattered_rx &&
1598             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
1599                 return -EINVAL;
1600
1601         rctl = IGC_READ_REG(hw, IGC_RCTL);
1602
1603         /* switch to jumbo mode if needed */
1604         if (mtu > RTE_ETHER_MTU) {
1605                 dev->data->dev_conf.rxmode.offloads |=
1606                         DEV_RX_OFFLOAD_JUMBO_FRAME;
1607                 rctl |= IGC_RCTL_LPE;
1608         } else {
1609                 dev->data->dev_conf.rxmode.offloads &=
1610                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1611                 rctl &= ~IGC_RCTL_LPE;
1612         }
1613         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1614
1615         /* update max frame size */
1616         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1617
1618         IGC_WRITE_REG(hw, IGC_RLPML,
1619                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1620
1621         return 0;
1622 }
1623
1624 static int
1625 eth_igc_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1626                 uint32_t index, uint32_t pool)
1627 {
1628         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1629
1630         igc_rar_set(hw, mac_addr->addr_bytes, index);
1631         RTE_SET_USED(pool);
1632         return 0;
1633 }
1634
1635 static void
1636 eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1637 {
1638         uint8_t addr[RTE_ETHER_ADDR_LEN];
1639         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1640
1641         memset(addr, 0, sizeof(addr));
1642         igc_rar_set(hw, addr, index);
1643 }
1644
1645 static int
1646 eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
1647                         struct rte_ether_addr *addr)
1648 {
1649         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1650         igc_rar_set(hw, addr->addr_bytes, 0);
1651         return 0;
1652 }
1653
1654 static int
1655 eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
1656                          struct rte_ether_addr *mc_addr_set,
1657                          uint32_t nb_mc_addr)
1658 {
1659         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1660         igc_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
1661         return 0;
1662 }
1663
1664 /*
1665  * Read hardware registers
1666  */
1667 static void
1668 igc_read_stats_registers(struct igc_hw *hw, struct igc_hw_stats *stats)
1669 {
1670         int pause_frames;
1671
1672         uint64_t old_gprc  = stats->gprc;
1673         uint64_t old_gptc  = stats->gptc;
1674         uint64_t old_tpr   = stats->tpr;
1675         uint64_t old_tpt   = stats->tpt;
1676         uint64_t old_rpthc = stats->rpthc;
1677         uint64_t old_hgptc = stats->hgptc;
1678
1679         stats->crcerrs += IGC_READ_REG(hw, IGC_CRCERRS);
1680         stats->algnerrc += IGC_READ_REG(hw, IGC_ALGNERRC);
1681         stats->rxerrc += IGC_READ_REG(hw, IGC_RXERRC);
1682         stats->mpc += IGC_READ_REG(hw, IGC_MPC);
1683         stats->scc += IGC_READ_REG(hw, IGC_SCC);
1684         stats->ecol += IGC_READ_REG(hw, IGC_ECOL);
1685
1686         stats->mcc += IGC_READ_REG(hw, IGC_MCC);
1687         stats->latecol += IGC_READ_REG(hw, IGC_LATECOL);
1688         stats->colc += IGC_READ_REG(hw, IGC_COLC);
1689
1690         stats->dc += IGC_READ_REG(hw, IGC_DC);
1691         stats->tncrs += IGC_READ_REG(hw, IGC_TNCRS);
1692         stats->htdpmc += IGC_READ_REG(hw, IGC_HTDPMC);
1693         stats->rlec += IGC_READ_REG(hw, IGC_RLEC);
1694         stats->xonrxc += IGC_READ_REG(hw, IGC_XONRXC);
1695         stats->xontxc += IGC_READ_REG(hw, IGC_XONTXC);
1696
1697         /*
1698          * For watchdog management we need to know if we have been
1699          * paused during the last interval, so capture that here.
1700          */
1701         pause_frames = IGC_READ_REG(hw, IGC_XOFFRXC);
1702         stats->xoffrxc += pause_frames;
1703         stats->xofftxc += IGC_READ_REG(hw, IGC_XOFFTXC);
1704         stats->fcruc += IGC_READ_REG(hw, IGC_FCRUC);
1705         stats->prc64 += IGC_READ_REG(hw, IGC_PRC64);
1706         stats->prc127 += IGC_READ_REG(hw, IGC_PRC127);
1707         stats->prc255 += IGC_READ_REG(hw, IGC_PRC255);
1708         stats->prc511 += IGC_READ_REG(hw, IGC_PRC511);
1709         stats->prc1023 += IGC_READ_REG(hw, IGC_PRC1023);
1710         stats->prc1522 += IGC_READ_REG(hw, IGC_PRC1522);
1711         stats->gprc += IGC_READ_REG(hw, IGC_GPRC);
1712         stats->bprc += IGC_READ_REG(hw, IGC_BPRC);
1713         stats->mprc += IGC_READ_REG(hw, IGC_MPRC);
1714         stats->gptc += IGC_READ_REG(hw, IGC_GPTC);
1715
1716         /* For the 64-bit byte counters the low dword must be read first. */
1717         /* Both registers clear on the read of the high dword */
1718
1719         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1720         stats->gorc += IGC_READ_REG(hw, IGC_GORCL);
1721         stats->gorc += ((uint64_t)IGC_READ_REG(hw, IGC_GORCH) << 32);
1722         stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN;
1723         stats->gotc += IGC_READ_REG(hw, IGC_GOTCL);
1724         stats->gotc += ((uint64_t)IGC_READ_REG(hw, IGC_GOTCH) << 32);
1725         stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN;
1726
1727         stats->rnbc += IGC_READ_REG(hw, IGC_RNBC);
1728         stats->ruc += IGC_READ_REG(hw, IGC_RUC);
1729         stats->rfc += IGC_READ_REG(hw, IGC_RFC);
1730         stats->roc += IGC_READ_REG(hw, IGC_ROC);
1731         stats->rjc += IGC_READ_REG(hw, IGC_RJC);
1732
1733         stats->mgprc += IGC_READ_REG(hw, IGC_MGTPRC);
1734         stats->mgpdc += IGC_READ_REG(hw, IGC_MGTPDC);
1735         stats->mgptc += IGC_READ_REG(hw, IGC_MGTPTC);
1736         stats->b2ospc += IGC_READ_REG(hw, IGC_B2OSPC);
1737         stats->b2ogprc += IGC_READ_REG(hw, IGC_B2OGPRC);
1738         stats->o2bgptc += IGC_READ_REG(hw, IGC_O2BGPTC);
1739         stats->o2bspc += IGC_READ_REG(hw, IGC_O2BSPC);
1740
1741         stats->tpr += IGC_READ_REG(hw, IGC_TPR);
1742         stats->tpt += IGC_READ_REG(hw, IGC_TPT);
1743
1744         stats->tor += IGC_READ_REG(hw, IGC_TORL);
1745         stats->tor += ((uint64_t)IGC_READ_REG(hw, IGC_TORH) << 32);
1746         stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN;
1747         stats->tot += IGC_READ_REG(hw, IGC_TOTL);
1748         stats->tot += ((uint64_t)IGC_READ_REG(hw, IGC_TOTH) << 32);
1749         stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN;
1750
1751         stats->ptc64 += IGC_READ_REG(hw, IGC_PTC64);
1752         stats->ptc127 += IGC_READ_REG(hw, IGC_PTC127);
1753         stats->ptc255 += IGC_READ_REG(hw, IGC_PTC255);
1754         stats->ptc511 += IGC_READ_REG(hw, IGC_PTC511);
1755         stats->ptc1023 += IGC_READ_REG(hw, IGC_PTC1023);
1756         stats->ptc1522 += IGC_READ_REG(hw, IGC_PTC1522);
1757         stats->mptc += IGC_READ_REG(hw, IGC_MPTC);
1758         stats->bptc += IGC_READ_REG(hw, IGC_BPTC);
1759         stats->tsctc += IGC_READ_REG(hw, IGC_TSCTC);
1760
1761         stats->iac += IGC_READ_REG(hw, IGC_IAC);
1762         stats->rpthc += IGC_READ_REG(hw, IGC_RPTHC);
1763         stats->hgptc += IGC_READ_REG(hw, IGC_HGPTC);
1764         stats->icrxdmtc += IGC_READ_REG(hw, IGC_ICRXDMTC);
1765
1766         /* Host to Card Statistics */
1767         stats->hgorc += IGC_READ_REG(hw, IGC_HGORCL);
1768         stats->hgorc += ((uint64_t)IGC_READ_REG(hw, IGC_HGORCH) << 32);
1769         stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN;
1770         stats->hgotc += IGC_READ_REG(hw, IGC_HGOTCL);
1771         stats->hgotc += ((uint64_t)IGC_READ_REG(hw, IGC_HGOTCH) << 32);
1772         stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN;
1773         stats->lenerrs += IGC_READ_REG(hw, IGC_LENERRS);
1774 }
1775
1776 /*
1777  * Write 0 to all queue status registers
1778  */
1779 static void
1780 igc_reset_queue_stats_register(struct igc_hw *hw)
1781 {
1782         int i;
1783
1784         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1785                 IGC_WRITE_REG(hw, IGC_PQGPRC(i), 0);
1786                 IGC_WRITE_REG(hw, IGC_PQGPTC(i), 0);
1787                 IGC_WRITE_REG(hw, IGC_PQGORC(i), 0);
1788                 IGC_WRITE_REG(hw, IGC_PQGOTC(i), 0);
1789                 IGC_WRITE_REG(hw, IGC_PQMPRC(i), 0);
1790                 IGC_WRITE_REG(hw, IGC_RQDPC(i), 0);
1791                 IGC_WRITE_REG(hw, IGC_TQDPC(i), 0);
1792         }
1793 }
1794
1795 /*
1796  * Read all hardware queue status registers
1797  */
1798 static void
1799 igc_read_queue_stats_register(struct rte_eth_dev *dev)
1800 {
1801         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1802         struct igc_hw_queue_stats *queue_stats =
1803                                 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1804         int i;
1805
1806         /*
1807          * This register is not cleared on read. Furthermore, the register wraps
1808          * around back to 0x00000000 on the next increment when reaching a value
1809          * of 0xFFFFFFFF and then continues normal count operation.
1810          */
1811         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1812                 union {
1813                         u64 ddword;
1814                         u32 dword[2];
1815                 } value;
1816                 u32 tmp;
1817
1818                 /*
1819                  * Read the register first, if the value is smaller than that
1820                  * previous read, that mean the register has been overflowed,
1821                  * then we add the high 4 bytes by 1 and replace the low 4
1822                  * bytes by the new value.
1823                  */
1824                 tmp = IGC_READ_REG(hw, IGC_PQGPRC(i));
1825                 value.ddword = queue_stats->pqgprc[i];
1826                 if (value.dword[U32_0_IN_U64] > tmp)
1827                         value.dword[U32_1_IN_U64]++;
1828                 value.dword[U32_0_IN_U64] = tmp;
1829                 queue_stats->pqgprc[i] = value.ddword;
1830
1831                 tmp = IGC_READ_REG(hw, IGC_PQGPTC(i));
1832                 value.ddword = queue_stats->pqgptc[i];
1833                 if (value.dword[U32_0_IN_U64] > tmp)
1834                         value.dword[U32_1_IN_U64]++;
1835                 value.dword[U32_0_IN_U64] = tmp;
1836                 queue_stats->pqgptc[i] = value.ddword;
1837
1838                 tmp = IGC_READ_REG(hw, IGC_PQGORC(i));
1839                 value.ddword = queue_stats->pqgorc[i];
1840                 if (value.dword[U32_0_IN_U64] > tmp)
1841                         value.dword[U32_1_IN_U64]++;
1842                 value.dword[U32_0_IN_U64] = tmp;
1843                 queue_stats->pqgorc[i] = value.ddword;
1844
1845                 tmp = IGC_READ_REG(hw, IGC_PQGOTC(i));
1846                 value.ddword = queue_stats->pqgotc[i];
1847                 if (value.dword[U32_0_IN_U64] > tmp)
1848                         value.dword[U32_1_IN_U64]++;
1849                 value.dword[U32_0_IN_U64] = tmp;
1850                 queue_stats->pqgotc[i] = value.ddword;
1851
1852                 tmp = IGC_READ_REG(hw, IGC_PQMPRC(i));
1853                 value.ddword = queue_stats->pqmprc[i];
1854                 if (value.dword[U32_0_IN_U64] > tmp)
1855                         value.dword[U32_1_IN_U64]++;
1856                 value.dword[U32_0_IN_U64] = tmp;
1857                 queue_stats->pqmprc[i] = value.ddword;
1858
1859                 tmp = IGC_READ_REG(hw, IGC_RQDPC(i));
1860                 value.ddword = queue_stats->rqdpc[i];
1861                 if (value.dword[U32_0_IN_U64] > tmp)
1862                         value.dword[U32_1_IN_U64]++;
1863                 value.dword[U32_0_IN_U64] = tmp;
1864                 queue_stats->rqdpc[i] = value.ddword;
1865
1866                 tmp = IGC_READ_REG(hw, IGC_TQDPC(i));
1867                 value.ddword = queue_stats->tqdpc[i];
1868                 if (value.dword[U32_0_IN_U64] > tmp)
1869                         value.dword[U32_1_IN_U64]++;
1870                 value.dword[U32_0_IN_U64] = tmp;
1871                 queue_stats->tqdpc[i] = value.ddword;
1872         }
1873 }
1874
1875 static int
1876 eth_igc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1877 {
1878         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1879         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1880         struct igc_hw_stats *stats = IGC_DEV_PRIVATE_STATS(dev);
1881         struct igc_hw_queue_stats *queue_stats =
1882                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1883         int i;
1884
1885         /*
1886          * Cancel status handler since it will read the queue status registers
1887          */
1888         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1889
1890         /* Read status register */
1891         igc_read_queue_stats_register(dev);
1892         igc_read_stats_registers(hw, stats);
1893
1894         if (rte_stats == NULL) {
1895                 /* Restart queue status handler */
1896                 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1897                                 igc_update_queue_stats_handler, dev);
1898                 return -EINVAL;
1899         }
1900
1901         /* Rx Errors */
1902         rte_stats->imissed = stats->mpc;
1903         rte_stats->ierrors = stats->crcerrs +
1904                         stats->rlec + stats->ruc + stats->roc +
1905                         stats->rxerrc + stats->algnerrc;
1906
1907         /* Tx Errors */
1908         rte_stats->oerrors = stats->ecol + stats->latecol;
1909
1910         rte_stats->ipackets = stats->gprc;
1911         rte_stats->opackets = stats->gptc;
1912         rte_stats->ibytes   = stats->gorc;
1913         rte_stats->obytes   = stats->gotc;
1914
1915         /* Get per-queue statuses */
1916         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1917                 /* GET TX queue statuses */
1918                 int map_id = igc->txq_stats_map[i];
1919                 if (map_id >= 0) {
1920                         rte_stats->q_opackets[map_id] += queue_stats->pqgptc[i];
1921                         rte_stats->q_obytes[map_id] += queue_stats->pqgotc[i];
1922                 }
1923                 /* Get RX queue statuses */
1924                 map_id = igc->rxq_stats_map[i];
1925                 if (map_id >= 0) {
1926                         rte_stats->q_ipackets[map_id] += queue_stats->pqgprc[i];
1927                         rte_stats->q_ibytes[map_id] += queue_stats->pqgorc[i];
1928                         rte_stats->q_errors[map_id] += queue_stats->rqdpc[i];
1929                 }
1930         }
1931
1932         /* Restart queue status handler */
1933         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1934                         igc_update_queue_stats_handler, dev);
1935         return 0;
1936 }
1937
1938 static int
1939 eth_igc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1940                    unsigned int n)
1941 {
1942         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1943         struct igc_hw_stats *hw_stats =
1944                         IGC_DEV_PRIVATE_STATS(dev);
1945         unsigned int i;
1946
1947         igc_read_stats_registers(hw, hw_stats);
1948
1949         if (n < IGC_NB_XSTATS)
1950                 return IGC_NB_XSTATS;
1951
1952         /* If this is a reset xstats is NULL, and we have cleared the
1953          * registers by reading them.
1954          */
1955         if (!xstats)
1956                 return 0;
1957
1958         /* Extended stats */
1959         for (i = 0; i < IGC_NB_XSTATS; i++) {
1960                 xstats[i].id = i;
1961                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1962                         rte_igc_stats_strings[i].offset);
1963         }
1964
1965         return IGC_NB_XSTATS;
1966 }
1967
1968 static int
1969 eth_igc_xstats_reset(struct rte_eth_dev *dev)
1970 {
1971         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1972         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1973         struct igc_hw_queue_stats *queue_stats =
1974                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1975
1976         /* Cancel queue status handler for avoid conflict */
1977         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1978
1979         /* HW registers are cleared on read */
1980         igc_reset_queue_stats_register(hw);
1981         igc_read_stats_registers(hw, hw_stats);
1982
1983         /* Reset software totals */
1984         memset(hw_stats, 0, sizeof(*hw_stats));
1985         memset(queue_stats, 0, sizeof(*queue_stats));
1986
1987         /* Restart the queue status handler */
1988         rte_eal_alarm_set(IGC_ALARM_INTERVAL, igc_update_queue_stats_handler,
1989                         dev);
1990
1991         return 0;
1992 }
1993
1994 static int
1995 eth_igc_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1996         struct rte_eth_xstat_name *xstats_names, unsigned int size)
1997 {
1998         unsigned int i;
1999
2000         if (xstats_names == NULL)
2001                 return IGC_NB_XSTATS;
2002
2003         if (size < IGC_NB_XSTATS) {
2004                 PMD_DRV_LOG(ERR, "not enough buffers!");
2005                 return IGC_NB_XSTATS;
2006         }
2007
2008         for (i = 0; i < IGC_NB_XSTATS; i++)
2009                 strlcpy(xstats_names[i].name, rte_igc_stats_strings[i].name,
2010                         sizeof(xstats_names[i].name));
2011
2012         return IGC_NB_XSTATS;
2013 }
2014
2015 static int
2016 eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
2017                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
2018                 unsigned int limit)
2019 {
2020         unsigned int i;
2021
2022         if (!ids)
2023                 return eth_igc_xstats_get_names(dev, xstats_names, limit);
2024
2025         for (i = 0; i < limit; i++) {
2026                 if (ids[i] >= IGC_NB_XSTATS) {
2027                         PMD_DRV_LOG(ERR, "id value isn't valid");
2028                         return -EINVAL;
2029                 }
2030                 strlcpy(xstats_names[i].name,
2031                         rte_igc_stats_strings[ids[i]].name,
2032                         sizeof(xstats_names[i].name));
2033         }
2034         return limit;
2035 }
2036
2037 static int
2038 eth_igc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
2039                 uint64_t *values, unsigned int n)
2040 {
2041         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2042         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
2043         unsigned int i;
2044
2045         igc_read_stats_registers(hw, hw_stats);
2046
2047         if (!ids) {
2048                 if (n < IGC_NB_XSTATS)
2049                         return IGC_NB_XSTATS;
2050
2051                 /* If this is a reset xstats is NULL, and we have cleared the
2052                  * registers by reading them.
2053                  */
2054                 if (!values)
2055                         return 0;
2056
2057                 /* Extended stats */
2058                 for (i = 0; i < IGC_NB_XSTATS; i++)
2059                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2060                                         rte_igc_stats_strings[i].offset);
2061
2062                 return IGC_NB_XSTATS;
2063
2064         } else {
2065                 for (i = 0; i < n; i++) {
2066                         if (ids[i] >= IGC_NB_XSTATS) {
2067                                 PMD_DRV_LOG(ERR, "id value isn't valid");
2068                                 return -EINVAL;
2069                         }
2070                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2071                                         rte_igc_stats_strings[ids[i]].offset);
2072                 }
2073                 return n;
2074         }
2075 }
2076
2077 static int
2078 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
2079                 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx)
2080 {
2081         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
2082
2083         /* check queue id is valid */
2084         if (queue_id >= IGC_QUEUE_PAIRS_NUM) {
2085                 PMD_DRV_LOG(ERR, "queue id(%u) error, max is %u",
2086                         queue_id, IGC_QUEUE_PAIRS_NUM - 1);
2087                 return -EINVAL;
2088         }
2089
2090         /* store the mapping status id */
2091         if (is_rx)
2092                 igc->rxq_stats_map[queue_id] = stat_idx;
2093         else
2094                 igc->txq_stats_map[queue_id] = stat_idx;
2095
2096         return 0;
2097 }
2098
2099 static int
2100 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2101 {
2102         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2103         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2104         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2105         uint32_t vec = IGC_MISC_VEC_ID;
2106
2107         if (rte_intr_allow_others(intr_handle))
2108                 vec = IGC_RX_VEC_START;
2109
2110         uint32_t mask = 1u << (queue_id + vec);
2111
2112         IGC_WRITE_REG(hw, IGC_EIMC, mask);
2113         IGC_WRITE_FLUSH(hw);
2114
2115         return 0;
2116 }
2117
2118 static int
2119 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2120 {
2121         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2122         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2123         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2124         uint32_t vec = IGC_MISC_VEC_ID;
2125
2126         if (rte_intr_allow_others(intr_handle))
2127                 vec = IGC_RX_VEC_START;
2128
2129         uint32_t mask = 1u << (queue_id + vec);
2130
2131         IGC_WRITE_REG(hw, IGC_EIMS, mask);
2132         IGC_WRITE_FLUSH(hw);
2133
2134         rte_intr_enable(intr_handle);
2135
2136         return 0;
2137 }
2138
2139 static int
2140 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2141 {
2142         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2143         uint32_t ctrl;
2144         int tx_pause;
2145         int rx_pause;
2146
2147         fc_conf->pause_time = hw->fc.pause_time;
2148         fc_conf->high_water = hw->fc.high_water;
2149         fc_conf->low_water = hw->fc.low_water;
2150         fc_conf->send_xon = hw->fc.send_xon;
2151         fc_conf->autoneg = hw->mac.autoneg;
2152
2153         /*
2154          * Return rx_pause and tx_pause status according to actual setting of
2155          * the TFCE and RFCE bits in the CTRL register.
2156          */
2157         ctrl = IGC_READ_REG(hw, IGC_CTRL);
2158         if (ctrl & IGC_CTRL_TFCE)
2159                 tx_pause = 1;
2160         else
2161                 tx_pause = 0;
2162
2163         if (ctrl & IGC_CTRL_RFCE)
2164                 rx_pause = 1;
2165         else
2166                 rx_pause = 0;
2167
2168         if (rx_pause && tx_pause)
2169                 fc_conf->mode = RTE_FC_FULL;
2170         else if (rx_pause)
2171                 fc_conf->mode = RTE_FC_RX_PAUSE;
2172         else if (tx_pause)
2173                 fc_conf->mode = RTE_FC_TX_PAUSE;
2174         else
2175                 fc_conf->mode = RTE_FC_NONE;
2176
2177         return 0;
2178 }
2179
2180 static int
2181 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2182 {
2183         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2184         uint32_t rx_buf_size;
2185         uint32_t max_high_water;
2186         uint32_t rctl;
2187         int err;
2188
2189         if (fc_conf->autoneg != hw->mac.autoneg)
2190                 return -ENOTSUP;
2191
2192         rx_buf_size = igc_get_rx_buffer_size(hw);
2193         PMD_DRV_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2194
2195         /* At least reserve one Ethernet frame for watermark */
2196         max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN;
2197         if (fc_conf->high_water > max_high_water ||
2198                 fc_conf->high_water < fc_conf->low_water) {
2199                 PMD_DRV_LOG(ERR,
2200                         "Incorrect high(%u)/low(%u) water value, max is %u",
2201                         fc_conf->high_water, fc_conf->low_water,
2202                         max_high_water);
2203                 return -EINVAL;
2204         }
2205
2206         switch (fc_conf->mode) {
2207         case RTE_FC_NONE:
2208                 hw->fc.requested_mode = igc_fc_none;
2209                 break;
2210         case RTE_FC_RX_PAUSE:
2211                 hw->fc.requested_mode = igc_fc_rx_pause;
2212                 break;
2213         case RTE_FC_TX_PAUSE:
2214                 hw->fc.requested_mode = igc_fc_tx_pause;
2215                 break;
2216         case RTE_FC_FULL:
2217                 hw->fc.requested_mode = igc_fc_full;
2218                 break;
2219         default:
2220                 PMD_DRV_LOG(ERR, "unsupported fc mode: %u", fc_conf->mode);
2221                 return -EINVAL;
2222         }
2223
2224         hw->fc.pause_time     = fc_conf->pause_time;
2225         hw->fc.high_water     = fc_conf->high_water;
2226         hw->fc.low_water      = fc_conf->low_water;
2227         hw->fc.send_xon       = fc_conf->send_xon;
2228
2229         err = igc_setup_link_generic(hw);
2230         if (err == IGC_SUCCESS) {
2231                 /**
2232                  * check if we want to forward MAC frames - driver doesn't have
2233                  * native capability to do that, so we'll write the registers
2234                  * ourselves
2235                  **/
2236                 rctl = IGC_READ_REG(hw, IGC_RCTL);
2237
2238                 /* set or clear MFLCN.PMCF bit depending on configuration */
2239                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2240                         rctl |= IGC_RCTL_PMCF;
2241                 else
2242                         rctl &= ~IGC_RCTL_PMCF;
2243
2244                 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
2245                 IGC_WRITE_FLUSH(hw);
2246
2247                 return 0;
2248         }
2249
2250         PMD_DRV_LOG(ERR, "igc_setup_link_generic = 0x%x", err);
2251         return -EIO;
2252 }
2253
2254 static int
2255 eth_igc_rss_reta_update(struct rte_eth_dev *dev,
2256                         struct rte_eth_rss_reta_entry64 *reta_conf,
2257                         uint16_t reta_size)
2258 {
2259         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2260         uint16_t i;
2261
2262         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2263                 PMD_DRV_LOG(ERR,
2264                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2265                         reta_size, ETH_RSS_RETA_SIZE_128);
2266                 return -EINVAL;
2267         }
2268
2269         RTE_BUILD_BUG_ON(ETH_RSS_RETA_SIZE_128 % IGC_RSS_RDT_REG_SIZE);
2270
2271         /* set redirection table */
2272         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2273                 union igc_rss_reta_reg reta, reg;
2274                 uint16_t idx, shift;
2275                 uint8_t j, mask;
2276
2277                 idx = i / RTE_RETA_GROUP_SIZE;
2278                 shift = i % RTE_RETA_GROUP_SIZE;
2279                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2280                                 IGC_RSS_RDT_REG_SIZE_MASK);
2281
2282                 /* if no need to update the register */
2283                 if (!mask ||
2284                     shift > (RTE_RETA_GROUP_SIZE - IGC_RSS_RDT_REG_SIZE))
2285                         continue;
2286
2287                 /* check mask whether need to read the register value first */
2288                 if (mask == IGC_RSS_RDT_REG_SIZE_MASK)
2289                         reg.dword = 0;
2290                 else
2291                         reg.dword = IGC_READ_REG_LE_VALUE(hw,
2292                                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2293
2294                 /* update the register */
2295                 RTE_BUILD_BUG_ON(sizeof(reta.bytes) != IGC_RSS_RDT_REG_SIZE);
2296                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2297                         if (mask & (1u << j))
2298                                 reta.bytes[j] =
2299                                         (uint8_t)reta_conf[idx].reta[shift + j];
2300                         else
2301                                 reta.bytes[j] = reg.bytes[j];
2302                 }
2303                 IGC_WRITE_REG_LE_VALUE(hw,
2304                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE), reta.dword);
2305         }
2306
2307         return 0;
2308 }
2309
2310 static int
2311 eth_igc_rss_reta_query(struct rte_eth_dev *dev,
2312                        struct rte_eth_rss_reta_entry64 *reta_conf,
2313                        uint16_t reta_size)
2314 {
2315         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2316         uint16_t i;
2317
2318         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2319                 PMD_DRV_LOG(ERR,
2320                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2321                         reta_size, ETH_RSS_RETA_SIZE_128);
2322                 return -EINVAL;
2323         }
2324
2325         RTE_BUILD_BUG_ON(ETH_RSS_RETA_SIZE_128 % IGC_RSS_RDT_REG_SIZE);
2326
2327         /* read redirection table */
2328         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2329                 union igc_rss_reta_reg reta;
2330                 uint16_t idx, shift;
2331                 uint8_t j, mask;
2332
2333                 idx = i / RTE_RETA_GROUP_SIZE;
2334                 shift = i % RTE_RETA_GROUP_SIZE;
2335                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2336                                 IGC_RSS_RDT_REG_SIZE_MASK);
2337
2338                 /* if no need to read register */
2339                 if (!mask ||
2340                     shift > (RTE_RETA_GROUP_SIZE - IGC_RSS_RDT_REG_SIZE))
2341                         continue;
2342
2343                 /* read register and get the queue index */
2344                 RTE_BUILD_BUG_ON(sizeof(reta.bytes) != IGC_RSS_RDT_REG_SIZE);
2345                 reta.dword = IGC_READ_REG_LE_VALUE(hw,
2346                                 IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2347                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2348                         if (mask & (1u << j))
2349                                 reta_conf[idx].reta[shift + j] = reta.bytes[j];
2350                 }
2351         }
2352
2353         return 0;
2354 }
2355
2356 static int
2357 eth_igc_rss_hash_update(struct rte_eth_dev *dev,
2358                         struct rte_eth_rss_conf *rss_conf)
2359 {
2360         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2361         igc_hw_rss_hash_set(hw, rss_conf);
2362         return 0;
2363 }
2364
2365 static int
2366 eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
2367                         struct rte_eth_rss_conf *rss_conf)
2368 {
2369         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2370         uint32_t *hash_key = (uint32_t *)rss_conf->rss_key;
2371         uint32_t mrqc;
2372         uint64_t rss_hf;
2373
2374         if (hash_key != NULL) {
2375                 int i;
2376
2377                 /* if not enough space for store hash key */
2378                 if (rss_conf->rss_key_len != IGC_HKEY_SIZE) {
2379                         PMD_DRV_LOG(ERR,
2380                                 "RSS hash key size %u in parameter doesn't match the hardware hash key size %u",
2381                                 rss_conf->rss_key_len, IGC_HKEY_SIZE);
2382                         return -EINVAL;
2383                 }
2384
2385                 /* read RSS key from register */
2386                 for (i = 0; i < IGC_HKEY_MAX_INDEX; i++)
2387                         hash_key[i] = IGC_READ_REG_LE_VALUE(hw, IGC_RSSRK(i));
2388         }
2389
2390         /* get RSS functions configured in MRQC register */
2391         mrqc = IGC_READ_REG(hw, IGC_MRQC);
2392         if ((mrqc & IGC_MRQC_ENABLE_RSS_4Q) == 0)
2393                 return 0;
2394
2395         rss_hf = 0;
2396         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4)
2397                 rss_hf |= ETH_RSS_IPV4;
2398         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_TCP)
2399                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2400         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6)
2401                 rss_hf |= ETH_RSS_IPV6;
2402         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_EX)
2403                 rss_hf |= ETH_RSS_IPV6_EX;
2404         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP)
2405                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
2406         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP_EX)
2407                 rss_hf |= ETH_RSS_IPV6_TCP_EX;
2408         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_UDP)
2409                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2410         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP)
2411                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
2412         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP_EX)
2413                 rss_hf |= ETH_RSS_IPV6_UDP_EX;
2414
2415         rss_conf->rss_hf |= rss_hf;
2416         return 0;
2417 }
2418
2419 static int
2420 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2421 {
2422         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2423         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2424         uint32_t vfta;
2425         uint32_t vid_idx;
2426         uint32_t vid_bit;
2427
2428         vid_idx = (vlan_id >> IGC_VFTA_ENTRY_SHIFT) & IGC_VFTA_ENTRY_MASK;
2429         vid_bit = 1u << (vlan_id & IGC_VFTA_ENTRY_BIT_SHIFT_MASK);
2430         vfta = shadow_vfta->vfta[vid_idx];
2431         if (on)
2432                 vfta |= vid_bit;
2433         else
2434                 vfta &= ~vid_bit;
2435         IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, vid_idx, vfta);
2436
2437         /* update local VFTA copy */
2438         shadow_vfta->vfta[vid_idx] = vfta;
2439
2440         return 0;
2441 }
2442
2443 static void
2444 igc_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2445 {
2446         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2447         igc_read_reg_check_clear_bits(hw, IGC_RCTL,
2448                         IGC_RCTL_CFIEN | IGC_RCTL_VFE);
2449 }
2450
2451 static void
2452 igc_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2453 {
2454         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2455         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2456         uint32_t reg_val;
2457         int i;
2458
2459         /* Filter Table Enable, CFI not used for packet acceptance */
2460         reg_val = IGC_READ_REG(hw, IGC_RCTL);
2461         reg_val &= ~IGC_RCTL_CFIEN;
2462         reg_val |= IGC_RCTL_VFE;
2463         IGC_WRITE_REG(hw, IGC_RCTL, reg_val);
2464
2465         /* restore VFTA table */
2466         for (i = 0; i < IGC_VFTA_SIZE; i++)
2467                 IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, i, shadow_vfta->vfta[i]);
2468 }
2469
2470 static void
2471 igc_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2472 {
2473         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2474
2475         igc_read_reg_check_clear_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2476 }
2477
2478 static void
2479 igc_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2480 {
2481         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2482
2483         igc_read_reg_check_set_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2484 }
2485
2486 static int
2487 igc_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2488 {
2489         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2490         uint32_t ctrl_ext;
2491
2492         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2493
2494         /* if extend vlan hasn't been enabled */
2495         if ((ctrl_ext & IGC_CTRL_EXT_EXT_VLAN) == 0)
2496                 return 0;
2497
2498         if ((dev->data->dev_conf.rxmode.offloads &
2499                         DEV_RX_OFFLOAD_JUMBO_FRAME) == 0)
2500                 goto write_ext_vlan;
2501
2502         /* Update maximum packet length */
2503         if (dev->data->dev_conf.rxmode.max_rx_pkt_len <
2504                 RTE_ETHER_MIN_MTU + VLAN_TAG_SIZE) {
2505                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, min is %u",
2506                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
2507                         VLAN_TAG_SIZE + RTE_ETHER_MIN_MTU);
2508                 return -EINVAL;
2509         }
2510         dev->data->dev_conf.rxmode.max_rx_pkt_len -= VLAN_TAG_SIZE;
2511         IGC_WRITE_REG(hw, IGC_RLPML,
2512                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
2513
2514 write_ext_vlan:
2515         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_EXT_VLAN);
2516         return 0;
2517 }
2518
2519 static int
2520 igc_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2521 {
2522         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2523         uint32_t ctrl_ext;
2524
2525         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2526
2527         /* if extend vlan has been enabled */
2528         if (ctrl_ext & IGC_CTRL_EXT_EXT_VLAN)
2529                 return 0;
2530
2531         if ((dev->data->dev_conf.rxmode.offloads &
2532                         DEV_RX_OFFLOAD_JUMBO_FRAME) == 0)
2533                 goto write_ext_vlan;
2534
2535         /* Update maximum packet length */
2536         if (dev->data->dev_conf.rxmode.max_rx_pkt_len >
2537                 MAX_RX_JUMBO_FRAME_SIZE - VLAN_TAG_SIZE) {
2538                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, max is %u",
2539                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2540                         VLAN_TAG_SIZE, MAX_RX_JUMBO_FRAME_SIZE);
2541                 return -EINVAL;
2542         }
2543         dev->data->dev_conf.rxmode.max_rx_pkt_len += VLAN_TAG_SIZE;
2544         IGC_WRITE_REG(hw, IGC_RLPML,
2545                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
2546
2547 write_ext_vlan:
2548         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_EXT_VLAN);
2549         return 0;
2550 }
2551
2552 static int
2553 eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2554 {
2555         struct rte_eth_rxmode *rxmode;
2556
2557         rxmode = &dev->data->dev_conf.rxmode;
2558         if (mask & ETH_VLAN_STRIP_MASK) {
2559                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2560                         igc_vlan_hw_strip_enable(dev);
2561                 else
2562                         igc_vlan_hw_strip_disable(dev);
2563         }
2564
2565         if (mask & ETH_VLAN_FILTER_MASK) {
2566                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
2567                         igc_vlan_hw_filter_enable(dev);
2568                 else
2569                         igc_vlan_hw_filter_disable(dev);
2570         }
2571
2572         if (mask & ETH_VLAN_EXTEND_MASK) {
2573                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
2574                         return igc_vlan_hw_extend_enable(dev);
2575                 else
2576                         return igc_vlan_hw_extend_disable(dev);
2577         }
2578
2579         return 0;
2580 }
2581
2582 static int
2583 eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
2584                       enum rte_vlan_type vlan_type,
2585                       uint16_t tpid)
2586 {
2587         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2588         uint32_t reg_val;
2589
2590         /* only outer TPID of double VLAN can be configured*/
2591         if (vlan_type == ETH_VLAN_TYPE_OUTER) {
2592                 reg_val = IGC_READ_REG(hw, IGC_VET);
2593                 reg_val = (reg_val & (~IGC_VET_EXT)) |
2594                         ((uint32_t)tpid << IGC_VET_EXT_SHIFT);
2595                 IGC_WRITE_REG(hw, IGC_VET, reg_val);
2596
2597                 return 0;
2598         }
2599
2600         /* all other TPID values are read-only*/
2601         PMD_DRV_LOG(ERR, "Not supported");
2602         return -ENOTSUP;
2603 }
2604
2605 static int
2606 eth_igc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2607         struct rte_pci_device *pci_dev)
2608 {
2609         PMD_INIT_FUNC_TRACE();
2610         return rte_eth_dev_pci_generic_probe(pci_dev,
2611                 sizeof(struct igc_adapter), eth_igc_dev_init);
2612 }
2613
2614 static int
2615 eth_igc_pci_remove(struct rte_pci_device *pci_dev)
2616 {
2617         PMD_INIT_FUNC_TRACE();
2618         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igc_dev_uninit);
2619 }
2620
2621 static struct rte_pci_driver rte_igc_pmd = {
2622         .id_table = pci_id_igc_map,
2623         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2624         .probe = eth_igc_pci_probe,
2625         .remove = eth_igc_pci_remove,
2626 };
2627
2628 RTE_PMD_REGISTER_PCI(net_igc, rte_igc_pmd);
2629 RTE_PMD_REGISTER_PCI_TABLE(net_igc, pci_id_igc_map);
2630 RTE_PMD_REGISTER_KMOD_DEP(net_igc, "* igb_uio | uio_pci_generic | vfio-pci");