drivers/net/igc/igc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7
   8 #include <rte_string_fns.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_malloc.h>
  14 #include <rte_alarm.h>
  15
  16 #include "igc_logs.h"
  17 #include "igc_txrx.h"
  18
  19 #define IGC_INTEL_VENDOR_ID             0x8086
  20
  21 /*
  22  * The overhead from MTU to max frame size.
  23  * Considering VLAN so tag needs to be counted.
  24  */
  25 #define IGC_ETH_OVERHEAD                (RTE_ETHER_HDR_LEN + \
  26                                         RTE_ETHER_CRC_LEN + VLAN_TAG_SIZE)
  27
  28 #define IGC_FC_PAUSE_TIME               0x0680
  29 #define IGC_LINK_UPDATE_CHECK_TIMEOUT   90  /* 9s */
  30 #define IGC_LINK_UPDATE_CHECK_INTERVAL  100 /* ms */
  31
  32 #define IGC_MISC_VEC_ID                 RTE_INTR_VEC_ZERO_OFFSET
  33 #define IGC_RX_VEC_START                RTE_INTR_VEC_RXTX_OFFSET
  34 #define IGC_MSIX_OTHER_INTR_VEC         0   /* MSI-X other interrupt vector */
  35 #define IGC_FLAG_NEED_LINK_UPDATE       (1u << 0)       /* need update link */
  36
  37 #define IGC_DEFAULT_RX_FREE_THRESH      32
  38
  39 #define IGC_DEFAULT_RX_PTHRESH          8
  40 #define IGC_DEFAULT_RX_HTHRESH          8
  41 #define IGC_DEFAULT_RX_WTHRESH          4
  42
  43 #define IGC_DEFAULT_TX_PTHRESH          8
  44 #define IGC_DEFAULT_TX_HTHRESH          1
  45 #define IGC_DEFAULT_TX_WTHRESH          16
  46
  47 /* MSI-X other interrupt vector */
  48 #define IGC_MSIX_OTHER_INTR_VEC         0
  49
  50 /* External VLAN Enable bit mask */
  51 #define IGC_CTRL_EXT_EXT_VLAN           (1u << 26)
  52
  53 /* External VLAN Ether Type bit mask and shift */
  54 #define IGC_VET_EXT                     0xFFFF0000
  55 #define IGC_VET_EXT_SHIFT               16
  56
  57 /* Per Queue Good Packets Received Count */
  58 #define IGC_PQGPRC(idx)         (0x10010 + 0x100 * (idx))
  59 /* Per Queue Good Octets Received Count */
  60 #define IGC_PQGORC(idx)         (0x10018 + 0x100 * (idx))
  61 /* Per Queue Good Octets Transmitted Count */
  62 #define IGC_PQGOTC(idx)         (0x10034 + 0x100 * (idx))
  63 /* Per Queue Multicast Packets Received Count */
  64 #define IGC_PQMPRC(idx)         (0x10038 + 0x100 * (idx))
  65 /* Transmit Queue Drop Packet Count */
  66 #define IGC_TQDPC(idx)          (0xe030 + 0x40 * (idx))
  67
  68 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
  69 #define U32_0_IN_U64            0       /* lower bytes of u64 */
  70 #define U32_1_IN_U64            1       /* higher bytes of u64 */
  71 #else
  72 #define U32_0_IN_U64            1
  73 #define U32_1_IN_U64            0
  74 #endif
  75
  76 #define IGC_ALARM_INTERVAL      8000000u
  77 /* us, about 13.6s some per-queue registers will wrap around back to 0. */
  78
  79 static const struct rte_eth_desc_lim rx_desc_lim = {
  80         .nb_max = IGC_MAX_RXD,
  81         .nb_min = IGC_MIN_RXD,
  82         .nb_align = IGC_RXD_ALIGN,
  83 };
  84
  85 static const struct rte_eth_desc_lim tx_desc_lim = {
  86         .nb_max = IGC_MAX_TXD,
  87         .nb_min = IGC_MIN_TXD,
  88         .nb_align = IGC_TXD_ALIGN,
  89         .nb_seg_max = IGC_TX_MAX_SEG,
  90         .nb_mtu_seg_max = IGC_TX_MAX_MTU_SEG,
  91 };
  92
  93 static const struct rte_pci_id pci_id_igc_map[] = {
  94         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_LM) },
  95         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_V)  },
  96         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_I)  },
  97         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_K)  },
  98         { .vendor_id = 0, /* sentinel */ },
  99 };
 100
 101 /* store statistics names and its offset in stats structure */
 102 struct rte_igc_xstats_name_off {
 103         char name[RTE_ETH_XSTATS_NAME_SIZE];
 104         unsigned int offset;
 105 };
 106
 107 static const struct rte_igc_xstats_name_off rte_igc_stats_strings[] = {
 108         {"rx_crc_errors", offsetof(struct igc_hw_stats, crcerrs)},
 109         {"rx_align_errors", offsetof(struct igc_hw_stats, algnerrc)},
 110         {"rx_errors", offsetof(struct igc_hw_stats, rxerrc)},
 111         {"rx_missed_packets", offsetof(struct igc_hw_stats, mpc)},
 112         {"tx_single_collision_packets", offsetof(struct igc_hw_stats, scc)},
 113         {"tx_multiple_collision_packets", offsetof(struct igc_hw_stats, mcc)},
 114         {"tx_excessive_collision_packets", offsetof(struct igc_hw_stats,
 115                 ecol)},
 116         {"tx_late_collisions", offsetof(struct igc_hw_stats, latecol)},
 117         {"tx_total_collisions", offsetof(struct igc_hw_stats, colc)},
 118         {"tx_deferred_packets", offsetof(struct igc_hw_stats, dc)},
 119         {"tx_no_carrier_sense_packets", offsetof(struct igc_hw_stats, tncrs)},
 120         {"tx_discarded_packets", offsetof(struct igc_hw_stats, htdpmc)},
 121         {"rx_length_errors", offsetof(struct igc_hw_stats, rlec)},
 122         {"rx_xon_packets", offsetof(struct igc_hw_stats, xonrxc)},
 123         {"tx_xon_packets", offsetof(struct igc_hw_stats, xontxc)},
 124         {"rx_xoff_packets", offsetof(struct igc_hw_stats, xoffrxc)},
 125         {"tx_xoff_packets", offsetof(struct igc_hw_stats, xofftxc)},
 126         {"rx_flow_control_unsupported_packets", offsetof(struct igc_hw_stats,
 127                 fcruc)},
 128         {"rx_size_64_packets", offsetof(struct igc_hw_stats, prc64)},
 129         {"rx_size_65_to_127_packets", offsetof(struct igc_hw_stats, prc127)},
 130         {"rx_size_128_to_255_packets", offsetof(struct igc_hw_stats, prc255)},
 131         {"rx_size_256_to_511_packets", offsetof(struct igc_hw_stats, prc511)},
 132         {"rx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 133                 prc1023)},
 134         {"rx_size_1024_to_max_packets", offsetof(struct igc_hw_stats,
 135                 prc1522)},
 136         {"rx_broadcast_packets", offsetof(struct igc_hw_stats, bprc)},
 137         {"rx_multicast_packets", offsetof(struct igc_hw_stats, mprc)},
 138         {"rx_undersize_errors", offsetof(struct igc_hw_stats, ruc)},
 139         {"rx_fragment_errors", offsetof(struct igc_hw_stats, rfc)},
 140         {"rx_oversize_errors", offsetof(struct igc_hw_stats, roc)},
 141         {"rx_jabber_errors", offsetof(struct igc_hw_stats, rjc)},
 142         {"rx_no_buffers", offsetof(struct igc_hw_stats, rnbc)},
 143         {"rx_management_packets", offsetof(struct igc_hw_stats, mgprc)},
 144         {"rx_management_dropped", offsetof(struct igc_hw_stats, mgpdc)},
 145         {"tx_management_packets", offsetof(struct igc_hw_stats, mgptc)},
 146         {"rx_total_packets", offsetof(struct igc_hw_stats, tpr)},
 147         {"tx_total_packets", offsetof(struct igc_hw_stats, tpt)},
 148         {"rx_total_bytes", offsetof(struct igc_hw_stats, tor)},
 149         {"tx_total_bytes", offsetof(struct igc_hw_stats, tot)},
 150         {"tx_size_64_packets", offsetof(struct igc_hw_stats, ptc64)},
 151         {"tx_size_65_to_127_packets", offsetof(struct igc_hw_stats, ptc127)},
 152         {"tx_size_128_to_255_packets", offsetof(struct igc_hw_stats, ptc255)},
 153         {"tx_size_256_to_511_packets", offsetof(struct igc_hw_stats, ptc511)},
 154         {"tx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 155                 ptc1023)},
 156         {"tx_size_1023_to_max_packets", offsetof(struct igc_hw_stats,
 157                 ptc1522)},
 158         {"tx_multicast_packets", offsetof(struct igc_hw_stats, mptc)},
 159         {"tx_broadcast_packets", offsetof(struct igc_hw_stats, bptc)},
 160         {"tx_tso_packets", offsetof(struct igc_hw_stats, tsctc)},
 161         {"rx_sent_to_host_packets", offsetof(struct igc_hw_stats, rpthc)},
 162         {"tx_sent_by_host_packets", offsetof(struct igc_hw_stats, hgptc)},
 163         {"interrupt_assert_count", offsetof(struct igc_hw_stats, iac)},
 164         {"rx_descriptor_lower_threshold",
 165                 offsetof(struct igc_hw_stats, icrxdmtc)},
 166 };
 167
 168 #define IGC_NB_XSTATS (sizeof(rte_igc_stats_strings) / \
 169                 sizeof(rte_igc_stats_strings[0]))
 170
 171 static int eth_igc_configure(struct rte_eth_dev *dev);
 172 static int eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
 173 static void eth_igc_stop(struct rte_eth_dev *dev);
 174 static int eth_igc_start(struct rte_eth_dev *dev);
 175 static int eth_igc_set_link_up(struct rte_eth_dev *dev);
 176 static int eth_igc_set_link_down(struct rte_eth_dev *dev);
 177 static void eth_igc_close(struct rte_eth_dev *dev);
 178 static int eth_igc_reset(struct rte_eth_dev *dev);
 179 static int eth_igc_promiscuous_enable(struct rte_eth_dev *dev);
 180 static int eth_igc_promiscuous_disable(struct rte_eth_dev *dev);
 181 static int eth_igc_fw_version_get(struct rte_eth_dev *dev,
 182                                 char *fw_version, size_t fw_size);
 183 static int eth_igc_infos_get(struct rte_eth_dev *dev,
 184                         struct rte_eth_dev_info *dev_info);
 185 static int eth_igc_led_on(struct rte_eth_dev *dev);
 186 static int eth_igc_led_off(struct rte_eth_dev *dev);
 187 static const uint32_t *eth_igc_supported_ptypes_get(struct rte_eth_dev *dev);
 188 static int eth_igc_rar_set(struct rte_eth_dev *dev,
 189                 struct rte_ether_addr *mac_addr, uint32_t index, uint32_t pool);
 190 static void eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 191 static int eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
 192                         struct rte_ether_addr *addr);
 193 static int eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
 194                          struct rte_ether_addr *mc_addr_set,
 195                          uint32_t nb_mc_addr);
 196 static int eth_igc_allmulticast_enable(struct rte_eth_dev *dev);
 197 static int eth_igc_allmulticast_disable(struct rte_eth_dev *dev);
 198 static int eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 199 static int eth_igc_stats_get(struct rte_eth_dev *dev,
 200                         struct rte_eth_stats *rte_stats);
 201 static int eth_igc_xstats_get(struct rte_eth_dev *dev,
 202                         struct rte_eth_xstat *xstats, unsigned int n);
 203 static int eth_igc_xstats_get_by_id(struct rte_eth_dev *dev,
 204                                 const uint64_t *ids,
 205                                 uint64_t *values, unsigned int n);
 206 static int eth_igc_xstats_get_names(struct rte_eth_dev *dev,
 207                                 struct rte_eth_xstat_name *xstats_names,
 208                                 unsigned int size);
 209 static int eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 210                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
 211                 unsigned int limit);
 212 static int eth_igc_xstats_reset(struct rte_eth_dev *dev);
 213 static int
 214 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
 215         uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx);
 216 static int
 217 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 218 static int
 219 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 220 static int
 221 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 222 static int
 223 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 224 static int eth_igc_rss_reta_update(struct rte_eth_dev *dev,
 225                         struct rte_eth_rss_reta_entry64 *reta_conf,
 226                         uint16_t reta_size);
 227 static int eth_igc_rss_reta_query(struct rte_eth_dev *dev,
 228                        struct rte_eth_rss_reta_entry64 *reta_conf,
 229                        uint16_t reta_size);
 230 static int eth_igc_rss_hash_update(struct rte_eth_dev *dev,
 231                         struct rte_eth_rss_conf *rss_conf);
 232 static int eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
 233                         struct rte_eth_rss_conf *rss_conf);
 234 static int
 235 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 236 static int eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 237 static int eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
 238                       enum rte_vlan_type vlan_type, uint16_t tpid);
 239
 240 static const struct eth_dev_ops eth_igc_ops = {
 241         .dev_configure          = eth_igc_configure,
 242         .link_update            = eth_igc_link_update,
 243         .dev_stop               = eth_igc_stop,
 244         .dev_start              = eth_igc_start,
 245         .dev_close              = eth_igc_close,
 246         .dev_reset              = eth_igc_reset,
 247         .dev_set_link_up        = eth_igc_set_link_up,
 248         .dev_set_link_down      = eth_igc_set_link_down,
 249         .promiscuous_enable     = eth_igc_promiscuous_enable,
 250         .promiscuous_disable    = eth_igc_promiscuous_disable,
 251         .allmulticast_enable    = eth_igc_allmulticast_enable,
 252         .allmulticast_disable   = eth_igc_allmulticast_disable,
 253         .fw_version_get         = eth_igc_fw_version_get,
 254         .dev_infos_get          = eth_igc_infos_get,
 255         .dev_led_on             = eth_igc_led_on,
 256         .dev_led_off            = eth_igc_led_off,
 257         .dev_supported_ptypes_get = eth_igc_supported_ptypes_get,
 258         .mtu_set                = eth_igc_mtu_set,
 259         .mac_addr_add           = eth_igc_rar_set,
 260         .mac_addr_remove        = eth_igc_rar_clear,
 261         .mac_addr_set           = eth_igc_default_mac_addr_set,
 262         .set_mc_addr_list       = eth_igc_set_mc_addr_list,
 263
 264         .rx_queue_setup         = eth_igc_rx_queue_setup,
 265         .rx_queue_release       = eth_igc_rx_queue_release,
 266         .rx_queue_count         = eth_igc_rx_queue_count,
 267         .rx_descriptor_done     = eth_igc_rx_descriptor_done,
 268         .rx_descriptor_status   = eth_igc_rx_descriptor_status,
 269         .tx_descriptor_status   = eth_igc_tx_descriptor_status,
 270         .tx_queue_setup         = eth_igc_tx_queue_setup,
 271         .tx_queue_release       = eth_igc_tx_queue_release,
 272         .tx_done_cleanup        = eth_igc_tx_done_cleanup,
 273         .rxq_info_get           = eth_igc_rxq_info_get,
 274         .txq_info_get           = eth_igc_txq_info_get,
 275         .stats_get              = eth_igc_stats_get,
 276         .xstats_get             = eth_igc_xstats_get,
 277         .xstats_get_by_id       = eth_igc_xstats_get_by_id,
 278         .xstats_get_names_by_id = eth_igc_xstats_get_names_by_id,
 279         .xstats_get_names       = eth_igc_xstats_get_names,
 280         .stats_reset            = eth_igc_xstats_reset,
 281         .xstats_reset           = eth_igc_xstats_reset,
 282         .queue_stats_mapping_set = eth_igc_queue_stats_mapping_set,
 283         .rx_queue_intr_enable   = eth_igc_rx_queue_intr_enable,
 284         .rx_queue_intr_disable  = eth_igc_rx_queue_intr_disable,
 285         .flow_ctrl_get          = eth_igc_flow_ctrl_get,
 286         .flow_ctrl_set          = eth_igc_flow_ctrl_set,
 287         .reta_update            = eth_igc_rss_reta_update,
 288         .reta_query             = eth_igc_rss_reta_query,
 289         .rss_hash_update        = eth_igc_rss_hash_update,
 290         .rss_hash_conf_get      = eth_igc_rss_hash_conf_get,
 291         .vlan_filter_set        = eth_igc_vlan_filter_set,
 292         .vlan_offload_set       = eth_igc_vlan_offload_set,
 293         .vlan_tpid_set          = eth_igc_vlan_tpid_set,
 294         .vlan_strip_queue_set   = eth_igc_vlan_strip_queue_set,
 295 };
 296
 297 /*
 298  * multiple queue mode checking
 299  */
 300 static int
 301 igc_check_mq_mode(struct rte_eth_dev *dev)
 302 {
 303         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 304         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
 305
 306         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 307                 PMD_INIT_LOG(ERR, "SRIOV is not supported.");
 308                 return -EINVAL;
 309         }
 310
 311         if (rx_mq_mode != ETH_MQ_RX_NONE &&
 312                 rx_mq_mode != ETH_MQ_RX_RSS) {
 313                 /* RSS together with VMDq not supported*/
 314                 PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
 315                                 rx_mq_mode);
 316                 return -EINVAL;
 317         }
 318
 319         /* To no break software that set invalid mode, only display
 320          * warning if invalid mode is used.
 321          */
 322         if (tx_mq_mode != ETH_MQ_TX_NONE)
 323                 PMD_INIT_LOG(WARNING,
 324                         "TX mode %d is not supported. Due to meaningless in this driver, just ignore",
 325                         tx_mq_mode);
 326
 327         return 0;
 328 }
 329
 330 static int
 331 eth_igc_configure(struct rte_eth_dev *dev)
 332 {
 333         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 334         int ret;
 335
 336         PMD_INIT_FUNC_TRACE();
 337
 338         ret  = igc_check_mq_mode(dev);
 339         if (ret != 0)
 340                 return ret;
 341
 342         intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 343         return 0;
 344 }
 345
 346 static int
 347 eth_igc_set_link_up(struct rte_eth_dev *dev)
 348 {
 349         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 350
 351         if (hw->phy.media_type == igc_media_type_copper)
 352                 igc_power_up_phy(hw);
 353         else
 354                 igc_power_up_fiber_serdes_link(hw);
 355         return 0;
 356 }
 357
 358 static int
 359 eth_igc_set_link_down(struct rte_eth_dev *dev)
 360 {
 361         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 362
 363         if (hw->phy.media_type == igc_media_type_copper)
 364                 igc_power_down_phy(hw);
 365         else
 366                 igc_shutdown_fiber_serdes_link(hw);
 367         return 0;
 368 }
 369
 370 /*
 371  * disable other interrupt
 372  */
 373 static void
 374 igc_intr_other_disable(struct rte_eth_dev *dev)
 375 {
 376         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 377         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 378         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 379
 380         if (rte_intr_allow_others(intr_handle) &&
 381                 dev->data->dev_conf.intr_conf.lsc) {
 382                 IGC_WRITE_REG(hw, IGC_EIMC, 1u << IGC_MSIX_OTHER_INTR_VEC);
 383         }
 384
 385         IGC_WRITE_REG(hw, IGC_IMC, ~0);
 386         IGC_WRITE_FLUSH(hw);
 387 }
 388
 389 /*
 390  * enable other interrupt
 391  */
 392 static inline void
 393 igc_intr_other_enable(struct rte_eth_dev *dev)
 394 {
 395         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 396         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 397         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 398         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 399
 400         if (rte_intr_allow_others(intr_handle) &&
 401                 dev->data->dev_conf.intr_conf.lsc) {
 402                 IGC_WRITE_REG(hw, IGC_EIMS, 1u << IGC_MSIX_OTHER_INTR_VEC);
 403         }
 404
 405         IGC_WRITE_REG(hw, IGC_IMS, intr->mask);
 406         IGC_WRITE_FLUSH(hw);
 407 }
 408
 409 /*
 410  * It reads ICR and gets interrupt causes, check it and set a bit flag
 411  * to update link status.
 412  */
 413 static void
 414 eth_igc_interrupt_get_status(struct rte_eth_dev *dev)
 415 {
 416         uint32_t icr;
 417         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 418         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 419
 420         /* read-on-clear nic registers here */
 421         icr = IGC_READ_REG(hw, IGC_ICR);
 422
 423         intr->flags = 0;
 424         if (icr & IGC_ICR_LSC)
 425                 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 426 }
 427
 428 /* return 0 means link status changed, -1 means not changed */
 429 static int
 430 eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 431 {
 432         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 433         struct rte_eth_link link;
 434         int link_check, count;
 435
 436         link_check = 0;
 437         hw->mac.get_link_status = 1;
 438
 439         /* possible wait-to-complete in up to 9 seconds */
 440         for (count = 0; count < IGC_LINK_UPDATE_CHECK_TIMEOUT; count++) {
 441                 /* Read the real link status */
 442                 switch (hw->phy.media_type) {
 443                 case igc_media_type_copper:
 444                         /* Do the work to read phy */
 445                         igc_check_for_link(hw);
 446                         link_check = !hw->mac.get_link_status;
 447                         break;
 448
 449                 case igc_media_type_fiber:
 450                         igc_check_for_link(hw);
 451                         link_check = (IGC_READ_REG(hw, IGC_STATUS) &
 452                                       IGC_STATUS_LU);
 453                         break;
 454
 455                 case igc_media_type_internal_serdes:
 456                         igc_check_for_link(hw);
 457                         link_check = hw->mac.serdes_has_link;
 458                         break;
 459
 460                 default:
 461                         break;
 462                 }
 463                 if (link_check || wait_to_complete == 0)
 464                         break;
 465                 rte_delay_ms(IGC_LINK_UPDATE_CHECK_INTERVAL);
 466         }
 467         memset(&link, 0, sizeof(link));
 468
 469         /* Now we check if a transition has happened */
 470         if (link_check) {
 471                 uint16_t duplex, speed;
 472                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
 473                 link.link_duplex = (duplex == FULL_DUPLEX) ?
 474                                 ETH_LINK_FULL_DUPLEX :
 475                                 ETH_LINK_HALF_DUPLEX;
 476                 link.link_speed = speed;
 477                 link.link_status = ETH_LINK_UP;
 478                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 479                                 ETH_LINK_SPEED_FIXED);
 480
 481                 if (speed == SPEED_2500) {
 482                         uint32_t tipg = IGC_READ_REG(hw, IGC_TIPG);
 483                         if ((tipg & IGC_TIPG_IPGT_MASK) != 0x0b) {
 484                                 tipg &= ~IGC_TIPG_IPGT_MASK;
 485                                 tipg |= 0x0b;
 486                                 IGC_WRITE_REG(hw, IGC_TIPG, tipg);
 487                         }
 488                 }
 489         } else {
 490                 link.link_speed = 0;
 491                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
 492                 link.link_status = ETH_LINK_DOWN;
 493                 link.link_autoneg = ETH_LINK_FIXED;
 494         }
 495
 496         return rte_eth_linkstatus_set(dev, &link);
 497 }
 498
 499 /*
 500  * It executes link_update after knowing an interrupt is present.
 501  */
 502 static void
 503 eth_igc_interrupt_action(struct rte_eth_dev *dev)
 504 {
 505         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 506         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 507         struct rte_eth_link link;
 508         int ret;
 509
 510         if (intr->flags & IGC_FLAG_NEED_LINK_UPDATE) {
 511                 intr->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
 512
 513                 /* set get_link_status to check register later */
 514                 ret = eth_igc_link_update(dev, 0);
 515
 516                 /* check if link has changed */
 517                 if (ret < 0)
 518                         return;
 519
 520                 rte_eth_linkstatus_get(dev, &link);
 521                 if (link.link_status)
 522                         PMD_DRV_LOG(INFO,
 523                                 " Port %d: Link Up - speed %u Mbps - %s",
 524                                 dev->data->port_id,
 525                                 (unsigned int)link.link_speed,
 526                                 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
 527                                 "full-duplex" : "half-duplex");
 528                 else
 529                         PMD_DRV_LOG(INFO, " Port %d: Link Down",
 530                                 dev->data->port_id);
 531
 532                 PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
 533                                 pci_dev->addr.domain,
 534                                 pci_dev->addr.bus,
 535                                 pci_dev->addr.devid,
 536                                 pci_dev->addr.function);
 537                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
 538                                 NULL);
 539         }
 540 }
 541
 542 /*
 543  * Interrupt handler which shall be registered at first.
 544  *
 545  * @handle
 546  *  Pointer to interrupt handle.
 547  * @param
 548  *  The address of parameter (struct rte_eth_dev *) registered before.
 549  */
 550 static void
 551 eth_igc_interrupt_handler(void *param)
 552 {
 553         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 554
 555         eth_igc_interrupt_get_status(dev);
 556         eth_igc_interrupt_action(dev);
 557 }
 558
 559 static void igc_read_queue_stats_register(struct rte_eth_dev *dev);
 560
 561 /*
 562  * Update the queue status every IGC_ALARM_INTERVAL time.
 563  * @param
 564  *  The address of parameter (struct rte_eth_dev *) registered before.
 565  */
 566 static void
 567 igc_update_queue_stats_handler(void *param)
 568 {
 569         struct rte_eth_dev *dev = param;
 570         igc_read_queue_stats_register(dev);
 571         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
 572                         igc_update_queue_stats_handler, dev);
 573 }
 574
 575 /*
 576  * rx,tx enable/disable
 577  */
 578 static void
 579 eth_igc_rxtx_control(struct rte_eth_dev *dev, bool enable)
 580 {
 581         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 582         uint32_t tctl, rctl;
 583
 584         tctl = IGC_READ_REG(hw, IGC_TCTL);
 585         rctl = IGC_READ_REG(hw, IGC_RCTL);
 586
 587         if (enable) {
 588                 /* enable Tx/Rx */
 589                 tctl |= IGC_TCTL_EN;
 590                 rctl |= IGC_RCTL_EN;
 591         } else {
 592                 /* disable Tx/Rx */
 593                 tctl &= ~IGC_TCTL_EN;
 594                 rctl &= ~IGC_RCTL_EN;
 595         }
 596         IGC_WRITE_REG(hw, IGC_TCTL, tctl);
 597         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
 598         IGC_WRITE_FLUSH(hw);
 599 }
 600
 601 /*
 602  *  This routine disables all traffic on the adapter by issuing a
 603  *  global reset on the MAC.
 604  */
 605 static void
 606 eth_igc_stop(struct rte_eth_dev *dev)
 607 {
 608         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 609         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 610         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 611         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 612         struct rte_eth_link link;
 613
 614         adapter->stopped = 1;
 615
 616         /* disable receive and transmit */
 617         eth_igc_rxtx_control(dev, false);
 618
 619         /* disable all MSI-X interrupts */
 620         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 621         IGC_WRITE_FLUSH(hw);
 622
 623         /* clear all MSI-X interrupts */
 624         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 625
 626         igc_intr_other_disable(dev);
 627
 628         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
 629
 630         /* disable intr eventfd mapping */
 631         rte_intr_disable(intr_handle);
 632
 633         igc_reset_hw(hw);
 634
 635         /* disable all wake up */
 636         IGC_WRITE_REG(hw, IGC_WUC, 0);
 637
 638         /* Set bit for Go Link disconnect */
 639         igc_read_reg_check_set_bits(hw, IGC_82580_PHY_POWER_MGMT,
 640                         IGC_82580_PM_GO_LINKD);
 641
 642         /* Power down the phy. Needed to make the link go Down */
 643         eth_igc_set_link_down(dev);
 644
 645         igc_dev_clear_queues(dev);
 646
 647         /* clear the recorded link status */
 648         memset(&link, 0, sizeof(link));
 649         rte_eth_linkstatus_set(dev, &link);
 650
 651         if (!rte_intr_allow_others(intr_handle))
 652                 /* resume to the default handler */
 653                 rte_intr_callback_register(intr_handle,
 654                                            eth_igc_interrupt_handler,
 655                                            (void *)dev);
 656
 657         /* Clean datapath event and queue/vec mapping */
 658         rte_intr_efd_disable(intr_handle);
 659         if (intr_handle->intr_vec != NULL) {
 660                 rte_free(intr_handle->intr_vec);
 661                 intr_handle->intr_vec = NULL;
 662         }
 663 }
 664
 665 /*
 666  * write interrupt vector allocation register
 667  * @hw
 668  *  board private structure
 669  * @queue_index
 670  *  queue index, valid 0,1,2,3
 671  * @tx
 672  *  tx:1, rx:0
 673  * @msix_vector
 674  *  msix-vector, valid 0,1,2,3,4
 675  */
 676 static void
 677 igc_write_ivar(struct igc_hw *hw, uint8_t queue_index,
 678                 bool tx, uint8_t msix_vector)
 679 {
 680         uint8_t offset = 0;
 681         uint8_t reg_index = queue_index >> 1;
 682         uint32_t val;
 683
 684         /*
 685          * IVAR(0)
 686          * bit31...24   bit23...16      bit15...8       bit7...0
 687          * TX1          RX1             TX0             RX0
 688          *
 689          * IVAR(1)
 690          * bit31...24   bit23...16      bit15...8       bit7...0
 691          * TX3          RX3             TX2             RX2
 692          */
 693
 694         if (tx)
 695                 offset = 8;
 696
 697         if (queue_index & 1)
 698                 offset += 16;
 699
 700         val = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, reg_index);
 701
 702         /* clear bits */
 703         val &= ~((uint32_t)0xFF << offset);
 704
 705         /* write vector and valid bit */
 706         val |= (uint32_t)(msix_vector | IGC_IVAR_VALID) << offset;
 707
 708         IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, reg_index, val);
 709 }
 710
 711 /* Sets up the hardware to generate MSI-X interrupts properly
 712  * @hw
 713  *  board private structure
 714  */
 715 static void
 716 igc_configure_msix_intr(struct rte_eth_dev *dev)
 717 {
 718         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 719         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 720         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 721
 722         uint32_t intr_mask;
 723         uint32_t vec = IGC_MISC_VEC_ID;
 724         uint32_t base = IGC_MISC_VEC_ID;
 725         uint32_t misc_shift = 0;
 726         int i;
 727
 728         /* won't configure msix register if no mapping is done
 729          * between intr vector and event fd
 730          */
 731         if (!rte_intr_dp_is_en(intr_handle))
 732                 return;
 733
 734         if (rte_intr_allow_others(intr_handle)) {
 735                 base = IGC_RX_VEC_START;
 736                 vec = base;
 737                 misc_shift = 1;
 738         }
 739
 740         /* turn on MSI-X capability first */
 741         IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE |
 742                                 IGC_GPIE_PBA | IGC_GPIE_EIAME |
 743                                 IGC_GPIE_NSICR);
 744         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
 745                 misc_shift;
 746
 747         if (dev->data->dev_conf.intr_conf.lsc)
 748                 intr_mask |= (1u << IGC_MSIX_OTHER_INTR_VEC);
 749
 750         /* enable msix auto-clear */
 751         igc_read_reg_check_set_bits(hw, IGC_EIAC, intr_mask);
 752
 753         /* set other cause interrupt vector */
 754         igc_read_reg_check_set_bits(hw, IGC_IVAR_MISC,
 755                 (uint32_t)(IGC_MSIX_OTHER_INTR_VEC | IGC_IVAR_VALID) << 8);
 756
 757         /* enable auto-mask */
 758         igc_read_reg_check_set_bits(hw, IGC_EIAM, intr_mask);
 759
 760         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 761                 igc_write_ivar(hw, i, 0, vec);
 762                 intr_handle->intr_vec[i] = vec;
 763                 if (vec < base + intr_handle->nb_efd - 1)
 764                         vec++;
 765         }
 766
 767         IGC_WRITE_FLUSH(hw);
 768 }
 769
 770 /**
 771  * It enables the interrupt mask and then enable the interrupt.
 772  *
 773  * @dev
 774  *  Pointer to struct rte_eth_dev.
 775  * @on
 776  *  Enable or Disable
 777  */
 778 static void
 779 igc_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
 780 {
 781         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 782
 783         if (on)
 784                 intr->mask |= IGC_ICR_LSC;
 785         else
 786                 intr->mask &= ~IGC_ICR_LSC;
 787 }
 788
 789 /*
 790  * It enables the interrupt.
 791  * It will be called once only during nic initialized.
 792  */
 793 static void
 794 igc_rxq_interrupt_setup(struct rte_eth_dev *dev)
 795 {
 796         uint32_t mask;
 797         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 798         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 799         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 800         int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0;
 801
 802         /* won't configure msix register if no mapping is done
 803          * between intr vector and event fd
 804          */
 805         if (!rte_intr_dp_is_en(intr_handle))
 806                 return;
 807
 808         mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << misc_shift;
 809         IGC_WRITE_REG(hw, IGC_EIMS, mask);
 810 }
 811
 812 /*
 813  *  Get hardware rx-buffer size.
 814  */
 815 static inline int
 816 igc_get_rx_buffer_size(struct igc_hw *hw)
 817 {
 818         return (IGC_READ_REG(hw, IGC_RXPBS) & 0x3f) << 10;
 819 }
 820
 821 /*
 822  * igc_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
 823  * For ASF and Pass Through versions of f/w this means
 824  * that the driver is loaded.
 825  */
 826 static void
 827 igc_hw_control_acquire(struct igc_hw *hw)
 828 {
 829         uint32_t ctrl_ext;
 830
 831         /* Let firmware know the driver has taken over */
 832         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 833         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 834 }
 835
 836 /*
 837  * igc_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
 838  * For ASF and Pass Through versions of f/w this means that the
 839  * driver is no longer loaded.
 840  */
 841 static void
 842 igc_hw_control_release(struct igc_hw *hw)
 843 {
 844         uint32_t ctrl_ext;
 845
 846         /* Let firmware taken over control of h/w */
 847         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 848         IGC_WRITE_REG(hw, IGC_CTRL_EXT,
 849                         ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 850 }
 851
 852 static int
 853 igc_hardware_init(struct igc_hw *hw)
 854 {
 855         uint32_t rx_buf_size;
 856         int diag;
 857
 858         /* Let the firmware know the OS is in control */
 859         igc_hw_control_acquire(hw);
 860
 861         /* Issue a global reset */
 862         igc_reset_hw(hw);
 863
 864         /* disable all wake up */
 865         IGC_WRITE_REG(hw, IGC_WUC, 0);
 866
 867         /*
 868          * Hardware flow control
 869          * - High water mark should allow for at least two standard size (1518)
 870          *   frames to be received after sending an XOFF.
 871          * - Low water mark works best when it is very near the high water mark.
 872          *   This allows the receiver to restart by sending XON when it has
 873          *   drained a bit. Here we use an arbitrary value of 1500 which will
 874          *   restart after one full frame is pulled from the buffer. There
 875          *   could be several smaller frames in the buffer and if so they will
 876          *   not trigger the XON until their total number reduces the buffer
 877          *   by 1500.
 878          */
 879         rx_buf_size = igc_get_rx_buffer_size(hw);
 880         hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2);
 881         hw->fc.low_water = hw->fc.high_water - 1500;
 882         hw->fc.pause_time = IGC_FC_PAUSE_TIME;
 883         hw->fc.send_xon = 1;
 884         hw->fc.requested_mode = igc_fc_full;
 885
 886         diag = igc_init_hw(hw);
 887         if (diag < 0)
 888                 return diag;
 889
 890         igc_get_phy_info(hw);
 891         igc_check_for_link(hw);
 892
 893         return 0;
 894 }
 895
 896 static int
 897 eth_igc_start(struct rte_eth_dev *dev)
 898 {
 899         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 900         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 901         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 902         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 903         uint32_t *speeds;
 904         int ret;
 905
 906         PMD_INIT_FUNC_TRACE();
 907
 908         /* disable all MSI-X interrupts */
 909         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 910         IGC_WRITE_FLUSH(hw);
 911
 912         /* clear all MSI-X interrupts */
 913         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 914
 915         /* disable uio/vfio intr/eventfd mapping */
 916         if (!adapter->stopped)
 917                 rte_intr_disable(intr_handle);
 918
 919         /* Power up the phy. Needed to make the link go Up */
 920         eth_igc_set_link_up(dev);
 921
 922         /* Put the address into the Receive Address Array */
 923         igc_rar_set(hw, hw->mac.addr, 0);
 924
 925         /* Initialize the hardware */
 926         if (igc_hardware_init(hw)) {
 927                 PMD_DRV_LOG(ERR, "Unable to initialize the hardware");
 928                 return -EIO;
 929         }
 930         adapter->stopped = 0;
 931
 932         /* check and configure queue intr-vector mapping */
 933         if (rte_intr_cap_multiple(intr_handle) &&
 934                 dev->data->dev_conf.intr_conf.rxq) {
 935                 uint32_t intr_vector = dev->data->nb_rx_queues;
 936                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 937                         return -1;
 938         }
 939
 940         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 941                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
 942                         dev->data->nb_rx_queues * sizeof(int), 0);
 943                 if (intr_handle->intr_vec == NULL) {
 944                         PMD_DRV_LOG(ERR,
 945                                 "Failed to allocate %d rx_queues intr_vec",
 946                                 dev->data->nb_rx_queues);
 947                         return -ENOMEM;
 948                 }
 949         }
 950
 951         /* configure msix for rx interrupt */
 952         igc_configure_msix_intr(dev);
 953
 954         igc_tx_init(dev);
 955
 956         /* This can fail when allocating mbufs for descriptor rings */
 957         ret = igc_rx_init(dev);
 958         if (ret) {
 959                 PMD_DRV_LOG(ERR, "Unable to initialize RX hardware");
 960                 igc_dev_clear_queues(dev);
 961                 return ret;
 962         }
 963
 964         igc_clear_hw_cntrs_base_generic(hw);
 965
 966         /* VLAN Offload Settings */
 967         eth_igc_vlan_offload_set(dev,
 968                 ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
 969                 ETH_VLAN_EXTEND_MASK);
 970
 971         /* Setup link speed and duplex */
 972         speeds = &dev->data->dev_conf.link_speeds;
 973         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
 974                 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
 975                 hw->mac.autoneg = 1;
 976         } else {
 977                 int num_speeds = 0;
 978                 bool autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
 979
 980                 /* Reset */
 981                 hw->phy.autoneg_advertised = 0;
 982
 983                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
 984                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
 985                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
 986                                 ETH_LINK_SPEED_FIXED)) {
 987                         num_speeds = -1;
 988                         goto error_invalid_config;
 989                 }
 990                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
 991                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
 992                         num_speeds++;
 993                 }
 994                 if (*speeds & ETH_LINK_SPEED_10M) {
 995                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
 996                         num_speeds++;
 997                 }
 998                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
 999                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1000                         num_speeds++;
1001                 }
1002                 if (*speeds & ETH_LINK_SPEED_100M) {
1003                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1004                         num_speeds++;
1005                 }
1006                 if (*speeds & ETH_LINK_SPEED_1G) {
1007                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1008                         num_speeds++;
1009                 }
1010                 if (*speeds & ETH_LINK_SPEED_2_5G) {
1011                         hw->phy.autoneg_advertised |= ADVERTISE_2500_FULL;
1012                         num_speeds++;
1013                 }
1014                 if (num_speeds == 0 || (!autoneg && num_speeds > 1))
1015                         goto error_invalid_config;
1016
1017                 /* Set/reset the mac.autoneg based on the link speed,
1018                  * fixed or not
1019                  */
1020                 if (!autoneg) {
1021                         hw->mac.autoneg = 0;
1022                         hw->mac.forced_speed_duplex =
1023                                         hw->phy.autoneg_advertised;
1024                 } else {
1025                         hw->mac.autoneg = 1;
1026                 }
1027         }
1028
1029         igc_setup_link(hw);
1030
1031         if (rte_intr_allow_others(intr_handle)) {
1032                 /* check if lsc interrupt is enabled */
1033                 if (dev->data->dev_conf.intr_conf.lsc)
1034                         igc_lsc_interrupt_setup(dev, 1);
1035                 else
1036                         igc_lsc_interrupt_setup(dev, 0);
1037         } else {
1038                 rte_intr_callback_unregister(intr_handle,
1039                                              eth_igc_interrupt_handler,
1040                                              (void *)dev);
1041                 if (dev->data->dev_conf.intr_conf.lsc)
1042                         PMD_DRV_LOG(INFO,
1043                                 "LSC won't enable because of no intr multiplex");
1044         }
1045
1046         /* enable uio/vfio intr/eventfd mapping */
1047         rte_intr_enable(intr_handle);
1048
1049         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1050                         igc_update_queue_stats_handler, dev);
1051
1052         /* check if rxq interrupt is enabled */
1053         if (dev->data->dev_conf.intr_conf.rxq &&
1054                         rte_intr_dp_is_en(intr_handle))
1055                 igc_rxq_interrupt_setup(dev);
1056
1057         /* resume enabled intr since hw reset */
1058         igc_intr_other_enable(dev);
1059
1060         eth_igc_rxtx_control(dev, true);
1061         eth_igc_link_update(dev, 0);
1062
1063         return 0;
1064
1065 error_invalid_config:
1066         PMD_DRV_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1067                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1068         igc_dev_clear_queues(dev);
1069         return -EINVAL;
1070 }
1071
1072 static int
1073 igc_reset_swfw_lock(struct igc_hw *hw)
1074 {
1075         int ret_val;
1076
1077         /*
1078          * Do mac ops initialization manually here, since we will need
1079          * some function pointers set by this call.
1080          */
1081         ret_val = igc_init_mac_params(hw);
1082         if (ret_val)
1083                 return ret_val;
1084
1085         /*
1086          * SMBI lock should not fail in this early stage. If this is the case,
1087          * it is due to an improper exit of the application.
1088          * So force the release of the faulty lock.
1089          */
1090         if (igc_get_hw_semaphore_generic(hw) < 0)
1091                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
1092
1093         igc_put_hw_semaphore_generic(hw);
1094
1095         if (hw->mac.ops.acquire_swfw_sync != NULL) {
1096                 uint16_t mask;
1097
1098                 /*
1099                  * Phy lock should not fail in this early stage.
1100                  * If this is the case, it is due to an improper exit of the
1101                  * application. So force the release of the faulty lock.
1102                  */
1103                 mask = IGC_SWFW_PHY0_SM;
1104                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
1105                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
1106                                     hw->bus.func);
1107                 }
1108                 hw->mac.ops.release_swfw_sync(hw, mask);
1109
1110                 /*
1111                  * This one is more tricky since it is common to all ports; but
1112                  * swfw_sync retries last long enough (1s) to be almost sure
1113                  * that if lock can not be taken it is due to an improper lock
1114                  * of the semaphore.
1115                  */
1116                 mask = IGC_SWFW_EEP_SM;
1117                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0)
1118                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
1119
1120                 hw->mac.ops.release_swfw_sync(hw, mask);
1121         }
1122
1123         return IGC_SUCCESS;
1124 }
1125
1126 /*
1127  * free all rx/tx queues.
1128  */
1129 static void
1130 igc_dev_free_queues(struct rte_eth_dev *dev)
1131 {
1132         uint16_t i;
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 eth_igc_rx_queue_release(dev->data->rx_queues[i]);
1136                 dev->data->rx_queues[i] = NULL;
1137         }
1138         dev->data->nb_rx_queues = 0;
1139
1140         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1141                 eth_igc_tx_queue_release(dev->data->tx_queues[i]);
1142                 dev->data->tx_queues[i] = NULL;
1143         }
1144         dev->data->nb_tx_queues = 0;
1145 }
1146
1147 static void
1148 eth_igc_close(struct rte_eth_dev *dev)
1149 {
1150         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1151         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1152         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1153         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
1154         int retry = 0;
1155
1156         PMD_INIT_FUNC_TRACE();
1157
1158         if (!adapter->stopped)
1159                 eth_igc_stop(dev);
1160
1161         igc_intr_other_disable(dev);
1162         do {
1163                 int ret = rte_intr_callback_unregister(intr_handle,
1164                                 eth_igc_interrupt_handler, dev);
1165                 if (ret >= 0 || ret == -ENOENT || ret == -EINVAL)
1166                         break;
1167
1168                 PMD_DRV_LOG(ERR, "intr callback unregister failed: %d", ret);
1169                 DELAY(200 * 1000); /* delay 200ms */
1170         } while (retry++ < 5);
1171
1172         igc_phy_hw_reset(hw);
1173         igc_hw_control_release(hw);
1174         igc_dev_free_queues(dev);
1175
1176         /* Reset any pending lock */
1177         igc_reset_swfw_lock(hw);
1178 }
1179
1180 static void
1181 igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
1182 {
1183         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1184
1185         hw->vendor_id = pci_dev->id.vendor_id;
1186         hw->device_id = pci_dev->id.device_id;
1187         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1188         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1189 }
1190
1191 static int
1192 eth_igc_dev_init(struct rte_eth_dev *dev)
1193 {
1194         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1195         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1196         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1197         int i, error = 0;
1198
1199         PMD_INIT_FUNC_TRACE();
1200         dev->dev_ops = &eth_igc_ops;
1201
1202         /*
1203          * for secondary processes, we don't initialize any further as primary
1204          * has already done this work. Only check we don't need a different
1205          * RX function.
1206          */
1207         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1208                 return 0;
1209
1210         rte_eth_copy_pci_info(dev, pci_dev);
1211
1212         hw->back = pci_dev;
1213         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1214
1215         igc_identify_hardware(dev, pci_dev);
1216         if (igc_setup_init_funcs(hw, false) != IGC_SUCCESS) {
1217                 error = -EIO;
1218                 goto err_late;
1219         }
1220
1221         igc_get_bus_info(hw);
1222
1223         /* Reset any pending lock */
1224         if (igc_reset_swfw_lock(hw) != IGC_SUCCESS) {
1225                 error = -EIO;
1226                 goto err_late;
1227         }
1228
1229         /* Finish initialization */
1230         if (igc_setup_init_funcs(hw, true) != IGC_SUCCESS) {
1231                 error = -EIO;
1232                 goto err_late;
1233         }
1234
1235         hw->mac.autoneg = 1;
1236         hw->phy.autoneg_wait_to_complete = 0;
1237         hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
1238
1239         /* Copper options */
1240         if (hw->phy.media_type == igc_media_type_copper) {
1241                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1242                 hw->phy.disable_polarity_correction = 0;
1243                 hw->phy.ms_type = igc_ms_hw_default;
1244         }
1245
1246         /*
1247          * Start from a known state, this is important in reading the nvm
1248          * and mac from that.
1249          */
1250         igc_reset_hw(hw);
1251
1252         /* Make sure we have a good EEPROM before we read from it */
1253         if (igc_validate_nvm_checksum(hw) < 0) {
1254                 /*
1255                  * Some PCI-E parts fail the first check due to
1256                  * the link being in sleep state, call it again,
1257                  * if it fails a second time its a real issue.
1258                  */
1259                 if (igc_validate_nvm_checksum(hw) < 0) {
1260                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
1261                         error = -EIO;
1262                         goto err_late;
1263                 }
1264         }
1265
1266         /* Read the permanent MAC address out of the EEPROM */
1267         if (igc_read_mac_addr(hw) != 0) {
1268                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
1269                 error = -EIO;
1270                 goto err_late;
1271         }
1272
1273         /* Allocate memory for storing MAC addresses */
1274         dev->data->mac_addrs = rte_zmalloc("igc",
1275                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
1276         if (dev->data->mac_addrs == NULL) {
1277                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes for storing MAC",
1278                                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1279                 error = -ENOMEM;
1280                 goto err_late;
1281         }
1282
1283         /* Copy the permanent MAC address */
1284         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
1285                         &dev->data->mac_addrs[0]);
1286
1287         /* Now initialize the hardware */
1288         if (igc_hardware_init(hw) != 0) {
1289                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
1290                 rte_free(dev->data->mac_addrs);
1291                 dev->data->mac_addrs = NULL;
1292                 error = -ENODEV;
1293                 goto err_late;
1294         }
1295
1296         /* Pass the information to the rte_eth_dev_close() that it should also
1297          * release the private port resources.
1298          */
1299         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1300
1301         hw->mac.get_link_status = 1;
1302         igc->stopped = 0;
1303
1304         /* Indicate SOL/IDER usage */
1305         if (igc_check_reset_block(hw) < 0)
1306                 PMD_INIT_LOG(ERR,
1307                         "PHY reset is blocked due to SOL/IDER session.");
1308
1309         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
1310                         dev->data->port_id, pci_dev->id.vendor_id,
1311                         pci_dev->id.device_id);
1312
1313         rte_intr_callback_register(&pci_dev->intr_handle,
1314                         eth_igc_interrupt_handler, (void *)dev);
1315
1316         /* enable uio/vfio intr/eventfd mapping */
1317         rte_intr_enable(&pci_dev->intr_handle);
1318
1319         /* enable support intr */
1320         igc_intr_other_enable(dev);
1321
1322         /* initiate queue status */
1323         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1324                 igc->txq_stats_map[i] = -1;
1325                 igc->rxq_stats_map[i] = -1;
1326         }
1327
1328         return 0;
1329
1330 err_late:
1331         igc_hw_control_release(hw);
1332         return error;
1333 }
1334
1335 static int
1336 eth_igc_dev_uninit(__rte_unused struct rte_eth_dev *eth_dev)
1337 {
1338         PMD_INIT_FUNC_TRACE();
1339
1340         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1341                 return 0;
1342
1343         eth_igc_close(eth_dev);
1344         return 0;
1345 }
1346
1347 static int
1348 eth_igc_reset(struct rte_eth_dev *dev)
1349 {
1350         int ret;
1351
1352         PMD_INIT_FUNC_TRACE();
1353
1354         ret = eth_igc_dev_uninit(dev);
1355         if (ret)
1356                 return ret;
1357
1358         return eth_igc_dev_init(dev);
1359 }
1360
1361 static int
1362 eth_igc_promiscuous_enable(struct rte_eth_dev *dev)
1363 {
1364         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1365         uint32_t rctl;
1366
1367         rctl = IGC_READ_REG(hw, IGC_RCTL);
1368         rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1369         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1370         return 0;
1371 }
1372
1373 static int
1374 eth_igc_promiscuous_disable(struct rte_eth_dev *dev)
1375 {
1376         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1377         uint32_t rctl;
1378
1379         rctl = IGC_READ_REG(hw, IGC_RCTL);
1380         rctl &= (~IGC_RCTL_UPE);
1381         if (dev->data->all_multicast == 1)
1382                 rctl |= IGC_RCTL_MPE;
1383         else
1384                 rctl &= (~IGC_RCTL_MPE);
1385         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1386         return 0;
1387 }
1388
1389 static int
1390 eth_igc_allmulticast_enable(struct rte_eth_dev *dev)
1391 {
1392         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1393         uint32_t rctl;
1394
1395         rctl = IGC_READ_REG(hw, IGC_RCTL);
1396         rctl |= IGC_RCTL_MPE;
1397         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1398         return 0;
1399 }
1400
1401 static int
1402 eth_igc_allmulticast_disable(struct rte_eth_dev *dev)
1403 {
1404         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1405         uint32_t rctl;
1406
1407         if (dev->data->promiscuous == 1)
1408                 return 0;       /* must remain in all_multicast mode */
1409
1410         rctl = IGC_READ_REG(hw, IGC_RCTL);
1411         rctl &= (~IGC_RCTL_MPE);
1412         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1413         return 0;
1414 }
1415
1416 static int
1417 eth_igc_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
1418                        size_t fw_size)
1419 {
1420         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1421         struct igc_fw_version fw;
1422         int ret;
1423
1424         igc_get_fw_version(hw, &fw);
1425
1426         /* if option rom is valid, display its version too */
1427         if (fw.or_valid) {
1428                 ret = snprintf(fw_version, fw_size,
1429                          "%d.%d, 0x%08x, %d.%d.%d",
1430                          fw.eep_major, fw.eep_minor, fw.etrack_id,
1431                          fw.or_major, fw.or_build, fw.or_patch);
1432         /* no option rom */
1433         } else {
1434                 if (fw.etrack_id != 0X0000) {
1435                         ret = snprintf(fw_version, fw_size,
1436                                  "%d.%d, 0x%08x",
1437                                  fw.eep_major, fw.eep_minor,
1438                                  fw.etrack_id);
1439                 } else {
1440                         ret = snprintf(fw_version, fw_size,
1441                                  "%d.%d.%d",
1442                                  fw.eep_major, fw.eep_minor,
1443                                  fw.eep_build);
1444                 }
1445         }
1446
1447         ret += 1; /* add the size of '\0' */
1448         if (fw_size < (u32)ret)
1449                 return ret;
1450         else
1451                 return 0;
1452 }
1453
1454 static int
1455 eth_igc_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1456 {
1457         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1458
1459         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1460         dev_info->max_rx_pktlen = MAX_RX_JUMBO_FRAME_SIZE;
1461         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1462         dev_info->rx_offload_capa = IGC_RX_OFFLOAD_ALL;
1463         dev_info->tx_offload_capa = IGC_TX_OFFLOAD_ALL;
1464         dev_info->rx_queue_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1465
1466         dev_info->max_rx_queues = IGC_QUEUE_PAIRS_NUM;
1467         dev_info->max_tx_queues = IGC_QUEUE_PAIRS_NUM;
1468         dev_info->max_vmdq_pools = 0;
1469
1470         dev_info->hash_key_size = IGC_HKEY_MAX_INDEX * sizeof(uint32_t);
1471         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1472         dev_info->flow_type_rss_offloads = IGC_RSS_OFFLOAD_ALL;
1473
1474         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1475                 .rx_thresh = {
1476                         .pthresh = IGC_DEFAULT_RX_PTHRESH,
1477                         .hthresh = IGC_DEFAULT_RX_HTHRESH,
1478                         .wthresh = IGC_DEFAULT_RX_WTHRESH,
1479                 },
1480                 .rx_free_thresh = IGC_DEFAULT_RX_FREE_THRESH,
1481                 .rx_drop_en = 0,
1482                 .offloads = 0,
1483         };
1484
1485         dev_info->default_txconf = (struct rte_eth_txconf) {
1486                 .tx_thresh = {
1487                         .pthresh = IGC_DEFAULT_TX_PTHRESH,
1488                         .hthresh = IGC_DEFAULT_TX_HTHRESH,
1489                         .wthresh = IGC_DEFAULT_TX_WTHRESH,
1490                 },
1491                 .offloads = 0,
1492         };
1493
1494         dev_info->rx_desc_lim = rx_desc_lim;
1495         dev_info->tx_desc_lim = tx_desc_lim;
1496
1497         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1498                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1499                         ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G;
1500
1501         dev_info->max_mtu = dev_info->max_rx_pktlen - IGC_ETH_OVERHEAD;
1502         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
1503         return 0;
1504 }
1505
1506 static int
1507 eth_igc_led_on(struct rte_eth_dev *dev)
1508 {
1509         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1510
1511         return igc_led_on(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1512 }
1513
1514 static int
1515 eth_igc_led_off(struct rte_eth_dev *dev)
1516 {
1517         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1518
1519         return igc_led_off(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1520 }
1521
1522 static const uint32_t *
1523 eth_igc_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev)
1524 {
1525         static const uint32_t ptypes[] = {
1526                 /* refers to rx_desc_pkt_info_to_pkt_type() */
1527                 RTE_PTYPE_L2_ETHER,
1528                 RTE_PTYPE_L3_IPV4,
1529                 RTE_PTYPE_L3_IPV4_EXT,
1530                 RTE_PTYPE_L3_IPV6,
1531                 RTE_PTYPE_L3_IPV6_EXT,
1532                 RTE_PTYPE_L4_TCP,
1533                 RTE_PTYPE_L4_UDP,
1534                 RTE_PTYPE_L4_SCTP,
1535                 RTE_PTYPE_TUNNEL_IP,
1536                 RTE_PTYPE_INNER_L3_IPV6,
1537                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1538                 RTE_PTYPE_INNER_L4_TCP,
1539                 RTE_PTYPE_INNER_L4_UDP,
1540                 RTE_PTYPE_UNKNOWN
1541         };
1542
1543         return ptypes;
1544 }
1545
1546 static int
1547 eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1548 {
1549         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1550         uint32_t frame_size = mtu + IGC_ETH_OVERHEAD;
1551         uint32_t rctl;
1552
1553         /* if extend vlan has been enabled */
1554         if (IGC_READ_REG(hw, IGC_CTRL_EXT) & IGC_CTRL_EXT_EXT_VLAN)
1555                 frame_size += VLAN_TAG_SIZE;
1556
1557         /* check that mtu is within the allowed range */
1558         if (mtu < RTE_ETHER_MIN_MTU ||
1559                 frame_size > MAX_RX_JUMBO_FRAME_SIZE)
1560                 return -EINVAL;
1561
1562         /*
1563          * refuse mtu that requires the support of scattered packets when
1564          * this feature has not been enabled before.
1565          */
1566         if (!dev->data->scattered_rx &&
1567             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
1568                 return -EINVAL;
1569
1570         rctl = IGC_READ_REG(hw, IGC_RCTL);
1571
1572         /* switch to jumbo mode if needed */
1573         if (mtu > RTE_ETHER_MTU) {
1574                 dev->data->dev_conf.rxmode.offloads |=
1575                         DEV_RX_OFFLOAD_JUMBO_FRAME;
1576                 rctl |= IGC_RCTL_LPE;
1577         } else {
1578                 dev->data->dev_conf.rxmode.offloads &=
1579                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1580                 rctl &= ~IGC_RCTL_LPE;
1581         }
1582         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1583
1584         /* update max frame size */
1585         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1586
1587         IGC_WRITE_REG(hw, IGC_RLPML,
1588                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1589
1590         return 0;
1591 }
1592
1593 static int
1594 eth_igc_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1595                 uint32_t index, uint32_t pool)
1596 {
1597         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1598
1599         igc_rar_set(hw, mac_addr->addr_bytes, index);
1600         RTE_SET_USED(pool);
1601         return 0;
1602 }
1603
1604 static void
1605 eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1606 {
1607         uint8_t addr[RTE_ETHER_ADDR_LEN];
1608         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1609
1610         memset(addr, 0, sizeof(addr));
1611         igc_rar_set(hw, addr, index);
1612 }
1613
1614 static int
1615 eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
1616                         struct rte_ether_addr *addr)
1617 {
1618         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1619         igc_rar_set(hw, addr->addr_bytes, 0);
1620         return 0;
1621 }
1622
1623 static int
1624 eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
1625                          struct rte_ether_addr *mc_addr_set,
1626                          uint32_t nb_mc_addr)
1627 {
1628         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1629         igc_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
1630         return 0;
1631 }
1632
1633 /*
1634  * Read hardware registers
1635  */
1636 static void
1637 igc_read_stats_registers(struct igc_hw *hw, struct igc_hw_stats *stats)
1638 {
1639         int pause_frames;
1640
1641         uint64_t old_gprc  = stats->gprc;
1642         uint64_t old_gptc  = stats->gptc;
1643         uint64_t old_tpr   = stats->tpr;
1644         uint64_t old_tpt   = stats->tpt;
1645         uint64_t old_rpthc = stats->rpthc;
1646         uint64_t old_hgptc = stats->hgptc;
1647
1648         stats->crcerrs += IGC_READ_REG(hw, IGC_CRCERRS);
1649         stats->algnerrc += IGC_READ_REG(hw, IGC_ALGNERRC);
1650         stats->rxerrc += IGC_READ_REG(hw, IGC_RXERRC);
1651         stats->mpc += IGC_READ_REG(hw, IGC_MPC);
1652         stats->scc += IGC_READ_REG(hw, IGC_SCC);
1653         stats->ecol += IGC_READ_REG(hw, IGC_ECOL);
1654
1655         stats->mcc += IGC_READ_REG(hw, IGC_MCC);
1656         stats->latecol += IGC_READ_REG(hw, IGC_LATECOL);
1657         stats->colc += IGC_READ_REG(hw, IGC_COLC);
1658
1659         stats->dc += IGC_READ_REG(hw, IGC_DC);
1660         stats->tncrs += IGC_READ_REG(hw, IGC_TNCRS);
1661         stats->htdpmc += IGC_READ_REG(hw, IGC_HTDPMC);
1662         stats->rlec += IGC_READ_REG(hw, IGC_RLEC);
1663         stats->xonrxc += IGC_READ_REG(hw, IGC_XONRXC);
1664         stats->xontxc += IGC_READ_REG(hw, IGC_XONTXC);
1665
1666         /*
1667          * For watchdog management we need to know if we have been
1668          * paused during the last interval, so capture that here.
1669          */
1670         pause_frames = IGC_READ_REG(hw, IGC_XOFFRXC);
1671         stats->xoffrxc += pause_frames;
1672         stats->xofftxc += IGC_READ_REG(hw, IGC_XOFFTXC);
1673         stats->fcruc += IGC_READ_REG(hw, IGC_FCRUC);
1674         stats->prc64 += IGC_READ_REG(hw, IGC_PRC64);
1675         stats->prc127 += IGC_READ_REG(hw, IGC_PRC127);
1676         stats->prc255 += IGC_READ_REG(hw, IGC_PRC255);
1677         stats->prc511 += IGC_READ_REG(hw, IGC_PRC511);
1678         stats->prc1023 += IGC_READ_REG(hw, IGC_PRC1023);
1679         stats->prc1522 += IGC_READ_REG(hw, IGC_PRC1522);
1680         stats->gprc += IGC_READ_REG(hw, IGC_GPRC);
1681         stats->bprc += IGC_READ_REG(hw, IGC_BPRC);
1682         stats->mprc += IGC_READ_REG(hw, IGC_MPRC);
1683         stats->gptc += IGC_READ_REG(hw, IGC_GPTC);
1684
1685         /* For the 64-bit byte counters the low dword must be read first. */
1686         /* Both registers clear on the read of the high dword */
1687
1688         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1689         stats->gorc += IGC_READ_REG(hw, IGC_GORCL);
1690         stats->gorc += ((uint64_t)IGC_READ_REG(hw, IGC_GORCH) << 32);
1691         stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN;
1692         stats->gotc += IGC_READ_REG(hw, IGC_GOTCL);
1693         stats->gotc += ((uint64_t)IGC_READ_REG(hw, IGC_GOTCH) << 32);
1694         stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN;
1695
1696         stats->rnbc += IGC_READ_REG(hw, IGC_RNBC);
1697         stats->ruc += IGC_READ_REG(hw, IGC_RUC);
1698         stats->rfc += IGC_READ_REG(hw, IGC_RFC);
1699         stats->roc += IGC_READ_REG(hw, IGC_ROC);
1700         stats->rjc += IGC_READ_REG(hw, IGC_RJC);
1701
1702         stats->mgprc += IGC_READ_REG(hw, IGC_MGTPRC);
1703         stats->mgpdc += IGC_READ_REG(hw, IGC_MGTPDC);
1704         stats->mgptc += IGC_READ_REG(hw, IGC_MGTPTC);
1705         stats->b2ospc += IGC_READ_REG(hw, IGC_B2OSPC);
1706         stats->b2ogprc += IGC_READ_REG(hw, IGC_B2OGPRC);
1707         stats->o2bgptc += IGC_READ_REG(hw, IGC_O2BGPTC);
1708         stats->o2bspc += IGC_READ_REG(hw, IGC_O2BSPC);
1709
1710         stats->tpr += IGC_READ_REG(hw, IGC_TPR);
1711         stats->tpt += IGC_READ_REG(hw, IGC_TPT);
1712
1713         stats->tor += IGC_READ_REG(hw, IGC_TORL);
1714         stats->tor += ((uint64_t)IGC_READ_REG(hw, IGC_TORH) << 32);
1715         stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN;
1716         stats->tot += IGC_READ_REG(hw, IGC_TOTL);
1717         stats->tot += ((uint64_t)IGC_READ_REG(hw, IGC_TOTH) << 32);
1718         stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN;
1719
1720         stats->ptc64 += IGC_READ_REG(hw, IGC_PTC64);
1721         stats->ptc127 += IGC_READ_REG(hw, IGC_PTC127);
1722         stats->ptc255 += IGC_READ_REG(hw, IGC_PTC255);
1723         stats->ptc511 += IGC_READ_REG(hw, IGC_PTC511);
1724         stats->ptc1023 += IGC_READ_REG(hw, IGC_PTC1023);
1725         stats->ptc1522 += IGC_READ_REG(hw, IGC_PTC1522);
1726         stats->mptc += IGC_READ_REG(hw, IGC_MPTC);
1727         stats->bptc += IGC_READ_REG(hw, IGC_BPTC);
1728         stats->tsctc += IGC_READ_REG(hw, IGC_TSCTC);
1729
1730         stats->iac += IGC_READ_REG(hw, IGC_IAC);
1731         stats->rpthc += IGC_READ_REG(hw, IGC_RPTHC);
1732         stats->hgptc += IGC_READ_REG(hw, IGC_HGPTC);
1733         stats->icrxdmtc += IGC_READ_REG(hw, IGC_ICRXDMTC);
1734
1735         /* Host to Card Statistics */
1736         stats->hgorc += IGC_READ_REG(hw, IGC_HGORCL);
1737         stats->hgorc += ((uint64_t)IGC_READ_REG(hw, IGC_HGORCH) << 32);
1738         stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN;
1739         stats->hgotc += IGC_READ_REG(hw, IGC_HGOTCL);
1740         stats->hgotc += ((uint64_t)IGC_READ_REG(hw, IGC_HGOTCH) << 32);
1741         stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN;
1742         stats->lenerrs += IGC_READ_REG(hw, IGC_LENERRS);
1743 }
1744
1745 /*
1746  * Write 0 to all queue status registers
1747  */
1748 static void
1749 igc_reset_queue_stats_register(struct igc_hw *hw)
1750 {
1751         int i;
1752
1753         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1754                 IGC_WRITE_REG(hw, IGC_PQGPRC(i), 0);
1755                 IGC_WRITE_REG(hw, IGC_PQGPTC(i), 0);
1756                 IGC_WRITE_REG(hw, IGC_PQGORC(i), 0);
1757                 IGC_WRITE_REG(hw, IGC_PQGOTC(i), 0);
1758                 IGC_WRITE_REG(hw, IGC_PQMPRC(i), 0);
1759                 IGC_WRITE_REG(hw, IGC_RQDPC(i), 0);
1760                 IGC_WRITE_REG(hw, IGC_TQDPC(i), 0);
1761         }
1762 }
1763
1764 /*
1765  * Read all hardware queue status registers
1766  */
1767 static void
1768 igc_read_queue_stats_register(struct rte_eth_dev *dev)
1769 {
1770         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1771         struct igc_hw_queue_stats *queue_stats =
1772                                 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1773         int i;
1774
1775         /*
1776          * This register is not cleared on read. Furthermore, the register wraps
1777          * around back to 0x00000000 on the next increment when reaching a value
1778          * of 0xFFFFFFFF and then continues normal count operation.
1779          */
1780         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1781                 union {
1782                         u64 ddword;
1783                         u32 dword[2];
1784                 } value;
1785                 u32 tmp;
1786
1787                 /*
1788                  * Read the register first, if the value is smaller than that
1789                  * previous read, that mean the register has been overflowed,
1790                  * then we add the high 4 bytes by 1 and replace the low 4
1791                  * bytes by the new value.
1792                  */
1793                 tmp = IGC_READ_REG(hw, IGC_PQGPRC(i));
1794                 value.ddword = queue_stats->pqgprc[i];
1795                 if (value.dword[U32_0_IN_U64] > tmp)
1796                         value.dword[U32_1_IN_U64]++;
1797                 value.dword[U32_0_IN_U64] = tmp;
1798                 queue_stats->pqgprc[i] = value.ddword;
1799
1800                 tmp = IGC_READ_REG(hw, IGC_PQGPTC(i));
1801                 value.ddword = queue_stats->pqgptc[i];
1802                 if (value.dword[U32_0_IN_U64] > tmp)
1803                         value.dword[U32_1_IN_U64]++;
1804                 value.dword[U32_0_IN_U64] = tmp;
1805                 queue_stats->pqgptc[i] = value.ddword;
1806
1807                 tmp = IGC_READ_REG(hw, IGC_PQGORC(i));
1808                 value.ddword = queue_stats->pqgorc[i];
1809                 if (value.dword[U32_0_IN_U64] > tmp)
1810                         value.dword[U32_1_IN_U64]++;
1811                 value.dword[U32_0_IN_U64] = tmp;
1812                 queue_stats->pqgorc[i] = value.ddword;
1813
1814                 tmp = IGC_READ_REG(hw, IGC_PQGOTC(i));
1815                 value.ddword = queue_stats->pqgotc[i];
1816                 if (value.dword[U32_0_IN_U64] > tmp)
1817                         value.dword[U32_1_IN_U64]++;
1818                 value.dword[U32_0_IN_U64] = tmp;
1819                 queue_stats->pqgotc[i] = value.ddword;
1820
1821                 tmp = IGC_READ_REG(hw, IGC_PQMPRC(i));
1822                 value.ddword = queue_stats->pqmprc[i];
1823                 if (value.dword[U32_0_IN_U64] > tmp)
1824                         value.dword[U32_1_IN_U64]++;
1825                 value.dword[U32_0_IN_U64] = tmp;
1826                 queue_stats->pqmprc[i] = value.ddword;
1827
1828                 tmp = IGC_READ_REG(hw, IGC_RQDPC(i));
1829                 value.ddword = queue_stats->rqdpc[i];
1830                 if (value.dword[U32_0_IN_U64] > tmp)
1831                         value.dword[U32_1_IN_U64]++;
1832                 value.dword[U32_0_IN_U64] = tmp;
1833                 queue_stats->rqdpc[i] = value.ddword;
1834
1835                 tmp = IGC_READ_REG(hw, IGC_TQDPC(i));
1836                 value.ddword = queue_stats->tqdpc[i];
1837                 if (value.dword[U32_0_IN_U64] > tmp)
1838                         value.dword[U32_1_IN_U64]++;
1839                 value.dword[U32_0_IN_U64] = tmp;
1840                 queue_stats->tqdpc[i] = value.ddword;
1841         }
1842 }
1843
1844 static int
1845 eth_igc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1846 {
1847         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1848         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1849         struct igc_hw_stats *stats = IGC_DEV_PRIVATE_STATS(dev);
1850         struct igc_hw_queue_stats *queue_stats =
1851                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1852         int i;
1853
1854         /*
1855          * Cancel status handler since it will read the queue status registers
1856          */
1857         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1858
1859         /* Read status register */
1860         igc_read_queue_stats_register(dev);
1861         igc_read_stats_registers(hw, stats);
1862
1863         if (rte_stats == NULL) {
1864                 /* Restart queue status handler */
1865                 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1866                                 igc_update_queue_stats_handler, dev);
1867                 return -EINVAL;
1868         }
1869
1870         /* Rx Errors */
1871         rte_stats->imissed = stats->mpc;
1872         rte_stats->ierrors = stats->crcerrs +
1873                         stats->rlec + stats->ruc + stats->roc +
1874                         stats->rxerrc + stats->algnerrc;
1875
1876         /* Tx Errors */
1877         rte_stats->oerrors = stats->ecol + stats->latecol;
1878
1879         rte_stats->ipackets = stats->gprc;
1880         rte_stats->opackets = stats->gptc;
1881         rte_stats->ibytes   = stats->gorc;
1882         rte_stats->obytes   = stats->gotc;
1883
1884         /* Get per-queue statuses */
1885         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1886                 /* GET TX queue statuses */
1887                 int map_id = igc->txq_stats_map[i];
1888                 if (map_id >= 0) {
1889                         rte_stats->q_opackets[map_id] += queue_stats->pqgptc[i];
1890                         rte_stats->q_obytes[map_id] += queue_stats->pqgotc[i];
1891                 }
1892                 /* Get RX queue statuses */
1893                 map_id = igc->rxq_stats_map[i];
1894                 if (map_id >= 0) {
1895                         rte_stats->q_ipackets[map_id] += queue_stats->pqgprc[i];
1896                         rte_stats->q_ibytes[map_id] += queue_stats->pqgorc[i];
1897                         rte_stats->q_errors[map_id] += queue_stats->rqdpc[i];
1898                 }
1899         }
1900
1901         /* Restart queue status handler */
1902         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1903                         igc_update_queue_stats_handler, dev);
1904         return 0;
1905 }
1906
1907 static int
1908 eth_igc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1909                    unsigned int n)
1910 {
1911         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1912         struct igc_hw_stats *hw_stats =
1913                         IGC_DEV_PRIVATE_STATS(dev);
1914         unsigned int i;
1915
1916         igc_read_stats_registers(hw, hw_stats);
1917
1918         if (n < IGC_NB_XSTATS)
1919                 return IGC_NB_XSTATS;
1920
1921         /* If this is a reset xstats is NULL, and we have cleared the
1922          * registers by reading them.
1923          */
1924         if (!xstats)
1925                 return 0;
1926
1927         /* Extended stats */
1928         for (i = 0; i < IGC_NB_XSTATS; i++) {
1929                 xstats[i].id = i;
1930                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1931                         rte_igc_stats_strings[i].offset);
1932         }
1933
1934         return IGC_NB_XSTATS;
1935 }
1936
1937 static int
1938 eth_igc_xstats_reset(struct rte_eth_dev *dev)
1939 {
1940         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1941         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1942         struct igc_hw_queue_stats *queue_stats =
1943                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1944
1945         /* Cancel queue status handler for avoid conflict */
1946         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1947
1948         /* HW registers are cleared on read */
1949         igc_reset_queue_stats_register(hw);
1950         igc_read_stats_registers(hw, hw_stats);
1951
1952         /* Reset software totals */
1953         memset(hw_stats, 0, sizeof(*hw_stats));
1954         memset(queue_stats, 0, sizeof(*queue_stats));
1955
1956         /* Restart the queue status handler */
1957         rte_eal_alarm_set(IGC_ALARM_INTERVAL, igc_update_queue_stats_handler,
1958                         dev);
1959
1960         return 0;
1961 }
1962
1963 static int
1964 eth_igc_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1965         struct rte_eth_xstat_name *xstats_names, unsigned int size)
1966 {
1967         unsigned int i;
1968
1969         if (xstats_names == NULL)
1970                 return IGC_NB_XSTATS;
1971
1972         if (size < IGC_NB_XSTATS) {
1973                 PMD_DRV_LOG(ERR, "not enough buffers!");
1974                 return IGC_NB_XSTATS;
1975         }
1976
1977         for (i = 0; i < IGC_NB_XSTATS; i++)
1978                 strlcpy(xstats_names[i].name, rte_igc_stats_strings[i].name,
1979                         sizeof(xstats_names[i].name));
1980
1981         return IGC_NB_XSTATS;
1982 }
1983
1984 static int
1985 eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
1986                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
1987                 unsigned int limit)
1988 {
1989         unsigned int i;
1990
1991         if (!ids)
1992                 return eth_igc_xstats_get_names(dev, xstats_names, limit);
1993
1994         for (i = 0; i < limit; i++) {
1995                 if (ids[i] >= IGC_NB_XSTATS) {
1996                         PMD_DRV_LOG(ERR, "id value isn't valid");
1997                         return -EINVAL;
1998                 }
1999                 strlcpy(xstats_names[i].name,
2000                         rte_igc_stats_strings[ids[i]].name,
2001                         sizeof(xstats_names[i].name));
2002         }
2003         return limit;
2004 }
2005
2006 static int
2007 eth_igc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
2008                 uint64_t *values, unsigned int n)
2009 {
2010         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2011         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
2012         unsigned int i;
2013
2014         igc_read_stats_registers(hw, hw_stats);
2015
2016         if (!ids) {
2017                 if (n < IGC_NB_XSTATS)
2018                         return IGC_NB_XSTATS;
2019
2020                 /* If this is a reset xstats is NULL, and we have cleared the
2021                  * registers by reading them.
2022                  */
2023                 if (!values)
2024                         return 0;
2025
2026                 /* Extended stats */
2027                 for (i = 0; i < IGC_NB_XSTATS; i++)
2028                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2029                                         rte_igc_stats_strings[i].offset);
2030
2031                 return IGC_NB_XSTATS;
2032
2033         } else {
2034                 for (i = 0; i < n; i++) {
2035                         if (ids[i] >= IGC_NB_XSTATS) {
2036                                 PMD_DRV_LOG(ERR, "id value isn't valid");
2037                                 return -EINVAL;
2038                         }
2039                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2040                                         rte_igc_stats_strings[ids[i]].offset);
2041                 }
2042                 return n;
2043         }
2044 }
2045
2046 static int
2047 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
2048                 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx)
2049 {
2050         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
2051
2052         /* check queue id is valid */
2053         if (queue_id >= IGC_QUEUE_PAIRS_NUM) {
2054                 PMD_DRV_LOG(ERR, "queue id(%u) error, max is %u",
2055                         queue_id, IGC_QUEUE_PAIRS_NUM - 1);
2056                 return -EINVAL;
2057         }
2058
2059         /* store the mapping status id */
2060         if (is_rx)
2061                 igc->rxq_stats_map[queue_id] = stat_idx;
2062         else
2063                 igc->txq_stats_map[queue_id] = stat_idx;
2064
2065         return 0;
2066 }
2067
2068 static int
2069 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2070 {
2071         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2072         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2073         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2074         uint32_t vec = IGC_MISC_VEC_ID;
2075
2076         if (rte_intr_allow_others(intr_handle))
2077                 vec = IGC_RX_VEC_START;
2078
2079         uint32_t mask = 1u << (queue_id + vec);
2080
2081         IGC_WRITE_REG(hw, IGC_EIMC, mask);
2082         IGC_WRITE_FLUSH(hw);
2083
2084         return 0;
2085 }
2086
2087 static int
2088 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2089 {
2090         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2091         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2092         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2093         uint32_t vec = IGC_MISC_VEC_ID;
2094
2095         if (rte_intr_allow_others(intr_handle))
2096                 vec = IGC_RX_VEC_START;
2097
2098         uint32_t mask = 1u << (queue_id + vec);
2099
2100         IGC_WRITE_REG(hw, IGC_EIMS, mask);
2101         IGC_WRITE_FLUSH(hw);
2102
2103         rte_intr_enable(intr_handle);
2104
2105         return 0;
2106 }
2107
2108 static int
2109 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2110 {
2111         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2112         uint32_t ctrl;
2113         int tx_pause;
2114         int rx_pause;
2115
2116         fc_conf->pause_time = hw->fc.pause_time;
2117         fc_conf->high_water = hw->fc.high_water;
2118         fc_conf->low_water = hw->fc.low_water;
2119         fc_conf->send_xon = hw->fc.send_xon;
2120         fc_conf->autoneg = hw->mac.autoneg;
2121
2122         /*
2123          * Return rx_pause and tx_pause status according to actual setting of
2124          * the TFCE and RFCE bits in the CTRL register.
2125          */
2126         ctrl = IGC_READ_REG(hw, IGC_CTRL);
2127         if (ctrl & IGC_CTRL_TFCE)
2128                 tx_pause = 1;
2129         else
2130                 tx_pause = 0;
2131
2132         if (ctrl & IGC_CTRL_RFCE)
2133                 rx_pause = 1;
2134         else
2135                 rx_pause = 0;
2136
2137         if (rx_pause && tx_pause)
2138                 fc_conf->mode = RTE_FC_FULL;
2139         else if (rx_pause)
2140                 fc_conf->mode = RTE_FC_RX_PAUSE;
2141         else if (tx_pause)
2142                 fc_conf->mode = RTE_FC_TX_PAUSE;
2143         else
2144                 fc_conf->mode = RTE_FC_NONE;
2145
2146         return 0;
2147 }
2148
2149 static int
2150 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2151 {
2152         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2153         uint32_t rx_buf_size;
2154         uint32_t max_high_water;
2155         uint32_t rctl;
2156         int err;
2157
2158         if (fc_conf->autoneg != hw->mac.autoneg)
2159                 return -ENOTSUP;
2160
2161         rx_buf_size = igc_get_rx_buffer_size(hw);
2162         PMD_DRV_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2163
2164         /* At least reserve one Ethernet frame for watermark */
2165         max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN;
2166         if (fc_conf->high_water > max_high_water ||
2167                 fc_conf->high_water < fc_conf->low_water) {
2168                 PMD_DRV_LOG(ERR,
2169                         "Incorrect high(%u)/low(%u) water value, max is %u",
2170                         fc_conf->high_water, fc_conf->low_water,
2171                         max_high_water);
2172                 return -EINVAL;
2173         }
2174
2175         switch (fc_conf->mode) {
2176         case RTE_FC_NONE:
2177                 hw->fc.requested_mode = igc_fc_none;
2178                 break;
2179         case RTE_FC_RX_PAUSE:
2180                 hw->fc.requested_mode = igc_fc_rx_pause;
2181                 break;
2182         case RTE_FC_TX_PAUSE:
2183                 hw->fc.requested_mode = igc_fc_tx_pause;
2184                 break;
2185         case RTE_FC_FULL:
2186                 hw->fc.requested_mode = igc_fc_full;
2187                 break;
2188         default:
2189                 PMD_DRV_LOG(ERR, "unsupported fc mode: %u", fc_conf->mode);
2190                 return -EINVAL;
2191         }
2192
2193         hw->fc.pause_time     = fc_conf->pause_time;
2194         hw->fc.high_water     = fc_conf->high_water;
2195         hw->fc.low_water      = fc_conf->low_water;
2196         hw->fc.send_xon       = fc_conf->send_xon;
2197
2198         err = igc_setup_link_generic(hw);
2199         if (err == IGC_SUCCESS) {
2200                 /**
2201                  * check if we want to forward MAC frames - driver doesn't have
2202                  * native capability to do that, so we'll write the registers
2203                  * ourselves
2204                  **/
2205                 rctl = IGC_READ_REG(hw, IGC_RCTL);
2206
2207                 /* set or clear MFLCN.PMCF bit depending on configuration */
2208                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2209                         rctl |= IGC_RCTL_PMCF;
2210                 else
2211                         rctl &= ~IGC_RCTL_PMCF;
2212
2213                 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
2214                 IGC_WRITE_FLUSH(hw);
2215
2216                 return 0;
2217         }
2218
2219         PMD_DRV_LOG(ERR, "igc_setup_link_generic = 0x%x", err);
2220         return -EIO;
2221 }
2222
2223 static int
2224 eth_igc_rss_reta_update(struct rte_eth_dev *dev,
2225                         struct rte_eth_rss_reta_entry64 *reta_conf,
2226                         uint16_t reta_size)
2227 {
2228         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2229         uint16_t i;
2230
2231         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2232                 PMD_DRV_LOG(ERR,
2233                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2234                         reta_size, ETH_RSS_RETA_SIZE_128);
2235                 return -EINVAL;
2236         }
2237
2238         /* set redirection table */
2239         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2240                 union igc_rss_reta_reg reta, reg;
2241                 uint16_t idx, shift;
2242                 uint8_t j, mask;
2243
2244                 idx = i / RTE_RETA_GROUP_SIZE;
2245                 shift = i % RTE_RETA_GROUP_SIZE;
2246                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2247                                 IGC_RSS_RDT_REG_SIZE_MASK);
2248
2249                 /* if no need to update the register */
2250                 if (!mask)
2251                         continue;
2252
2253                 /* check mask whether need to read the register value first */
2254                 if (mask == IGC_RSS_RDT_REG_SIZE_MASK)
2255                         reg.dword = 0;
2256                 else
2257                         reg.dword = IGC_READ_REG_LE_VALUE(hw,
2258                                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2259
2260                 /* update the register */
2261                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2262                         if (mask & (1u << j))
2263                                 reta.bytes[j] =
2264                                         (uint8_t)reta_conf[idx].reta[shift + j];
2265                         else
2266                                 reta.bytes[j] = reg.bytes[j];
2267                 }
2268                 IGC_WRITE_REG_LE_VALUE(hw,
2269                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE), reta.dword);
2270         }
2271
2272         return 0;
2273 }
2274
2275 static int
2276 eth_igc_rss_reta_query(struct rte_eth_dev *dev,
2277                        struct rte_eth_rss_reta_entry64 *reta_conf,
2278                        uint16_t reta_size)
2279 {
2280         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2281         uint16_t i;
2282
2283         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2284                 PMD_DRV_LOG(ERR,
2285                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2286                         reta_size, ETH_RSS_RETA_SIZE_128);
2287                 return -EINVAL;
2288         }
2289
2290         /* read redirection table */
2291         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2292                 union igc_rss_reta_reg reta;
2293                 uint16_t idx, shift;
2294                 uint8_t j, mask;
2295
2296                 idx = i / RTE_RETA_GROUP_SIZE;
2297                 shift = i % RTE_RETA_GROUP_SIZE;
2298                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2299                                 IGC_RSS_RDT_REG_SIZE_MASK);
2300
2301                 /* if no need to read register */
2302                 if (!mask)
2303                         continue;
2304
2305                 /* read register and get the queue index */
2306                 reta.dword = IGC_READ_REG_LE_VALUE(hw,
2307                                 IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2308                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2309                         if (mask & (1u << j))
2310                                 reta_conf[idx].reta[shift + j] = reta.bytes[j];
2311                 }
2312         }
2313
2314         return 0;
2315 }
2316
2317 static int
2318 eth_igc_rss_hash_update(struct rte_eth_dev *dev,
2319                         struct rte_eth_rss_conf *rss_conf)
2320 {
2321         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2322         igc_hw_rss_hash_set(hw, rss_conf);
2323         return 0;
2324 }
2325
2326 static int
2327 eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
2328                         struct rte_eth_rss_conf *rss_conf)
2329 {
2330         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2331         uint32_t *hash_key = (uint32_t *)rss_conf->rss_key;
2332         uint32_t mrqc;
2333         uint64_t rss_hf;
2334
2335         if (hash_key != NULL) {
2336                 int i;
2337
2338                 /* if not enough space for store hash key */
2339                 if (rss_conf->rss_key_len != IGC_HKEY_SIZE) {
2340                         PMD_DRV_LOG(ERR,
2341                                 "RSS hash key size %u in parameter doesn't match the hardware hash key size %u",
2342                                 rss_conf->rss_key_len, IGC_HKEY_SIZE);
2343                         return -EINVAL;
2344                 }
2345
2346                 /* read RSS key from register */
2347                 for (i = 0; i < IGC_HKEY_MAX_INDEX; i++)
2348                         hash_key[i] = IGC_READ_REG_LE_VALUE(hw, IGC_RSSRK(i));
2349         }
2350
2351         /* get RSS functions configured in MRQC register */
2352         mrqc = IGC_READ_REG(hw, IGC_MRQC);
2353         if ((mrqc & IGC_MRQC_ENABLE_RSS_4Q) == 0)
2354                 return 0;
2355
2356         rss_hf = 0;
2357         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4)
2358                 rss_hf |= ETH_RSS_IPV4;
2359         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_TCP)
2360                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2361         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6)
2362                 rss_hf |= ETH_RSS_IPV6;
2363         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_EX)
2364                 rss_hf |= ETH_RSS_IPV6_EX;
2365         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP)
2366                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
2367         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP_EX)
2368                 rss_hf |= ETH_RSS_IPV6_TCP_EX;
2369         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_UDP)
2370                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2371         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP)
2372                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
2373         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP_EX)
2374                 rss_hf |= ETH_RSS_IPV6_UDP_EX;
2375
2376         rss_conf->rss_hf |= rss_hf;
2377         return 0;
2378 }
2379
2380 static int
2381 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2382 {
2383         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2384         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2385         uint32_t vfta;
2386         uint32_t vid_idx;
2387         uint32_t vid_bit;
2388
2389         vid_idx = (vlan_id >> IGC_VFTA_ENTRY_SHIFT) & IGC_VFTA_ENTRY_MASK;
2390         vid_bit = 1u << (vlan_id & IGC_VFTA_ENTRY_BIT_SHIFT_MASK);
2391         vfta = shadow_vfta->vfta[vid_idx];
2392         if (on)
2393                 vfta |= vid_bit;
2394         else
2395                 vfta &= ~vid_bit;
2396         IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, vid_idx, vfta);
2397
2398         /* update local VFTA copy */
2399         shadow_vfta->vfta[vid_idx] = vfta;
2400
2401         return 0;
2402 }
2403
2404 static void
2405 igc_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2406 {
2407         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2408         igc_read_reg_check_clear_bits(hw, IGC_RCTL,
2409                         IGC_RCTL_CFIEN | IGC_RCTL_VFE);
2410 }
2411
2412 static void
2413 igc_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2414 {
2415         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2416         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2417         uint32_t reg_val;
2418         int i;
2419
2420         /* Filter Table Enable, CFI not used for packet acceptance */
2421         reg_val = IGC_READ_REG(hw, IGC_RCTL);
2422         reg_val &= ~IGC_RCTL_CFIEN;
2423         reg_val |= IGC_RCTL_VFE;
2424         IGC_WRITE_REG(hw, IGC_RCTL, reg_val);
2425
2426         /* restore VFTA table */
2427         for (i = 0; i < IGC_VFTA_SIZE; i++)
2428                 IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, i, shadow_vfta->vfta[i]);
2429 }
2430
2431 static void
2432 igc_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2433 {
2434         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2435
2436         igc_read_reg_check_clear_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2437 }
2438
2439 static void
2440 igc_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2441 {
2442         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2443
2444         igc_read_reg_check_set_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2445 }
2446
2447 static int
2448 igc_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2449 {
2450         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2451         uint32_t ctrl_ext;
2452
2453         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2454
2455         /* if extend vlan hasn't been enabled */
2456         if ((ctrl_ext & IGC_CTRL_EXT_EXT_VLAN) == 0)
2457                 return 0;
2458
2459         if ((dev->data->dev_conf.rxmode.offloads &
2460                         DEV_RX_OFFLOAD_JUMBO_FRAME) == 0)
2461                 goto write_ext_vlan;
2462
2463         /* Update maximum packet length */
2464         if (dev->data->dev_conf.rxmode.max_rx_pkt_len <
2465                 RTE_ETHER_MIN_MTU + VLAN_TAG_SIZE) {
2466                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, min is %u",
2467                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
2468                         VLAN_TAG_SIZE + RTE_ETHER_MIN_MTU);
2469                 return -EINVAL;
2470         }
2471         dev->data->dev_conf.rxmode.max_rx_pkt_len -= VLAN_TAG_SIZE;
2472         IGC_WRITE_REG(hw, IGC_RLPML,
2473                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
2474
2475 write_ext_vlan:
2476         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_EXT_VLAN);
2477         return 0;
2478 }
2479
2480 static int
2481 igc_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2482 {
2483         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2484         uint32_t ctrl_ext;
2485
2486         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2487
2488         /* if extend vlan has been enabled */
2489         if (ctrl_ext & IGC_CTRL_EXT_EXT_VLAN)
2490                 return 0;
2491
2492         if ((dev->data->dev_conf.rxmode.offloads &
2493                         DEV_RX_OFFLOAD_JUMBO_FRAME) == 0)
2494                 goto write_ext_vlan;
2495
2496         /* Update maximum packet length */
2497         if (dev->data->dev_conf.rxmode.max_rx_pkt_len >
2498                 MAX_RX_JUMBO_FRAME_SIZE - VLAN_TAG_SIZE) {
2499                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, max is %u",
2500                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2501                         VLAN_TAG_SIZE, MAX_RX_JUMBO_FRAME_SIZE);
2502                 return -EINVAL;
2503         }
2504         dev->data->dev_conf.rxmode.max_rx_pkt_len += VLAN_TAG_SIZE;
2505         IGC_WRITE_REG(hw, IGC_RLPML,
2506                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
2507
2508 write_ext_vlan:
2509         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_EXT_VLAN);
2510         return 0;
2511 }
2512
2513 static int
2514 eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2515 {
2516         struct rte_eth_rxmode *rxmode;
2517
2518         rxmode = &dev->data->dev_conf.rxmode;
2519         if (mask & ETH_VLAN_STRIP_MASK) {
2520                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2521                         igc_vlan_hw_strip_enable(dev);
2522                 else
2523                         igc_vlan_hw_strip_disable(dev);
2524         }
2525
2526         if (mask & ETH_VLAN_FILTER_MASK) {
2527                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
2528                         igc_vlan_hw_filter_enable(dev);
2529                 else
2530                         igc_vlan_hw_filter_disable(dev);
2531         }
2532
2533         if (mask & ETH_VLAN_EXTEND_MASK) {
2534                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
2535                         return igc_vlan_hw_extend_enable(dev);
2536                 else
2537                         return igc_vlan_hw_extend_disable(dev);
2538         }
2539
2540         return 0;
2541 }
2542
2543 static int
2544 eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
2545                       enum rte_vlan_type vlan_type,
2546                       uint16_t tpid)
2547 {
2548         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2549         uint32_t reg_val;
2550
2551         /* only outer TPID of double VLAN can be configured*/
2552         if (vlan_type == ETH_VLAN_TYPE_OUTER) {
2553                 reg_val = IGC_READ_REG(hw, IGC_VET);
2554                 reg_val = (reg_val & (~IGC_VET_EXT)) |
2555                         ((uint32_t)tpid << IGC_VET_EXT_SHIFT);
2556                 IGC_WRITE_REG(hw, IGC_VET, reg_val);
2557
2558                 return 0;
2559         }
2560
2561         /* all other TPID values are read-only*/
2562         PMD_DRV_LOG(ERR, "Not supported");
2563         return -ENOTSUP;
2564 }
2565
2566 static int
2567 eth_igc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2568         struct rte_pci_device *pci_dev)
2569 {
2570         PMD_INIT_FUNC_TRACE();
2571         return rte_eth_dev_pci_generic_probe(pci_dev,
2572                 sizeof(struct igc_adapter), eth_igc_dev_init);
2573 }
2574
2575 static int
2576 eth_igc_pci_remove(struct rte_pci_device *pci_dev)
2577 {
2578         PMD_INIT_FUNC_TRACE();
2579         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igc_dev_uninit);
2580 }
2581
2582 static struct rte_pci_driver rte_igc_pmd = {
2583         .id_table = pci_id_igc_map,
2584         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2585         .probe = eth_igc_pci_probe,
2586         .remove = eth_igc_pci_remove,
2587 };
2588
2589 RTE_PMD_REGISTER_PCI(net_igc, rte_igc_pmd);
2590 RTE_PMD_REGISTER_PCI_TABLE(net_igc, pci_id_igc_map);
2591 RTE_PMD_REGISTER_KMOD_DEP(net_igc, "* igb_uio | uio_pci_generic | vfio-pci");