drivers/net/igc/igc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7
   8 #include <rte_string_fns.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <ethdev_driver.h>
  12 #include <ethdev_pci.h>
  13 #include <rte_malloc.h>
  14 #include <rte_alarm.h>
  15
  16 #include "igc_logs.h"
  17 #include "igc_txrx.h"
  18 #include "igc_filter.h"
  19 #include "igc_flow.h"
  20
  21 #define IGC_INTEL_VENDOR_ID             0x8086
  22
  23 #define IGC_FC_PAUSE_TIME               0x0680
  24 #define IGC_LINK_UPDATE_CHECK_TIMEOUT   90  /* 9s */
  25 #define IGC_LINK_UPDATE_CHECK_INTERVAL  100 /* ms */
  26
  27 #define IGC_MISC_VEC_ID                 RTE_INTR_VEC_ZERO_OFFSET
  28 #define IGC_RX_VEC_START                RTE_INTR_VEC_RXTX_OFFSET
  29 #define IGC_MSIX_OTHER_INTR_VEC         0   /* MSI-X other interrupt vector */
  30 #define IGC_FLAG_NEED_LINK_UPDATE       (1u << 0)       /* need update link */
  31
  32 #define IGC_DEFAULT_RX_FREE_THRESH      32
  33
  34 #define IGC_DEFAULT_RX_PTHRESH          8
  35 #define IGC_DEFAULT_RX_HTHRESH          8
  36 #define IGC_DEFAULT_RX_WTHRESH          4
  37
  38 #define IGC_DEFAULT_TX_PTHRESH          8
  39 #define IGC_DEFAULT_TX_HTHRESH          1
  40 #define IGC_DEFAULT_TX_WTHRESH          16
  41
  42 /* MSI-X other interrupt vector */
  43 #define IGC_MSIX_OTHER_INTR_VEC         0
  44
  45 /* External VLAN Enable bit mask */
  46 #define IGC_CTRL_EXT_EXT_VLAN           (1u << 26)
  47
  48 /* Speed select */
  49 #define IGC_CTRL_SPEED_MASK             (7u << 8)
  50 #define IGC_CTRL_SPEED_2500             (6u << 8)
  51
  52 /* External VLAN Ether Type bit mask and shift */
  53 #define IGC_VET_EXT                     0xFFFF0000
  54 #define IGC_VET_EXT_SHIFT               16
  55
  56 /* Force EEE Auto-negotiation */
  57 #define IGC_EEER_EEE_FRC_AN             (1u << 28)
  58
  59 /* Per Queue Good Packets Received Count */
  60 #define IGC_PQGPRC(idx)         (0x10010 + 0x100 * (idx))
  61 /* Per Queue Good Octets Received Count */
  62 #define IGC_PQGORC(idx)         (0x10018 + 0x100 * (idx))
  63 /* Per Queue Good Octets Transmitted Count */
  64 #define IGC_PQGOTC(idx)         (0x10034 + 0x100 * (idx))
  65 /* Per Queue Multicast Packets Received Count */
  66 #define IGC_PQMPRC(idx)         (0x10038 + 0x100 * (idx))
  67 /* Transmit Queue Drop Packet Count */
  68 #define IGC_TQDPC(idx)          (0xe030 + 0x40 * (idx))
  69
  70 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
  71 #define U32_0_IN_U64            0       /* lower bytes of u64 */
  72 #define U32_1_IN_U64            1       /* higher bytes of u64 */
  73 #else
  74 #define U32_0_IN_U64            1
  75 #define U32_1_IN_U64            0
  76 #endif
  77
  78 #define IGC_ALARM_INTERVAL      8000000u
  79 /* us, about 13.6s some per-queue registers will wrap around back to 0. */
  80
  81 static const struct rte_eth_desc_lim rx_desc_lim = {
  82         .nb_max = IGC_MAX_RXD,
  83         .nb_min = IGC_MIN_RXD,
  84         .nb_align = IGC_RXD_ALIGN,
  85 };
  86
  87 static const struct rte_eth_desc_lim tx_desc_lim = {
  88         .nb_max = IGC_MAX_TXD,
  89         .nb_min = IGC_MIN_TXD,
  90         .nb_align = IGC_TXD_ALIGN,
  91         .nb_seg_max = IGC_TX_MAX_SEG,
  92         .nb_mtu_seg_max = IGC_TX_MAX_MTU_SEG,
  93 };
  94
  95 static const struct rte_pci_id pci_id_igc_map[] = {
  96         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_LM) },
  97         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_V)  },
  98         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_I)  },
  99         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_K)  },
 100         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I226_K)  },
 101         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I226_LMVP)  },
 102         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I226_LM)  },
 103         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I226_V)  },
 104         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I226_IT)  },
 105         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I226_BLANK_NVM)  },
 106         { .vendor_id = 0, /* sentinel */ },
 107 };
 108
 109 /* store statistics names and its offset in stats structure */
 110 struct rte_igc_xstats_name_off {
 111         char name[RTE_ETH_XSTATS_NAME_SIZE];
 112         unsigned int offset;
 113 };
 114
 115 static const struct rte_igc_xstats_name_off rte_igc_stats_strings[] = {
 116         {"rx_crc_errors", offsetof(struct igc_hw_stats, crcerrs)},
 117         {"rx_align_errors", offsetof(struct igc_hw_stats, algnerrc)},
 118         {"rx_errors", offsetof(struct igc_hw_stats, rxerrc)},
 119         {"rx_missed_packets", offsetof(struct igc_hw_stats, mpc)},
 120         {"tx_single_collision_packets", offsetof(struct igc_hw_stats, scc)},
 121         {"tx_multiple_collision_packets", offsetof(struct igc_hw_stats, mcc)},
 122         {"tx_excessive_collision_packets", offsetof(struct igc_hw_stats,
 123                 ecol)},
 124         {"tx_late_collisions", offsetof(struct igc_hw_stats, latecol)},
 125         {"tx_total_collisions", offsetof(struct igc_hw_stats, colc)},
 126         {"tx_deferred_packets", offsetof(struct igc_hw_stats, dc)},
 127         {"tx_no_carrier_sense_packets", offsetof(struct igc_hw_stats, tncrs)},
 128         {"tx_discarded_packets", offsetof(struct igc_hw_stats, htdpmc)},
 129         {"rx_length_errors", offsetof(struct igc_hw_stats, rlec)},
 130         {"rx_xon_packets", offsetof(struct igc_hw_stats, xonrxc)},
 131         {"tx_xon_packets", offsetof(struct igc_hw_stats, xontxc)},
 132         {"rx_xoff_packets", offsetof(struct igc_hw_stats, xoffrxc)},
 133         {"tx_xoff_packets", offsetof(struct igc_hw_stats, xofftxc)},
 134         {"rx_flow_control_unsupported_packets", offsetof(struct igc_hw_stats,
 135                 fcruc)},
 136         {"rx_size_64_packets", offsetof(struct igc_hw_stats, prc64)},
 137         {"rx_size_65_to_127_packets", offsetof(struct igc_hw_stats, prc127)},
 138         {"rx_size_128_to_255_packets", offsetof(struct igc_hw_stats, prc255)},
 139         {"rx_size_256_to_511_packets", offsetof(struct igc_hw_stats, prc511)},
 140         {"rx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 141                 prc1023)},
 142         {"rx_size_1024_to_max_packets", offsetof(struct igc_hw_stats,
 143                 prc1522)},
 144         {"rx_broadcast_packets", offsetof(struct igc_hw_stats, bprc)},
 145         {"rx_multicast_packets", offsetof(struct igc_hw_stats, mprc)},
 146         {"rx_undersize_errors", offsetof(struct igc_hw_stats, ruc)},
 147         {"rx_fragment_errors", offsetof(struct igc_hw_stats, rfc)},
 148         {"rx_oversize_errors", offsetof(struct igc_hw_stats, roc)},
 149         {"rx_jabber_errors", offsetof(struct igc_hw_stats, rjc)},
 150         {"rx_no_buffers", offsetof(struct igc_hw_stats, rnbc)},
 151         {"rx_management_packets", offsetof(struct igc_hw_stats, mgprc)},
 152         {"rx_management_dropped", offsetof(struct igc_hw_stats, mgpdc)},
 153         {"tx_management_packets", offsetof(struct igc_hw_stats, mgptc)},
 154         {"rx_total_packets", offsetof(struct igc_hw_stats, tpr)},
 155         {"tx_total_packets", offsetof(struct igc_hw_stats, tpt)},
 156         {"rx_total_bytes", offsetof(struct igc_hw_stats, tor)},
 157         {"tx_total_bytes", offsetof(struct igc_hw_stats, tot)},
 158         {"tx_size_64_packets", offsetof(struct igc_hw_stats, ptc64)},
 159         {"tx_size_65_to_127_packets", offsetof(struct igc_hw_stats, ptc127)},
 160         {"tx_size_128_to_255_packets", offsetof(struct igc_hw_stats, ptc255)},
 161         {"tx_size_256_to_511_packets", offsetof(struct igc_hw_stats, ptc511)},
 162         {"tx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 163                 ptc1023)},
 164         {"tx_size_1023_to_max_packets", offsetof(struct igc_hw_stats,
 165                 ptc1522)},
 166         {"tx_multicast_packets", offsetof(struct igc_hw_stats, mptc)},
 167         {"tx_broadcast_packets", offsetof(struct igc_hw_stats, bptc)},
 168         {"tx_tso_packets", offsetof(struct igc_hw_stats, tsctc)},
 169         {"rx_sent_to_host_packets", offsetof(struct igc_hw_stats, rpthc)},
 170         {"tx_sent_by_host_packets", offsetof(struct igc_hw_stats, hgptc)},
 171         {"interrupt_assert_count", offsetof(struct igc_hw_stats, iac)},
 172         {"rx_descriptor_lower_threshold",
 173                 offsetof(struct igc_hw_stats, icrxdmtc)},
 174 };
 175
 176 #define IGC_NB_XSTATS (sizeof(rte_igc_stats_strings) / \
 177                 sizeof(rte_igc_stats_strings[0]))
 178
 179 static int eth_igc_configure(struct rte_eth_dev *dev);
 180 static int eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
 181 static int eth_igc_stop(struct rte_eth_dev *dev);
 182 static int eth_igc_start(struct rte_eth_dev *dev);
 183 static int eth_igc_set_link_up(struct rte_eth_dev *dev);
 184 static int eth_igc_set_link_down(struct rte_eth_dev *dev);
 185 static int eth_igc_close(struct rte_eth_dev *dev);
 186 static int eth_igc_reset(struct rte_eth_dev *dev);
 187 static int eth_igc_promiscuous_enable(struct rte_eth_dev *dev);
 188 static int eth_igc_promiscuous_disable(struct rte_eth_dev *dev);
 189 static int eth_igc_fw_version_get(struct rte_eth_dev *dev,
 190                                 char *fw_version, size_t fw_size);
 191 static int eth_igc_infos_get(struct rte_eth_dev *dev,
 192                         struct rte_eth_dev_info *dev_info);
 193 static int eth_igc_led_on(struct rte_eth_dev *dev);
 194 static int eth_igc_led_off(struct rte_eth_dev *dev);
 195 static const uint32_t *eth_igc_supported_ptypes_get(struct rte_eth_dev *dev);
 196 static int eth_igc_rar_set(struct rte_eth_dev *dev,
 197                 struct rte_ether_addr *mac_addr, uint32_t index, uint32_t pool);
 198 static void eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 199 static int eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
 200                         struct rte_ether_addr *addr);
 201 static int eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
 202                          struct rte_ether_addr *mc_addr_set,
 203                          uint32_t nb_mc_addr);
 204 static int eth_igc_allmulticast_enable(struct rte_eth_dev *dev);
 205 static int eth_igc_allmulticast_disable(struct rte_eth_dev *dev);
 206 static int eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 207 static int eth_igc_stats_get(struct rte_eth_dev *dev,
 208                         struct rte_eth_stats *rte_stats);
 209 static int eth_igc_xstats_get(struct rte_eth_dev *dev,
 210                         struct rte_eth_xstat *xstats, unsigned int n);
 211 static int eth_igc_xstats_get_by_id(struct rte_eth_dev *dev,
 212                                 const uint64_t *ids,
 213                                 uint64_t *values, unsigned int n);
 214 static int eth_igc_xstats_get_names(struct rte_eth_dev *dev,
 215                                 struct rte_eth_xstat_name *xstats_names,
 216                                 unsigned int size);
 217 static int eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 218                 const uint64_t *ids, struct rte_eth_xstat_name *xstats_names,
 219                 unsigned int limit);
 220 static int eth_igc_xstats_reset(struct rte_eth_dev *dev);
 221 static int
 222 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
 223         uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx);
 224 static int
 225 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 226 static int
 227 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 228 static int
 229 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 230 static int
 231 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 232 static int eth_igc_rss_reta_update(struct rte_eth_dev *dev,
 233                         struct rte_eth_rss_reta_entry64 *reta_conf,
 234                         uint16_t reta_size);
 235 static int eth_igc_rss_reta_query(struct rte_eth_dev *dev,
 236                        struct rte_eth_rss_reta_entry64 *reta_conf,
 237                        uint16_t reta_size);
 238 static int eth_igc_rss_hash_update(struct rte_eth_dev *dev,
 239                         struct rte_eth_rss_conf *rss_conf);
 240 static int eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
 241                         struct rte_eth_rss_conf *rss_conf);
 242 static int
 243 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 244 static int eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 245 static int eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
 246                       enum rte_vlan_type vlan_type, uint16_t tpid);
 247
 248 static const struct eth_dev_ops eth_igc_ops = {
 249         .dev_configure          = eth_igc_configure,
 250         .link_update            = eth_igc_link_update,
 251         .dev_stop               = eth_igc_stop,
 252         .dev_start              = eth_igc_start,
 253         .dev_close              = eth_igc_close,
 254         .dev_reset              = eth_igc_reset,
 255         .dev_set_link_up        = eth_igc_set_link_up,
 256         .dev_set_link_down      = eth_igc_set_link_down,
 257         .promiscuous_enable     = eth_igc_promiscuous_enable,
 258         .promiscuous_disable    = eth_igc_promiscuous_disable,
 259         .allmulticast_enable    = eth_igc_allmulticast_enable,
 260         .allmulticast_disable   = eth_igc_allmulticast_disable,
 261         .fw_version_get         = eth_igc_fw_version_get,
 262         .dev_infos_get          = eth_igc_infos_get,
 263         .dev_led_on             = eth_igc_led_on,
 264         .dev_led_off            = eth_igc_led_off,
 265         .dev_supported_ptypes_get = eth_igc_supported_ptypes_get,
 266         .mtu_set                = eth_igc_mtu_set,
 267         .mac_addr_add           = eth_igc_rar_set,
 268         .mac_addr_remove        = eth_igc_rar_clear,
 269         .mac_addr_set           = eth_igc_default_mac_addr_set,
 270         .set_mc_addr_list       = eth_igc_set_mc_addr_list,
 271
 272         .rx_queue_setup         = eth_igc_rx_queue_setup,
 273         .rx_queue_release       = eth_igc_rx_queue_release,
 274         .tx_queue_setup         = eth_igc_tx_queue_setup,
 275         .tx_queue_release       = eth_igc_tx_queue_release,
 276         .tx_done_cleanup        = eth_igc_tx_done_cleanup,
 277         .rxq_info_get           = eth_igc_rxq_info_get,
 278         .txq_info_get           = eth_igc_txq_info_get,
 279         .stats_get              = eth_igc_stats_get,
 280         .xstats_get             = eth_igc_xstats_get,
 281         .xstats_get_by_id       = eth_igc_xstats_get_by_id,
 282         .xstats_get_names_by_id = eth_igc_xstats_get_names_by_id,
 283         .xstats_get_names       = eth_igc_xstats_get_names,
 284         .stats_reset            = eth_igc_xstats_reset,
 285         .xstats_reset           = eth_igc_xstats_reset,
 286         .queue_stats_mapping_set = eth_igc_queue_stats_mapping_set,
 287         .rx_queue_intr_enable   = eth_igc_rx_queue_intr_enable,
 288         .rx_queue_intr_disable  = eth_igc_rx_queue_intr_disable,
 289         .flow_ctrl_get          = eth_igc_flow_ctrl_get,
 290         .flow_ctrl_set          = eth_igc_flow_ctrl_set,
 291         .reta_update            = eth_igc_rss_reta_update,
 292         .reta_query             = eth_igc_rss_reta_query,
 293         .rss_hash_update        = eth_igc_rss_hash_update,
 294         .rss_hash_conf_get      = eth_igc_rss_hash_conf_get,
 295         .vlan_filter_set        = eth_igc_vlan_filter_set,
 296         .vlan_offload_set       = eth_igc_vlan_offload_set,
 297         .vlan_tpid_set          = eth_igc_vlan_tpid_set,
 298         .vlan_strip_queue_set   = eth_igc_vlan_strip_queue_set,
 299         .flow_ops_get           = eth_igc_flow_ops_get,
 300 };
 301
 302 /*
 303  * multiple queue mode checking
 304  */
 305 static int
 306 igc_check_mq_mode(struct rte_eth_dev *dev)
 307 {
 308         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 309         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
 310
 311         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 312                 PMD_INIT_LOG(ERR, "SRIOV is not supported.");
 313                 return -EINVAL;
 314         }
 315
 316         if (rx_mq_mode != RTE_ETH_MQ_RX_NONE &&
 317                 rx_mq_mode != RTE_ETH_MQ_RX_RSS) {
 318                 /* RSS together with VMDq not supported*/
 319                 PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
 320                                 rx_mq_mode);
 321                 return -EINVAL;
 322         }
 323
 324         /* To no break software that set invalid mode, only display
 325          * warning if invalid mode is used.
 326          */
 327         if (tx_mq_mode != RTE_ETH_MQ_TX_NONE)
 328                 PMD_INIT_LOG(WARNING,
 329                         "TX mode %d is not supported. Due to meaningless in this driver, just ignore",
 330                         tx_mq_mode);
 331
 332         return 0;
 333 }
 334
 335 static int
 336 eth_igc_configure(struct rte_eth_dev *dev)
 337 {
 338         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 339         int ret;
 340
 341         PMD_INIT_FUNC_TRACE();
 342
 343         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
 344                 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
 345
 346         ret  = igc_check_mq_mode(dev);
 347         if (ret != 0)
 348                 return ret;
 349
 350         intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 351         return 0;
 352 }
 353
 354 static int
 355 eth_igc_set_link_up(struct rte_eth_dev *dev)
 356 {
 357         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 358
 359         if (hw->phy.media_type == igc_media_type_copper)
 360                 igc_power_up_phy(hw);
 361         else
 362                 igc_power_up_fiber_serdes_link(hw);
 363         return 0;
 364 }
 365
 366 static int
 367 eth_igc_set_link_down(struct rte_eth_dev *dev)
 368 {
 369         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 370
 371         if (hw->phy.media_type == igc_media_type_copper)
 372                 igc_power_down_phy(hw);
 373         else
 374                 igc_shutdown_fiber_serdes_link(hw);
 375         return 0;
 376 }
 377
 378 /*
 379  * disable other interrupt
 380  */
 381 static void
 382 igc_intr_other_disable(struct rte_eth_dev *dev)
 383 {
 384         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 385         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 386         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 387
 388         if (rte_intr_allow_others(intr_handle) &&
 389                 dev->data->dev_conf.intr_conf.lsc) {
 390                 IGC_WRITE_REG(hw, IGC_EIMC, 1u << IGC_MSIX_OTHER_INTR_VEC);
 391         }
 392
 393         IGC_WRITE_REG(hw, IGC_IMC, ~0);
 394         IGC_WRITE_FLUSH(hw);
 395 }
 396
 397 /*
 398  * enable other interrupt
 399  */
 400 static inline void
 401 igc_intr_other_enable(struct rte_eth_dev *dev)
 402 {
 403         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 404         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 405         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 406         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 407
 408         if (rte_intr_allow_others(intr_handle) &&
 409                 dev->data->dev_conf.intr_conf.lsc) {
 410                 IGC_WRITE_REG(hw, IGC_EIMS, 1u << IGC_MSIX_OTHER_INTR_VEC);
 411         }
 412
 413         IGC_WRITE_REG(hw, IGC_IMS, intr->mask);
 414         IGC_WRITE_FLUSH(hw);
 415 }
 416
 417 /*
 418  * It reads ICR and gets interrupt causes, check it and set a bit flag
 419  * to update link status.
 420  */
 421 static void
 422 eth_igc_interrupt_get_status(struct rte_eth_dev *dev)
 423 {
 424         uint32_t icr;
 425         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 426         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 427
 428         /* read-on-clear nic registers here */
 429         icr = IGC_READ_REG(hw, IGC_ICR);
 430
 431         intr->flags = 0;
 432         if (icr & IGC_ICR_LSC)
 433                 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 434 }
 435
 436 /* return 0 means link status changed, -1 means not changed */
 437 static int
 438 eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 439 {
 440         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 441         struct rte_eth_link link;
 442         int link_check, count;
 443
 444         link_check = 0;
 445         hw->mac.get_link_status = 1;
 446
 447         /* possible wait-to-complete in up to 9 seconds */
 448         for (count = 0; count < IGC_LINK_UPDATE_CHECK_TIMEOUT; count++) {
 449                 /* Read the real link status */
 450                 switch (hw->phy.media_type) {
 451                 case igc_media_type_copper:
 452                         /* Do the work to read phy */
 453                         igc_check_for_link(hw);
 454                         link_check = !hw->mac.get_link_status;
 455                         break;
 456
 457                 case igc_media_type_fiber:
 458                         igc_check_for_link(hw);
 459                         link_check = (IGC_READ_REG(hw, IGC_STATUS) &
 460                                       IGC_STATUS_LU);
 461                         break;
 462
 463                 case igc_media_type_internal_serdes:
 464                         igc_check_for_link(hw);
 465                         link_check = hw->mac.serdes_has_link;
 466                         break;
 467
 468                 default:
 469                         break;
 470                 }
 471                 if (link_check || wait_to_complete == 0)
 472                         break;
 473                 rte_delay_ms(IGC_LINK_UPDATE_CHECK_INTERVAL);
 474         }
 475         memset(&link, 0, sizeof(link));
 476
 477         /* Now we check if a transition has happened */
 478         if (link_check) {
 479                 uint16_t duplex, speed;
 480                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
 481                 link.link_duplex = (duplex == FULL_DUPLEX) ?
 482                                 RTE_ETH_LINK_FULL_DUPLEX :
 483                                 RTE_ETH_LINK_HALF_DUPLEX;
 484                 link.link_speed = speed;
 485                 link.link_status = RTE_ETH_LINK_UP;
 486                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 487                                 RTE_ETH_LINK_SPEED_FIXED);
 488
 489                 if (speed == SPEED_2500) {
 490                         uint32_t tipg = IGC_READ_REG(hw, IGC_TIPG);
 491                         if ((tipg & IGC_TIPG_IPGT_MASK) != 0x0b) {
 492                                 tipg &= ~IGC_TIPG_IPGT_MASK;
 493                                 tipg |= 0x0b;
 494                                 IGC_WRITE_REG(hw, IGC_TIPG, tipg);
 495                         }
 496                 }
 497         } else {
 498                 link.link_speed = 0;
 499                 link.link_duplex = RTE_ETH_LINK_HALF_DUPLEX;
 500                 link.link_status = RTE_ETH_LINK_DOWN;
 501                 link.link_autoneg = RTE_ETH_LINK_FIXED;
 502         }
 503
 504         return rte_eth_linkstatus_set(dev, &link);
 505 }
 506
 507 /*
 508  * It executes link_update after knowing an interrupt is present.
 509  */
 510 static void
 511 eth_igc_interrupt_action(struct rte_eth_dev *dev)
 512 {
 513         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 514         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 515         struct rte_eth_link link;
 516         int ret;
 517
 518         if (intr->flags & IGC_FLAG_NEED_LINK_UPDATE) {
 519                 intr->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
 520
 521                 /* set get_link_status to check register later */
 522                 ret = eth_igc_link_update(dev, 0);
 523
 524                 /* check if link has changed */
 525                 if (ret < 0)
 526                         return;
 527
 528                 rte_eth_linkstatus_get(dev, &link);
 529                 if (link.link_status)
 530                         PMD_DRV_LOG(INFO,
 531                                 " Port %d: Link Up - speed %u Mbps - %s",
 532                                 dev->data->port_id,
 533                                 (unsigned int)link.link_speed,
 534                                 link.link_duplex == RTE_ETH_LINK_FULL_DUPLEX ?
 535                                 "full-duplex" : "half-duplex");
 536                 else
 537                         PMD_DRV_LOG(INFO, " Port %d: Link Down",
 538                                 dev->data->port_id);
 539
 540                 PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
 541                                 pci_dev->addr.domain,
 542                                 pci_dev->addr.bus,
 543                                 pci_dev->addr.devid,
 544                                 pci_dev->addr.function);
 545                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 546         }
 547 }
 548
 549 /*
 550  * Interrupt handler which shall be registered at first.
 551  *
 552  * @handle
 553  *  Pointer to interrupt handle.
 554  * @param
 555  *  The address of parameter (struct rte_eth_dev *) registered before.
 556  */
 557 static void
 558 eth_igc_interrupt_handler(void *param)
 559 {
 560         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 561
 562         eth_igc_interrupt_get_status(dev);
 563         eth_igc_interrupt_action(dev);
 564 }
 565
 566 static void igc_read_queue_stats_register(struct rte_eth_dev *dev);
 567
 568 /*
 569  * Update the queue status every IGC_ALARM_INTERVAL time.
 570  * @param
 571  *  The address of parameter (struct rte_eth_dev *) registered before.
 572  */
 573 static void
 574 igc_update_queue_stats_handler(void *param)
 575 {
 576         struct rte_eth_dev *dev = param;
 577         igc_read_queue_stats_register(dev);
 578         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
 579                         igc_update_queue_stats_handler, dev);
 580 }
 581
 582 /*
 583  * rx,tx enable/disable
 584  */
 585 static void
 586 eth_igc_rxtx_control(struct rte_eth_dev *dev, bool enable)
 587 {
 588         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 589         uint32_t tctl, rctl;
 590
 591         tctl = IGC_READ_REG(hw, IGC_TCTL);
 592         rctl = IGC_READ_REG(hw, IGC_RCTL);
 593
 594         if (enable) {
 595                 /* enable Tx/Rx */
 596                 tctl |= IGC_TCTL_EN;
 597                 rctl |= IGC_RCTL_EN;
 598         } else {
 599                 /* disable Tx/Rx */
 600                 tctl &= ~IGC_TCTL_EN;
 601                 rctl &= ~IGC_RCTL_EN;
 602         }
 603         IGC_WRITE_REG(hw, IGC_TCTL, tctl);
 604         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
 605         IGC_WRITE_FLUSH(hw);
 606 }
 607
 608 /*
 609  *  This routine disables all traffic on the adapter by issuing a
 610  *  global reset on the MAC.
 611  */
 612 static int
 613 eth_igc_stop(struct rte_eth_dev *dev)
 614 {
 615         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 616         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 617         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 618         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 619         struct rte_eth_link link;
 620
 621         dev->data->dev_started = 0;
 622         adapter->stopped = 1;
 623
 624         /* disable receive and transmit */
 625         eth_igc_rxtx_control(dev, false);
 626
 627         /* disable all MSI-X interrupts */
 628         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 629         IGC_WRITE_FLUSH(hw);
 630
 631         /* clear all MSI-X interrupts */
 632         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 633
 634         igc_intr_other_disable(dev);
 635
 636         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
 637
 638         /* disable intr eventfd mapping */
 639         rte_intr_disable(intr_handle);
 640
 641         igc_reset_hw(hw);
 642
 643         /* disable all wake up */
 644         IGC_WRITE_REG(hw, IGC_WUC, 0);
 645
 646         /* disable checking EEE operation in MAC loopback mode */
 647         igc_read_reg_check_clear_bits(hw, IGC_EEER, IGC_EEER_EEE_FRC_AN);
 648
 649         /* Set bit for Go Link disconnect */
 650         igc_read_reg_check_set_bits(hw, IGC_82580_PHY_POWER_MGMT,
 651                         IGC_82580_PM_GO_LINKD);
 652
 653         /* Power down the phy. Needed to make the link go Down */
 654         eth_igc_set_link_down(dev);
 655
 656         igc_dev_clear_queues(dev);
 657
 658         /* clear the recorded link status */
 659         memset(&link, 0, sizeof(link));
 660         rte_eth_linkstatus_set(dev, &link);
 661
 662         if (!rte_intr_allow_others(intr_handle))
 663                 /* resume to the default handler */
 664                 rte_intr_callback_register(intr_handle,
 665                                            eth_igc_interrupt_handler,
 666                                            (void *)dev);
 667
 668         /* Clean datapath event and queue/vec mapping */
 669         rte_intr_efd_disable(intr_handle);
 670         rte_intr_vec_list_free(intr_handle);
 671
 672         return 0;
 673 }
 674
 675 /*
 676  * write interrupt vector allocation register
 677  * @hw
 678  *  board private structure
 679  * @queue_index
 680  *  queue index, valid 0,1,2,3
 681  * @tx
 682  *  tx:1, rx:0
 683  * @msix_vector
 684  *  msix-vector, valid 0,1,2,3,4
 685  */
 686 static void
 687 igc_write_ivar(struct igc_hw *hw, uint8_t queue_index,
 688                 bool tx, uint8_t msix_vector)
 689 {
 690         uint8_t offset = 0;
 691         uint8_t reg_index = queue_index >> 1;
 692         uint32_t val;
 693
 694         /*
 695          * IVAR(0)
 696          * bit31...24   bit23...16      bit15...8       bit7...0
 697          * TX1          RX1             TX0             RX0
 698          *
 699          * IVAR(1)
 700          * bit31...24   bit23...16      bit15...8       bit7...0
 701          * TX3          RX3             TX2             RX2
 702          */
 703
 704         if (tx)
 705                 offset = 8;
 706
 707         if (queue_index & 1)
 708                 offset += 16;
 709
 710         val = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, reg_index);
 711
 712         /* clear bits */
 713         val &= ~((uint32_t)0xFF << offset);
 714
 715         /* write vector and valid bit */
 716         val |= (uint32_t)(msix_vector | IGC_IVAR_VALID) << offset;
 717
 718         IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, reg_index, val);
 719 }
 720
 721 /* Sets up the hardware to generate MSI-X interrupts properly
 722  * @hw
 723  *  board private structure
 724  */
 725 static void
 726 igc_configure_msix_intr(struct rte_eth_dev *dev)
 727 {
 728         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 729         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 730         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 731
 732         uint32_t intr_mask;
 733         uint32_t vec = IGC_MISC_VEC_ID;
 734         uint32_t base = IGC_MISC_VEC_ID;
 735         uint32_t misc_shift = 0;
 736         int i, nb_efd;
 737
 738         /* won't configure msix register if no mapping is done
 739          * between intr vector and event fd
 740          */
 741         if (!rte_intr_dp_is_en(intr_handle))
 742                 return;
 743
 744         if (rte_intr_allow_others(intr_handle)) {
 745                 base = IGC_RX_VEC_START;
 746                 vec = base;
 747                 misc_shift = 1;
 748         }
 749
 750         /* turn on MSI-X capability first */
 751         IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE |
 752                                 IGC_GPIE_PBA | IGC_GPIE_EIAME |
 753                                 IGC_GPIE_NSICR);
 754
 755         nb_efd = rte_intr_nb_efd_get(intr_handle);
 756         if (nb_efd < 0)
 757                 return;
 758
 759         intr_mask = RTE_LEN2MASK(nb_efd, uint32_t) << misc_shift;
 760
 761         if (dev->data->dev_conf.intr_conf.lsc)
 762                 intr_mask |= (1u << IGC_MSIX_OTHER_INTR_VEC);
 763
 764         /* enable msix auto-clear */
 765         igc_read_reg_check_set_bits(hw, IGC_EIAC, intr_mask);
 766
 767         /* set other cause interrupt vector */
 768         igc_read_reg_check_set_bits(hw, IGC_IVAR_MISC,
 769                 (uint32_t)(IGC_MSIX_OTHER_INTR_VEC | IGC_IVAR_VALID) << 8);
 770
 771         /* enable auto-mask */
 772         igc_read_reg_check_set_bits(hw, IGC_EIAM, intr_mask);
 773
 774         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 775                 igc_write_ivar(hw, i, 0, vec);
 776                 rte_intr_vec_list_index_set(intr_handle, i, vec);
 777                 if (vec < base + rte_intr_nb_efd_get(intr_handle) - 1)
 778                         vec++;
 779         }
 780
 781         IGC_WRITE_FLUSH(hw);
 782 }
 783
 784 /**
 785  * It enables the interrupt mask and then enable the interrupt.
 786  *
 787  * @dev
 788  *  Pointer to struct rte_eth_dev.
 789  * @on
 790  *  Enable or Disable
 791  */
 792 static void
 793 igc_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
 794 {
 795         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 796
 797         if (on)
 798                 intr->mask |= IGC_ICR_LSC;
 799         else
 800                 intr->mask &= ~IGC_ICR_LSC;
 801 }
 802
 803 /*
 804  * It enables the interrupt.
 805  * It will be called once only during nic initialized.
 806  */
 807 static void
 808 igc_rxq_interrupt_setup(struct rte_eth_dev *dev)
 809 {
 810         uint32_t mask;
 811         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 812         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 813         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 814         int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0;
 815         int nb_efd;
 816
 817         /* won't configure msix register if no mapping is done
 818          * between intr vector and event fd
 819          */
 820         if (!rte_intr_dp_is_en(intr_handle))
 821                 return;
 822
 823         nb_efd = rte_intr_nb_efd_get(intr_handle);
 824         if (nb_efd < 0)
 825                 return;
 826
 827         mask = RTE_LEN2MASK(nb_efd, uint32_t) << misc_shift;
 828         IGC_WRITE_REG(hw, IGC_EIMS, mask);
 829 }
 830
 831 /*
 832  *  Get hardware rx-buffer size.
 833  */
 834 static inline int
 835 igc_get_rx_buffer_size(struct igc_hw *hw)
 836 {
 837         return (IGC_READ_REG(hw, IGC_RXPBS) & 0x3f) << 10;
 838 }
 839
 840 /*
 841  * igc_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
 842  * For ASF and Pass Through versions of f/w this means
 843  * that the driver is loaded.
 844  */
 845 static void
 846 igc_hw_control_acquire(struct igc_hw *hw)
 847 {
 848         uint32_t ctrl_ext;
 849
 850         /* Let firmware know the driver has taken over */
 851         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 852         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 853 }
 854
 855 /*
 856  * igc_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
 857  * For ASF and Pass Through versions of f/w this means that the
 858  * driver is no longer loaded.
 859  */
 860 static void
 861 igc_hw_control_release(struct igc_hw *hw)
 862 {
 863         uint32_t ctrl_ext;
 864
 865         /* Let firmware taken over control of h/w */
 866         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 867         IGC_WRITE_REG(hw, IGC_CTRL_EXT,
 868                         ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 869 }
 870
 871 static int
 872 igc_hardware_init(struct igc_hw *hw)
 873 {
 874         uint32_t rx_buf_size;
 875         int diag;
 876
 877         /* Let the firmware know the OS is in control */
 878         igc_hw_control_acquire(hw);
 879
 880         /* Issue a global reset */
 881         igc_reset_hw(hw);
 882
 883         /* disable all wake up */
 884         IGC_WRITE_REG(hw, IGC_WUC, 0);
 885
 886         /*
 887          * Hardware flow control
 888          * - High water mark should allow for at least two standard size (1518)
 889          *   frames to be received after sending an XOFF.
 890          * - Low water mark works best when it is very near the high water mark.
 891          *   This allows the receiver to restart by sending XON when it has
 892          *   drained a bit. Here we use an arbitrary value of 1500 which will
 893          *   restart after one full frame is pulled from the buffer. There
 894          *   could be several smaller frames in the buffer and if so they will
 895          *   not trigger the XON until their total number reduces the buffer
 896          *   by 1500.
 897          */
 898         rx_buf_size = igc_get_rx_buffer_size(hw);
 899         hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2);
 900         hw->fc.low_water = hw->fc.high_water - 1500;
 901         hw->fc.pause_time = IGC_FC_PAUSE_TIME;
 902         hw->fc.send_xon = 1;
 903         hw->fc.requested_mode = igc_fc_full;
 904
 905         diag = igc_init_hw(hw);
 906         if (diag < 0)
 907                 return diag;
 908
 909         igc_get_phy_info(hw);
 910         igc_check_for_link(hw);
 911
 912         return 0;
 913 }
 914
 915 static int
 916 eth_igc_start(struct rte_eth_dev *dev)
 917 {
 918         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 919         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 920         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 921         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 922         uint32_t *speeds;
 923         int ret;
 924
 925         PMD_INIT_FUNC_TRACE();
 926
 927         /* disable all MSI-X interrupts */
 928         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 929         IGC_WRITE_FLUSH(hw);
 930
 931         /* clear all MSI-X interrupts */
 932         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 933
 934         /* disable uio/vfio intr/eventfd mapping */
 935         if (!adapter->stopped)
 936                 rte_intr_disable(intr_handle);
 937
 938         /* Power up the phy. Needed to make the link go Up */
 939         eth_igc_set_link_up(dev);
 940
 941         /* Put the address into the Receive Address Array */
 942         igc_rar_set(hw, hw->mac.addr, 0);
 943
 944         /* Initialize the hardware */
 945         if (igc_hardware_init(hw)) {
 946                 PMD_DRV_LOG(ERR, "Unable to initialize the hardware");
 947                 return -EIO;
 948         }
 949         adapter->stopped = 0;
 950
 951         /* check and configure queue intr-vector mapping */
 952         if (rte_intr_cap_multiple(intr_handle) &&
 953                 dev->data->dev_conf.intr_conf.rxq) {
 954                 uint32_t intr_vector = dev->data->nb_rx_queues;
 955                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 956                         return -1;
 957         }
 958
 959         if (rte_intr_dp_is_en(intr_handle)) {
 960                 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec",
 961                                                   dev->data->nb_rx_queues)) {
 962                         PMD_DRV_LOG(ERR,
 963                                 "Failed to allocate %d rx_queues intr_vec",
 964                                 dev->data->nb_rx_queues);
 965                         return -ENOMEM;
 966                 }
 967         }
 968
 969         /* configure msix for rx interrupt */
 970         igc_configure_msix_intr(dev);
 971
 972         igc_tx_init(dev);
 973
 974         /* This can fail when allocating mbufs for descriptor rings */
 975         ret = igc_rx_init(dev);
 976         if (ret) {
 977                 PMD_DRV_LOG(ERR, "Unable to initialize RX hardware");
 978                 igc_dev_clear_queues(dev);
 979                 return ret;
 980         }
 981
 982         igc_clear_hw_cntrs_base_generic(hw);
 983
 984         /* VLAN Offload Settings */
 985         eth_igc_vlan_offload_set(dev,
 986                 RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK |
 987                 RTE_ETH_VLAN_EXTEND_MASK);
 988
 989         /* Setup link speed and duplex */
 990         speeds = &dev->data->dev_conf.link_speeds;
 991         if (*speeds == RTE_ETH_LINK_SPEED_AUTONEG) {
 992                 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
 993                 hw->mac.autoneg = 1;
 994         } else {
 995                 int num_speeds = 0;
 996
 997                 if (*speeds & RTE_ETH_LINK_SPEED_FIXED) {
 998                         PMD_DRV_LOG(ERR,
 999                                     "Force speed mode currently not supported");
1000                         igc_dev_clear_queues(dev);
1001                         return -EINVAL;
1002                 }
1003
1004                 hw->phy.autoneg_advertised = 0;
1005                 hw->mac.autoneg = 1;
1006
1007                 if (*speeds & ~(RTE_ETH_LINK_SPEED_10M_HD | RTE_ETH_LINK_SPEED_10M |
1008                                 RTE_ETH_LINK_SPEED_100M_HD | RTE_ETH_LINK_SPEED_100M |
1009                                 RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_2_5G)) {
1010                         num_speeds = -1;
1011                         goto error_invalid_config;
1012                 }
1013                 if (*speeds & RTE_ETH_LINK_SPEED_10M_HD) {
1014                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
1015                         num_speeds++;
1016                 }
1017                 if (*speeds & RTE_ETH_LINK_SPEED_10M) {
1018                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
1019                         num_speeds++;
1020                 }
1021                 if (*speeds & RTE_ETH_LINK_SPEED_100M_HD) {
1022                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1023                         num_speeds++;
1024                 }
1025                 if (*speeds & RTE_ETH_LINK_SPEED_100M) {
1026                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1027                         num_speeds++;
1028                 }
1029                 if (*speeds & RTE_ETH_LINK_SPEED_1G) {
1030                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1031                         num_speeds++;
1032                 }
1033                 if (*speeds & RTE_ETH_LINK_SPEED_2_5G) {
1034                         hw->phy.autoneg_advertised |= ADVERTISE_2500_FULL;
1035                         num_speeds++;
1036                 }
1037                 if (num_speeds == 0)
1038                         goto error_invalid_config;
1039         }
1040
1041         igc_setup_link(hw);
1042
1043         if (rte_intr_allow_others(intr_handle)) {
1044                 /* check if lsc interrupt is enabled */
1045                 if (dev->data->dev_conf.intr_conf.lsc)
1046                         igc_lsc_interrupt_setup(dev, 1);
1047                 else
1048                         igc_lsc_interrupt_setup(dev, 0);
1049         } else {
1050                 rte_intr_callback_unregister(intr_handle,
1051                                              eth_igc_interrupt_handler,
1052                                              (void *)dev);
1053                 if (dev->data->dev_conf.intr_conf.lsc)
1054                         PMD_DRV_LOG(INFO,
1055                                 "LSC won't enable because of no intr multiplex");
1056         }
1057
1058         /* enable uio/vfio intr/eventfd mapping */
1059         rte_intr_enable(intr_handle);
1060
1061         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1062                         igc_update_queue_stats_handler, dev);
1063
1064         /* check if rxq interrupt is enabled */
1065         if (dev->data->dev_conf.intr_conf.rxq &&
1066                         rte_intr_dp_is_en(intr_handle))
1067                 igc_rxq_interrupt_setup(dev);
1068
1069         /* resume enabled intr since hw reset */
1070         igc_intr_other_enable(dev);
1071
1072         eth_igc_rxtx_control(dev, true);
1073         eth_igc_link_update(dev, 0);
1074
1075         /* configure MAC-loopback mode */
1076         if (dev->data->dev_conf.lpbk_mode == 1) {
1077                 uint32_t reg_val;
1078
1079                 reg_val = IGC_READ_REG(hw, IGC_CTRL);
1080                 reg_val &= ~IGC_CTRL_SPEED_MASK;
1081                 reg_val |= IGC_CTRL_SLU | IGC_CTRL_FRCSPD |
1082                         IGC_CTRL_FRCDPX | IGC_CTRL_FD | IGC_CTRL_SPEED_2500;
1083                 IGC_WRITE_REG(hw, IGC_CTRL, reg_val);
1084
1085                 igc_read_reg_check_set_bits(hw, IGC_EEER, IGC_EEER_EEE_FRC_AN);
1086         }
1087
1088         return 0;
1089
1090 error_invalid_config:
1091         PMD_DRV_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1092                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1093         igc_dev_clear_queues(dev);
1094         return -EINVAL;
1095 }
1096
1097 static int
1098 igc_reset_swfw_lock(struct igc_hw *hw)
1099 {
1100         int ret_val;
1101
1102         /*
1103          * Do mac ops initialization manually here, since we will need
1104          * some function pointers set by this call.
1105          */
1106         ret_val = igc_init_mac_params(hw);
1107         if (ret_val)
1108                 return ret_val;
1109
1110         /*
1111          * SMBI lock should not fail in this early stage. If this is the case,
1112          * it is due to an improper exit of the application.
1113          * So force the release of the faulty lock.
1114          */
1115         if (igc_get_hw_semaphore_generic(hw) < 0)
1116                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
1117
1118         igc_put_hw_semaphore_generic(hw);
1119
1120         if (hw->mac.ops.acquire_swfw_sync != NULL) {
1121                 uint16_t mask;
1122
1123                 /*
1124                  * Phy lock should not fail in this early stage.
1125                  * If this is the case, it is due to an improper exit of the
1126                  * application. So force the release of the faulty lock.
1127                  */
1128                 mask = IGC_SWFW_PHY0_SM;
1129                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
1130                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
1131                                     hw->bus.func);
1132                 }
1133                 hw->mac.ops.release_swfw_sync(hw, mask);
1134
1135                 /*
1136                  * This one is more tricky since it is common to all ports; but
1137                  * swfw_sync retries last long enough (1s) to be almost sure
1138                  * that if lock can not be taken it is due to an improper lock
1139                  * of the semaphore.
1140                  */
1141                 mask = IGC_SWFW_EEP_SM;
1142                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0)
1143                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
1144
1145                 hw->mac.ops.release_swfw_sync(hw, mask);
1146         }
1147
1148         return IGC_SUCCESS;
1149 }
1150
1151 /*
1152  * free all rx/tx queues.
1153  */
1154 static void
1155 igc_dev_free_queues(struct rte_eth_dev *dev)
1156 {
1157         uint16_t i;
1158
1159         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1160                 eth_igc_rx_queue_release(dev, i);
1161                 dev->data->rx_queues[i] = NULL;
1162         }
1163         dev->data->nb_rx_queues = 0;
1164
1165         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1166                 eth_igc_tx_queue_release(dev, i);
1167                 dev->data->tx_queues[i] = NULL;
1168         }
1169         dev->data->nb_tx_queues = 0;
1170 }
1171
1172 static int
1173 eth_igc_close(struct rte_eth_dev *dev)
1174 {
1175         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1176         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1177         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1178         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
1179         int retry = 0;
1180         int ret = 0;
1181
1182         PMD_INIT_FUNC_TRACE();
1183         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1184                 return 0;
1185
1186         if (!adapter->stopped)
1187                 ret = eth_igc_stop(dev);
1188
1189         igc_flow_flush(dev, NULL);
1190         igc_clear_all_filter(dev);
1191
1192         igc_intr_other_disable(dev);
1193         do {
1194                 int ret = rte_intr_callback_unregister(intr_handle,
1195                                 eth_igc_interrupt_handler, dev);
1196                 if (ret >= 0 || ret == -ENOENT || ret == -EINVAL)
1197                         break;
1198
1199                 PMD_DRV_LOG(ERR, "intr callback unregister failed: %d", ret);
1200                 DELAY(200 * 1000); /* delay 200ms */
1201         } while (retry++ < 5);
1202
1203         igc_phy_hw_reset(hw);
1204         igc_hw_control_release(hw);
1205         igc_dev_free_queues(dev);
1206
1207         /* Reset any pending lock */
1208         igc_reset_swfw_lock(hw);
1209
1210         return ret;
1211 }
1212
1213 static void
1214 igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
1215 {
1216         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1217
1218         hw->vendor_id = pci_dev->id.vendor_id;
1219         hw->device_id = pci_dev->id.device_id;
1220         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1221         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1222 }
1223
1224 static int
1225 eth_igc_dev_init(struct rte_eth_dev *dev)
1226 {
1227         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1228         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1229         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1230         int i, error = 0;
1231
1232         PMD_INIT_FUNC_TRACE();
1233         dev->dev_ops = &eth_igc_ops;
1234         dev->rx_queue_count = eth_igc_rx_queue_count;
1235         dev->rx_descriptor_status = eth_igc_rx_descriptor_status;
1236         dev->tx_descriptor_status = eth_igc_tx_descriptor_status;
1237
1238         /*
1239          * for secondary processes, we don't initialize any further as primary
1240          * has already done this work. Only check we don't need a different
1241          * RX function.
1242          */
1243         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1244                 return 0;
1245
1246         rte_eth_copy_pci_info(dev, pci_dev);
1247         dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
1248
1249         hw->back = pci_dev;
1250         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1251
1252         igc_identify_hardware(dev, pci_dev);
1253         if (igc_setup_init_funcs(hw, false) != IGC_SUCCESS) {
1254                 error = -EIO;
1255                 goto err_late;
1256         }
1257
1258         igc_get_bus_info(hw);
1259
1260         /* Reset any pending lock */
1261         if (igc_reset_swfw_lock(hw) != IGC_SUCCESS) {
1262                 error = -EIO;
1263                 goto err_late;
1264         }
1265
1266         /* Finish initialization */
1267         if (igc_setup_init_funcs(hw, true) != IGC_SUCCESS) {
1268                 error = -EIO;
1269                 goto err_late;
1270         }
1271
1272         hw->mac.autoneg = 1;
1273         hw->phy.autoneg_wait_to_complete = 0;
1274         hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
1275
1276         /* Copper options */
1277         if (hw->phy.media_type == igc_media_type_copper) {
1278                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1279                 hw->phy.disable_polarity_correction = 0;
1280                 hw->phy.ms_type = igc_ms_hw_default;
1281         }
1282
1283         /*
1284          * Start from a known state, this is important in reading the nvm
1285          * and mac from that.
1286          */
1287         igc_reset_hw(hw);
1288
1289         /* Make sure we have a good EEPROM before we read from it */
1290         if (igc_validate_nvm_checksum(hw) < 0) {
1291                 /*
1292                  * Some PCI-E parts fail the first check due to
1293                  * the link being in sleep state, call it again,
1294                  * if it fails a second time its a real issue.
1295                  */
1296                 if (igc_validate_nvm_checksum(hw) < 0) {
1297                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
1298                         error = -EIO;
1299                         goto err_late;
1300                 }
1301         }
1302
1303         /* Read the permanent MAC address out of the EEPROM */
1304         if (igc_read_mac_addr(hw) != 0) {
1305                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
1306                 error = -EIO;
1307                 goto err_late;
1308         }
1309
1310         /* Allocate memory for storing MAC addresses */
1311         dev->data->mac_addrs = rte_zmalloc("igc",
1312                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
1313         if (dev->data->mac_addrs == NULL) {
1314                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes for storing MAC",
1315                                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1316                 error = -ENOMEM;
1317                 goto err_late;
1318         }
1319
1320         /* Copy the permanent MAC address */
1321         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
1322                         &dev->data->mac_addrs[0]);
1323
1324         /* Now initialize the hardware */
1325         if (igc_hardware_init(hw) != 0) {
1326                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
1327                 rte_free(dev->data->mac_addrs);
1328                 dev->data->mac_addrs = NULL;
1329                 error = -ENODEV;
1330                 goto err_late;
1331         }
1332
1333         hw->mac.get_link_status = 1;
1334         igc->stopped = 0;
1335
1336         /* Indicate SOL/IDER usage */
1337         if (igc_check_reset_block(hw) < 0)
1338                 PMD_INIT_LOG(ERR,
1339                         "PHY reset is blocked due to SOL/IDER session.");
1340
1341         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
1342                         dev->data->port_id, pci_dev->id.vendor_id,
1343                         pci_dev->id.device_id);
1344
1345         rte_intr_callback_register(pci_dev->intr_handle,
1346                         eth_igc_interrupt_handler, (void *)dev);
1347
1348         /* enable uio/vfio intr/eventfd mapping */
1349         rte_intr_enable(pci_dev->intr_handle);
1350
1351         /* enable support intr */
1352         igc_intr_other_enable(dev);
1353
1354         /* initiate queue status */
1355         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1356                 igc->txq_stats_map[i] = -1;
1357                 igc->rxq_stats_map[i] = -1;
1358         }
1359
1360         igc_flow_init(dev);
1361         igc_clear_all_filter(dev);
1362         return 0;
1363
1364 err_late:
1365         igc_hw_control_release(hw);
1366         return error;
1367 }
1368
1369 static int
1370 eth_igc_dev_uninit(__rte_unused struct rte_eth_dev *eth_dev)
1371 {
1372         PMD_INIT_FUNC_TRACE();
1373         eth_igc_close(eth_dev);
1374         return 0;
1375 }
1376
1377 static int
1378 eth_igc_reset(struct rte_eth_dev *dev)
1379 {
1380         int ret;
1381
1382         PMD_INIT_FUNC_TRACE();
1383
1384         ret = eth_igc_dev_uninit(dev);
1385         if (ret)
1386                 return ret;
1387
1388         return eth_igc_dev_init(dev);
1389 }
1390
1391 static int
1392 eth_igc_promiscuous_enable(struct rte_eth_dev *dev)
1393 {
1394         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1395         uint32_t rctl;
1396
1397         rctl = IGC_READ_REG(hw, IGC_RCTL);
1398         rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1399         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1400         return 0;
1401 }
1402
1403 static int
1404 eth_igc_promiscuous_disable(struct rte_eth_dev *dev)
1405 {
1406         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1407         uint32_t rctl;
1408
1409         rctl = IGC_READ_REG(hw, IGC_RCTL);
1410         rctl &= (~IGC_RCTL_UPE);
1411         if (dev->data->all_multicast == 1)
1412                 rctl |= IGC_RCTL_MPE;
1413         else
1414                 rctl &= (~IGC_RCTL_MPE);
1415         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1416         return 0;
1417 }
1418
1419 static int
1420 eth_igc_allmulticast_enable(struct rte_eth_dev *dev)
1421 {
1422         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1423         uint32_t rctl;
1424
1425         rctl = IGC_READ_REG(hw, IGC_RCTL);
1426         rctl |= IGC_RCTL_MPE;
1427         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1428         return 0;
1429 }
1430
1431 static int
1432 eth_igc_allmulticast_disable(struct rte_eth_dev *dev)
1433 {
1434         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1435         uint32_t rctl;
1436
1437         if (dev->data->promiscuous == 1)
1438                 return 0;       /* must remain in all_multicast mode */
1439
1440         rctl = IGC_READ_REG(hw, IGC_RCTL);
1441         rctl &= (~IGC_RCTL_MPE);
1442         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1443         return 0;
1444 }
1445
1446 static int
1447 eth_igc_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
1448                        size_t fw_size)
1449 {
1450         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1451         struct igc_fw_version fw;
1452         int ret;
1453
1454         igc_get_fw_version(hw, &fw);
1455
1456         /* if option rom is valid, display its version too */
1457         if (fw.or_valid) {
1458                 ret = snprintf(fw_version, fw_size,
1459                          "%d.%d, 0x%08x, %d.%d.%d",
1460                          fw.eep_major, fw.eep_minor, fw.etrack_id,
1461                          fw.or_major, fw.or_build, fw.or_patch);
1462         /* no option rom */
1463         } else {
1464                 if (fw.etrack_id != 0X0000) {
1465                         ret = snprintf(fw_version, fw_size,
1466                                  "%d.%d, 0x%08x",
1467                                  fw.eep_major, fw.eep_minor,
1468                                  fw.etrack_id);
1469                 } else {
1470                         ret = snprintf(fw_version, fw_size,
1471                                  "%d.%d.%d",
1472                                  fw.eep_major, fw.eep_minor,
1473                                  fw.eep_build);
1474                 }
1475         }
1476         if (ret < 0)
1477                 return -EINVAL;
1478
1479         ret += 1; /* add the size of '\0' */
1480         if (fw_size < (size_t)ret)
1481                 return ret;
1482         else
1483                 return 0;
1484 }
1485
1486 static int
1487 eth_igc_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1488 {
1489         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1490
1491         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1492         dev_info->max_rx_pktlen = MAX_RX_JUMBO_FRAME_SIZE;
1493         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1494         dev_info->dev_capa &= ~RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP;
1495         dev_info->rx_offload_capa = IGC_RX_OFFLOAD_ALL;
1496         dev_info->tx_offload_capa = IGC_TX_OFFLOAD_ALL;
1497         dev_info->rx_queue_offload_capa = RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
1498
1499         dev_info->max_rx_queues = IGC_QUEUE_PAIRS_NUM;
1500         dev_info->max_tx_queues = IGC_QUEUE_PAIRS_NUM;
1501         dev_info->max_vmdq_pools = 0;
1502
1503         dev_info->hash_key_size = IGC_HKEY_MAX_INDEX * sizeof(uint32_t);
1504         dev_info->reta_size = RTE_ETH_RSS_RETA_SIZE_128;
1505         dev_info->flow_type_rss_offloads = IGC_RSS_OFFLOAD_ALL;
1506
1507         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1508                 .rx_thresh = {
1509                         .pthresh = IGC_DEFAULT_RX_PTHRESH,
1510                         .hthresh = IGC_DEFAULT_RX_HTHRESH,
1511                         .wthresh = IGC_DEFAULT_RX_WTHRESH,
1512                 },
1513                 .rx_free_thresh = IGC_DEFAULT_RX_FREE_THRESH,
1514                 .rx_drop_en = 0,
1515                 .offloads = 0,
1516         };
1517
1518         dev_info->default_txconf = (struct rte_eth_txconf) {
1519                 .tx_thresh = {
1520                         .pthresh = IGC_DEFAULT_TX_PTHRESH,
1521                         .hthresh = IGC_DEFAULT_TX_HTHRESH,
1522                         .wthresh = IGC_DEFAULT_TX_WTHRESH,
1523                 },
1524                 .offloads = 0,
1525         };
1526
1527         dev_info->rx_desc_lim = rx_desc_lim;
1528         dev_info->tx_desc_lim = tx_desc_lim;
1529
1530         dev_info->speed_capa = RTE_ETH_LINK_SPEED_10M_HD | RTE_ETH_LINK_SPEED_10M |
1531                         RTE_ETH_LINK_SPEED_100M_HD | RTE_ETH_LINK_SPEED_100M |
1532                         RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_2_5G;
1533
1534         dev_info->max_mtu = dev_info->max_rx_pktlen - IGC_ETH_OVERHEAD;
1535         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
1536         return 0;
1537 }
1538
1539 static int
1540 eth_igc_led_on(struct rte_eth_dev *dev)
1541 {
1542         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1543
1544         return igc_led_on(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1545 }
1546
1547 static int
1548 eth_igc_led_off(struct rte_eth_dev *dev)
1549 {
1550         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1551
1552         return igc_led_off(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1553 }
1554
1555 static const uint32_t *
1556 eth_igc_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev)
1557 {
1558         static const uint32_t ptypes[] = {
1559                 /* refers to rx_desc_pkt_info_to_pkt_type() */
1560                 RTE_PTYPE_L2_ETHER,
1561                 RTE_PTYPE_L3_IPV4,
1562                 RTE_PTYPE_L3_IPV4_EXT,
1563                 RTE_PTYPE_L3_IPV6,
1564                 RTE_PTYPE_L3_IPV6_EXT,
1565                 RTE_PTYPE_L4_TCP,
1566                 RTE_PTYPE_L4_UDP,
1567                 RTE_PTYPE_L4_SCTP,
1568                 RTE_PTYPE_TUNNEL_IP,
1569                 RTE_PTYPE_INNER_L3_IPV6,
1570                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1571                 RTE_PTYPE_INNER_L4_TCP,
1572                 RTE_PTYPE_INNER_L4_UDP,
1573                 RTE_PTYPE_UNKNOWN
1574         };
1575
1576         return ptypes;
1577 }
1578
1579 static int
1580 eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1581 {
1582         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1583         uint32_t frame_size = mtu + IGC_ETH_OVERHEAD;
1584         uint32_t rctl;
1585
1586         /* if extend vlan has been enabled */
1587         if (IGC_READ_REG(hw, IGC_CTRL_EXT) & IGC_CTRL_EXT_EXT_VLAN)
1588                 frame_size += VLAN_TAG_SIZE;
1589
1590         /*
1591          * If device is started, refuse mtu that requires the support of
1592          * scattered packets when this feature has not been enabled before.
1593          */
1594         if (dev->data->dev_started && !dev->data->scattered_rx &&
1595             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM) {
1596                 PMD_INIT_LOG(ERR, "Stop port first.");
1597                 return -EINVAL;
1598         }
1599
1600         rctl = IGC_READ_REG(hw, IGC_RCTL);
1601         if (mtu > RTE_ETHER_MTU)
1602                 rctl |= IGC_RCTL_LPE;
1603         else
1604                 rctl &= ~IGC_RCTL_LPE;
1605         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1606
1607         IGC_WRITE_REG(hw, IGC_RLPML, frame_size);
1608
1609         return 0;
1610 }
1611
1612 static int
1613 eth_igc_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1614                 uint32_t index, uint32_t pool)
1615 {
1616         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1617
1618         igc_rar_set(hw, mac_addr->addr_bytes, index);
1619         RTE_SET_USED(pool);
1620         return 0;
1621 }
1622
1623 static void
1624 eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1625 {
1626         uint8_t addr[RTE_ETHER_ADDR_LEN];
1627         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1628
1629         memset(addr, 0, sizeof(addr));
1630         igc_rar_set(hw, addr, index);
1631 }
1632
1633 static int
1634 eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
1635                         struct rte_ether_addr *addr)
1636 {
1637         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1638         igc_rar_set(hw, addr->addr_bytes, 0);
1639         return 0;
1640 }
1641
1642 static int
1643 eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
1644                          struct rte_ether_addr *mc_addr_set,
1645                          uint32_t nb_mc_addr)
1646 {
1647         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1648         igc_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
1649         return 0;
1650 }
1651
1652 /*
1653  * Read hardware registers
1654  */
1655 static void
1656 igc_read_stats_registers(struct igc_hw *hw, struct igc_hw_stats *stats)
1657 {
1658         int pause_frames;
1659
1660         uint64_t old_gprc  = stats->gprc;
1661         uint64_t old_gptc  = stats->gptc;
1662         uint64_t old_tpr   = stats->tpr;
1663         uint64_t old_tpt   = stats->tpt;
1664         uint64_t old_rpthc = stats->rpthc;
1665         uint64_t old_hgptc = stats->hgptc;
1666
1667         stats->crcerrs += IGC_READ_REG(hw, IGC_CRCERRS);
1668         stats->algnerrc += IGC_READ_REG(hw, IGC_ALGNERRC);
1669         stats->rxerrc += IGC_READ_REG(hw, IGC_RXERRC);
1670         stats->mpc += IGC_READ_REG(hw, IGC_MPC);
1671         stats->scc += IGC_READ_REG(hw, IGC_SCC);
1672         stats->ecol += IGC_READ_REG(hw, IGC_ECOL);
1673
1674         stats->mcc += IGC_READ_REG(hw, IGC_MCC);
1675         stats->latecol += IGC_READ_REG(hw, IGC_LATECOL);
1676         stats->colc += IGC_READ_REG(hw, IGC_COLC);
1677
1678         stats->dc += IGC_READ_REG(hw, IGC_DC);
1679         stats->tncrs += IGC_READ_REG(hw, IGC_TNCRS);
1680         stats->htdpmc += IGC_READ_REG(hw, IGC_HTDPMC);
1681         stats->rlec += IGC_READ_REG(hw, IGC_RLEC);
1682         stats->xonrxc += IGC_READ_REG(hw, IGC_XONRXC);
1683         stats->xontxc += IGC_READ_REG(hw, IGC_XONTXC);
1684
1685         /*
1686          * For watchdog management we need to know if we have been
1687          * paused during the last interval, so capture that here.
1688          */
1689         pause_frames = IGC_READ_REG(hw, IGC_XOFFRXC);
1690         stats->xoffrxc += pause_frames;
1691         stats->xofftxc += IGC_READ_REG(hw, IGC_XOFFTXC);
1692         stats->fcruc += IGC_READ_REG(hw, IGC_FCRUC);
1693         stats->prc64 += IGC_READ_REG(hw, IGC_PRC64);
1694         stats->prc127 += IGC_READ_REG(hw, IGC_PRC127);
1695         stats->prc255 += IGC_READ_REG(hw, IGC_PRC255);
1696         stats->prc511 += IGC_READ_REG(hw, IGC_PRC511);
1697         stats->prc1023 += IGC_READ_REG(hw, IGC_PRC1023);
1698         stats->prc1522 += IGC_READ_REG(hw, IGC_PRC1522);
1699         stats->gprc += IGC_READ_REG(hw, IGC_GPRC);
1700         stats->bprc += IGC_READ_REG(hw, IGC_BPRC);
1701         stats->mprc += IGC_READ_REG(hw, IGC_MPRC);
1702         stats->gptc += IGC_READ_REG(hw, IGC_GPTC);
1703
1704         /* For the 64-bit byte counters the low dword must be read first. */
1705         /* Both registers clear on the read of the high dword */
1706
1707         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1708         stats->gorc += IGC_READ_REG(hw, IGC_GORCL);
1709         stats->gorc += ((uint64_t)IGC_READ_REG(hw, IGC_GORCH) << 32);
1710         stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN;
1711         stats->gotc += IGC_READ_REG(hw, IGC_GOTCL);
1712         stats->gotc += ((uint64_t)IGC_READ_REG(hw, IGC_GOTCH) << 32);
1713         stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN;
1714
1715         stats->rnbc += IGC_READ_REG(hw, IGC_RNBC);
1716         stats->ruc += IGC_READ_REG(hw, IGC_RUC);
1717         stats->rfc += IGC_READ_REG(hw, IGC_RFC);
1718         stats->roc += IGC_READ_REG(hw, IGC_ROC);
1719         stats->rjc += IGC_READ_REG(hw, IGC_RJC);
1720
1721         stats->mgprc += IGC_READ_REG(hw, IGC_MGTPRC);
1722         stats->mgpdc += IGC_READ_REG(hw, IGC_MGTPDC);
1723         stats->mgptc += IGC_READ_REG(hw, IGC_MGTPTC);
1724         stats->b2ospc += IGC_READ_REG(hw, IGC_B2OSPC);
1725         stats->b2ogprc += IGC_READ_REG(hw, IGC_B2OGPRC);
1726         stats->o2bgptc += IGC_READ_REG(hw, IGC_O2BGPTC);
1727         stats->o2bspc += IGC_READ_REG(hw, IGC_O2BSPC);
1728
1729         stats->tpr += IGC_READ_REG(hw, IGC_TPR);
1730         stats->tpt += IGC_READ_REG(hw, IGC_TPT);
1731
1732         stats->tor += IGC_READ_REG(hw, IGC_TORL);
1733         stats->tor += ((uint64_t)IGC_READ_REG(hw, IGC_TORH) << 32);
1734         stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN;
1735         stats->tot += IGC_READ_REG(hw, IGC_TOTL);
1736         stats->tot += ((uint64_t)IGC_READ_REG(hw, IGC_TOTH) << 32);
1737         stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN;
1738
1739         stats->ptc64 += IGC_READ_REG(hw, IGC_PTC64);
1740         stats->ptc127 += IGC_READ_REG(hw, IGC_PTC127);
1741         stats->ptc255 += IGC_READ_REG(hw, IGC_PTC255);
1742         stats->ptc511 += IGC_READ_REG(hw, IGC_PTC511);
1743         stats->ptc1023 += IGC_READ_REG(hw, IGC_PTC1023);
1744         stats->ptc1522 += IGC_READ_REG(hw, IGC_PTC1522);
1745         stats->mptc += IGC_READ_REG(hw, IGC_MPTC);
1746         stats->bptc += IGC_READ_REG(hw, IGC_BPTC);
1747         stats->tsctc += IGC_READ_REG(hw, IGC_TSCTC);
1748
1749         stats->iac += IGC_READ_REG(hw, IGC_IAC);
1750         stats->rpthc += IGC_READ_REG(hw, IGC_RPTHC);
1751         stats->hgptc += IGC_READ_REG(hw, IGC_HGPTC);
1752         stats->icrxdmtc += IGC_READ_REG(hw, IGC_ICRXDMTC);
1753
1754         /* Host to Card Statistics */
1755         stats->hgorc += IGC_READ_REG(hw, IGC_HGORCL);
1756         stats->hgorc += ((uint64_t)IGC_READ_REG(hw, IGC_HGORCH) << 32);
1757         stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN;
1758         stats->hgotc += IGC_READ_REG(hw, IGC_HGOTCL);
1759         stats->hgotc += ((uint64_t)IGC_READ_REG(hw, IGC_HGOTCH) << 32);
1760         stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN;
1761         stats->lenerrs += IGC_READ_REG(hw, IGC_LENERRS);
1762 }
1763
1764 /*
1765  * Write 0 to all queue status registers
1766  */
1767 static void
1768 igc_reset_queue_stats_register(struct igc_hw *hw)
1769 {
1770         int i;
1771
1772         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1773                 IGC_WRITE_REG(hw, IGC_PQGPRC(i), 0);
1774                 IGC_WRITE_REG(hw, IGC_PQGPTC(i), 0);
1775                 IGC_WRITE_REG(hw, IGC_PQGORC(i), 0);
1776                 IGC_WRITE_REG(hw, IGC_PQGOTC(i), 0);
1777                 IGC_WRITE_REG(hw, IGC_PQMPRC(i), 0);
1778                 IGC_WRITE_REG(hw, IGC_RQDPC(i), 0);
1779                 IGC_WRITE_REG(hw, IGC_TQDPC(i), 0);
1780         }
1781 }
1782
1783 /*
1784  * Read all hardware queue status registers
1785  */
1786 static void
1787 igc_read_queue_stats_register(struct rte_eth_dev *dev)
1788 {
1789         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1790         struct igc_hw_queue_stats *queue_stats =
1791                                 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1792         int i;
1793
1794         /*
1795          * This register is not cleared on read. Furthermore, the register wraps
1796          * around back to 0x00000000 on the next increment when reaching a value
1797          * of 0xFFFFFFFF and then continues normal count operation.
1798          */
1799         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1800                 union {
1801                         u64 ddword;
1802                         u32 dword[2];
1803                 } value;
1804                 u32 tmp;
1805
1806                 /*
1807                  * Read the register first, if the value is smaller than that
1808                  * previous read, that mean the register has been overflowed,
1809                  * then we add the high 4 bytes by 1 and replace the low 4
1810                  * bytes by the new value.
1811                  */
1812                 tmp = IGC_READ_REG(hw, IGC_PQGPRC(i));
1813                 value.ddword = queue_stats->pqgprc[i];
1814                 if (value.dword[U32_0_IN_U64] > tmp)
1815                         value.dword[U32_1_IN_U64]++;
1816                 value.dword[U32_0_IN_U64] = tmp;
1817                 queue_stats->pqgprc[i] = value.ddword;
1818
1819                 tmp = IGC_READ_REG(hw, IGC_PQGPTC(i));
1820                 value.ddword = queue_stats->pqgptc[i];
1821                 if (value.dword[U32_0_IN_U64] > tmp)
1822                         value.dword[U32_1_IN_U64]++;
1823                 value.dword[U32_0_IN_U64] = tmp;
1824                 queue_stats->pqgptc[i] = value.ddword;
1825
1826                 tmp = IGC_READ_REG(hw, IGC_PQGORC(i));
1827                 value.ddword = queue_stats->pqgorc[i];
1828                 if (value.dword[U32_0_IN_U64] > tmp)
1829                         value.dword[U32_1_IN_U64]++;
1830                 value.dword[U32_0_IN_U64] = tmp;
1831                 queue_stats->pqgorc[i] = value.ddword;
1832
1833                 tmp = IGC_READ_REG(hw, IGC_PQGOTC(i));
1834                 value.ddword = queue_stats->pqgotc[i];
1835                 if (value.dword[U32_0_IN_U64] > tmp)
1836                         value.dword[U32_1_IN_U64]++;
1837                 value.dword[U32_0_IN_U64] = tmp;
1838                 queue_stats->pqgotc[i] = value.ddword;
1839
1840                 tmp = IGC_READ_REG(hw, IGC_PQMPRC(i));
1841                 value.ddword = queue_stats->pqmprc[i];
1842                 if (value.dword[U32_0_IN_U64] > tmp)
1843                         value.dword[U32_1_IN_U64]++;
1844                 value.dword[U32_0_IN_U64] = tmp;
1845                 queue_stats->pqmprc[i] = value.ddword;
1846
1847                 tmp = IGC_READ_REG(hw, IGC_RQDPC(i));
1848                 value.ddword = queue_stats->rqdpc[i];
1849                 if (value.dword[U32_0_IN_U64] > tmp)
1850                         value.dword[U32_1_IN_U64]++;
1851                 value.dword[U32_0_IN_U64] = tmp;
1852                 queue_stats->rqdpc[i] = value.ddword;
1853
1854                 tmp = IGC_READ_REG(hw, IGC_TQDPC(i));
1855                 value.ddword = queue_stats->tqdpc[i];
1856                 if (value.dword[U32_0_IN_U64] > tmp)
1857                         value.dword[U32_1_IN_U64]++;
1858                 value.dword[U32_0_IN_U64] = tmp;
1859                 queue_stats->tqdpc[i] = value.ddword;
1860         }
1861 }
1862
1863 static int
1864 eth_igc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1865 {
1866         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1867         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1868         struct igc_hw_stats *stats = IGC_DEV_PRIVATE_STATS(dev);
1869         struct igc_hw_queue_stats *queue_stats =
1870                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1871         int i;
1872
1873         /*
1874          * Cancel status handler since it will read the queue status registers
1875          */
1876         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1877
1878         /* Read status register */
1879         igc_read_queue_stats_register(dev);
1880         igc_read_stats_registers(hw, stats);
1881
1882         if (rte_stats == NULL) {
1883                 /* Restart queue status handler */
1884                 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1885                                 igc_update_queue_stats_handler, dev);
1886                 return -EINVAL;
1887         }
1888
1889         /* Rx Errors */
1890         rte_stats->imissed = stats->mpc;
1891         rte_stats->ierrors = stats->crcerrs + stats->rlec +
1892                         stats->rxerrc + stats->algnerrc;
1893
1894         /* Tx Errors */
1895         rte_stats->oerrors = stats->ecol + stats->latecol;
1896
1897         rte_stats->ipackets = stats->gprc;
1898         rte_stats->opackets = stats->gptc;
1899         rte_stats->ibytes   = stats->gorc;
1900         rte_stats->obytes   = stats->gotc;
1901
1902         /* Get per-queue statuses */
1903         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1904                 /* GET TX queue statuses */
1905                 int map_id = igc->txq_stats_map[i];
1906                 if (map_id >= 0) {
1907                         rte_stats->q_opackets[map_id] += queue_stats->pqgptc[i];
1908                         rte_stats->q_obytes[map_id] += queue_stats->pqgotc[i];
1909                 }
1910                 /* Get RX queue statuses */
1911                 map_id = igc->rxq_stats_map[i];
1912                 if (map_id >= 0) {
1913                         rte_stats->q_ipackets[map_id] += queue_stats->pqgprc[i];
1914                         rte_stats->q_ibytes[map_id] += queue_stats->pqgorc[i];
1915                         rte_stats->q_errors[map_id] += queue_stats->rqdpc[i];
1916                 }
1917         }
1918
1919         /* Restart queue status handler */
1920         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1921                         igc_update_queue_stats_handler, dev);
1922         return 0;
1923 }
1924
1925 static int
1926 eth_igc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1927                    unsigned int n)
1928 {
1929         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1930         struct igc_hw_stats *hw_stats =
1931                         IGC_DEV_PRIVATE_STATS(dev);
1932         unsigned int i;
1933
1934         igc_read_stats_registers(hw, hw_stats);
1935
1936         if (n < IGC_NB_XSTATS)
1937                 return IGC_NB_XSTATS;
1938
1939         /* If this is a reset xstats is NULL, and we have cleared the
1940          * registers by reading them.
1941          */
1942         if (!xstats)
1943                 return 0;
1944
1945         /* Extended stats */
1946         for (i = 0; i < IGC_NB_XSTATS; i++) {
1947                 xstats[i].id = i;
1948                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1949                         rte_igc_stats_strings[i].offset);
1950         }
1951
1952         return IGC_NB_XSTATS;
1953 }
1954
1955 static int
1956 eth_igc_xstats_reset(struct rte_eth_dev *dev)
1957 {
1958         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1959         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1960         struct igc_hw_queue_stats *queue_stats =
1961                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1962
1963         /* Cancel queue status handler for avoid conflict */
1964         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1965
1966         /* HW registers are cleared on read */
1967         igc_reset_queue_stats_register(hw);
1968         igc_read_stats_registers(hw, hw_stats);
1969
1970         /* Reset software totals */
1971         memset(hw_stats, 0, sizeof(*hw_stats));
1972         memset(queue_stats, 0, sizeof(*queue_stats));
1973
1974         /* Restart the queue status handler */
1975         rte_eal_alarm_set(IGC_ALARM_INTERVAL, igc_update_queue_stats_handler,
1976                         dev);
1977
1978         return 0;
1979 }
1980
1981 static int
1982 eth_igc_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1983         struct rte_eth_xstat_name *xstats_names, unsigned int size)
1984 {
1985         unsigned int i;
1986
1987         if (xstats_names == NULL)
1988                 return IGC_NB_XSTATS;
1989
1990         if (size < IGC_NB_XSTATS) {
1991                 PMD_DRV_LOG(ERR, "not enough buffers!");
1992                 return IGC_NB_XSTATS;
1993         }
1994
1995         for (i = 0; i < IGC_NB_XSTATS; i++)
1996                 strlcpy(xstats_names[i].name, rte_igc_stats_strings[i].name,
1997                         sizeof(xstats_names[i].name));
1998
1999         return IGC_NB_XSTATS;
2000 }
2001
2002 static int
2003 eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
2004                 const uint64_t *ids, struct rte_eth_xstat_name *xstats_names,
2005                 unsigned int limit)
2006 {
2007         unsigned int i;
2008
2009         if (!ids)
2010                 return eth_igc_xstats_get_names(dev, xstats_names, limit);
2011
2012         for (i = 0; i < limit; i++) {
2013                 if (ids[i] >= IGC_NB_XSTATS) {
2014                         PMD_DRV_LOG(ERR, "id value isn't valid");
2015                         return -EINVAL;
2016                 }
2017                 strlcpy(xstats_names[i].name,
2018                         rte_igc_stats_strings[ids[i]].name,
2019                         sizeof(xstats_names[i].name));
2020         }
2021         return limit;
2022 }
2023
2024 static int
2025 eth_igc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
2026                 uint64_t *values, unsigned int n)
2027 {
2028         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2029         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
2030         unsigned int i;
2031
2032         igc_read_stats_registers(hw, hw_stats);
2033
2034         if (!ids) {
2035                 if (n < IGC_NB_XSTATS)
2036                         return IGC_NB_XSTATS;
2037
2038                 /* If this is a reset xstats is NULL, and we have cleared the
2039                  * registers by reading them.
2040                  */
2041                 if (!values)
2042                         return 0;
2043
2044                 /* Extended stats */
2045                 for (i = 0; i < IGC_NB_XSTATS; i++)
2046                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2047                                         rte_igc_stats_strings[i].offset);
2048
2049                 return IGC_NB_XSTATS;
2050
2051         } else {
2052                 for (i = 0; i < n; i++) {
2053                         if (ids[i] >= IGC_NB_XSTATS) {
2054                                 PMD_DRV_LOG(ERR, "id value isn't valid");
2055                                 return -EINVAL;
2056                         }
2057                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2058                                         rte_igc_stats_strings[ids[i]].offset);
2059                 }
2060                 return n;
2061         }
2062 }
2063
2064 static int
2065 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
2066                 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx)
2067 {
2068         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
2069
2070         /* check queue id is valid */
2071         if (queue_id >= IGC_QUEUE_PAIRS_NUM) {
2072                 PMD_DRV_LOG(ERR, "queue id(%u) error, max is %u",
2073                         queue_id, IGC_QUEUE_PAIRS_NUM - 1);
2074                 return -EINVAL;
2075         }
2076
2077         /* store the mapping status id */
2078         if (is_rx)
2079                 igc->rxq_stats_map[queue_id] = stat_idx;
2080         else
2081                 igc->txq_stats_map[queue_id] = stat_idx;
2082
2083         return 0;
2084 }
2085
2086 static int
2087 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2088 {
2089         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2090         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2091         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
2092         uint32_t vec = IGC_MISC_VEC_ID;
2093
2094         if (rte_intr_allow_others(intr_handle))
2095                 vec = IGC_RX_VEC_START;
2096
2097         uint32_t mask = 1u << (queue_id + vec);
2098
2099         IGC_WRITE_REG(hw, IGC_EIMC, mask);
2100         IGC_WRITE_FLUSH(hw);
2101
2102         return 0;
2103 }
2104
2105 static int
2106 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2107 {
2108         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2109         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2110         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
2111         uint32_t vec = IGC_MISC_VEC_ID;
2112
2113         if (rte_intr_allow_others(intr_handle))
2114                 vec = IGC_RX_VEC_START;
2115
2116         uint32_t mask = 1u << (queue_id + vec);
2117
2118         IGC_WRITE_REG(hw, IGC_EIMS, mask);
2119         IGC_WRITE_FLUSH(hw);
2120
2121         rte_intr_enable(intr_handle);
2122
2123         return 0;
2124 }
2125
2126 static int
2127 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2128 {
2129         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2130         uint32_t ctrl;
2131         int tx_pause;
2132         int rx_pause;
2133
2134         fc_conf->pause_time = hw->fc.pause_time;
2135         fc_conf->high_water = hw->fc.high_water;
2136         fc_conf->low_water = hw->fc.low_water;
2137         fc_conf->send_xon = hw->fc.send_xon;
2138         fc_conf->autoneg = hw->mac.autoneg;
2139
2140         /*
2141          * Return rx_pause and tx_pause status according to actual setting of
2142          * the TFCE and RFCE bits in the CTRL register.
2143          */
2144         ctrl = IGC_READ_REG(hw, IGC_CTRL);
2145         if (ctrl & IGC_CTRL_TFCE)
2146                 tx_pause = 1;
2147         else
2148                 tx_pause = 0;
2149
2150         if (ctrl & IGC_CTRL_RFCE)
2151                 rx_pause = 1;
2152         else
2153                 rx_pause = 0;
2154
2155         if (rx_pause && tx_pause)
2156                 fc_conf->mode = RTE_ETH_FC_FULL;
2157         else if (rx_pause)
2158                 fc_conf->mode = RTE_ETH_FC_RX_PAUSE;
2159         else if (tx_pause)
2160                 fc_conf->mode = RTE_ETH_FC_TX_PAUSE;
2161         else
2162                 fc_conf->mode = RTE_ETH_FC_NONE;
2163
2164         return 0;
2165 }
2166
2167 static int
2168 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2169 {
2170         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2171         uint32_t rx_buf_size;
2172         uint32_t max_high_water;
2173         uint32_t rctl;
2174         int err;
2175
2176         if (fc_conf->autoneg != hw->mac.autoneg)
2177                 return -ENOTSUP;
2178
2179         rx_buf_size = igc_get_rx_buffer_size(hw);
2180         PMD_DRV_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2181
2182         /* At least reserve one Ethernet frame for watermark */
2183         max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN;
2184         if (fc_conf->high_water > max_high_water ||
2185                 fc_conf->high_water < fc_conf->low_water) {
2186                 PMD_DRV_LOG(ERR,
2187                         "Incorrect high(%u)/low(%u) water value, max is %u",
2188                         fc_conf->high_water, fc_conf->low_water,
2189                         max_high_water);
2190                 return -EINVAL;
2191         }
2192
2193         switch (fc_conf->mode) {
2194         case RTE_ETH_FC_NONE:
2195                 hw->fc.requested_mode = igc_fc_none;
2196                 break;
2197         case RTE_ETH_FC_RX_PAUSE:
2198                 hw->fc.requested_mode = igc_fc_rx_pause;
2199                 break;
2200         case RTE_ETH_FC_TX_PAUSE:
2201                 hw->fc.requested_mode = igc_fc_tx_pause;
2202                 break;
2203         case RTE_ETH_FC_FULL:
2204                 hw->fc.requested_mode = igc_fc_full;
2205                 break;
2206         default:
2207                 PMD_DRV_LOG(ERR, "unsupported fc mode: %u", fc_conf->mode);
2208                 return -EINVAL;
2209         }
2210
2211         hw->fc.pause_time     = fc_conf->pause_time;
2212         hw->fc.high_water     = fc_conf->high_water;
2213         hw->fc.low_water      = fc_conf->low_water;
2214         hw->fc.send_xon       = fc_conf->send_xon;
2215
2216         err = igc_setup_link_generic(hw);
2217         if (err == IGC_SUCCESS) {
2218                 /**
2219                  * check if we want to forward MAC frames - driver doesn't have
2220                  * native capability to do that, so we'll write the registers
2221                  * ourselves
2222                  **/
2223                 rctl = IGC_READ_REG(hw, IGC_RCTL);
2224
2225                 /* set or clear MFLCN.PMCF bit depending on configuration */
2226                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2227                         rctl |= IGC_RCTL_PMCF;
2228                 else
2229                         rctl &= ~IGC_RCTL_PMCF;
2230
2231                 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
2232                 IGC_WRITE_FLUSH(hw);
2233
2234                 return 0;
2235         }
2236
2237         PMD_DRV_LOG(ERR, "igc_setup_link_generic = 0x%x", err);
2238         return -EIO;
2239 }
2240
2241 static int
2242 eth_igc_rss_reta_update(struct rte_eth_dev *dev,
2243                         struct rte_eth_rss_reta_entry64 *reta_conf,
2244                         uint16_t reta_size)
2245 {
2246         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2247         uint16_t i;
2248
2249         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2250                 PMD_DRV_LOG(ERR,
2251                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2252                         reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2253                 return -EINVAL;
2254         }
2255
2256         RTE_BUILD_BUG_ON(RTE_ETH_RSS_RETA_SIZE_128 % IGC_RSS_RDT_REG_SIZE);
2257
2258         /* set redirection table */
2259         for (i = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2260                 union igc_rss_reta_reg reta, reg;
2261                 uint16_t idx, shift;
2262                 uint8_t j, mask;
2263
2264                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2265                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2266                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2267                                 IGC_RSS_RDT_REG_SIZE_MASK);
2268
2269                 /* if no need to update the register */
2270                 if (!mask ||
2271                     shift > (RTE_ETH_RETA_GROUP_SIZE - IGC_RSS_RDT_REG_SIZE))
2272                         continue;
2273
2274                 /* check mask whether need to read the register value first */
2275                 if (mask == IGC_RSS_RDT_REG_SIZE_MASK)
2276                         reg.dword = 0;
2277                 else
2278                         reg.dword = IGC_READ_REG_LE_VALUE(hw,
2279                                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2280
2281                 /* update the register */
2282                 RTE_BUILD_BUG_ON(sizeof(reta.bytes) != IGC_RSS_RDT_REG_SIZE);
2283                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2284                         if (mask & (1u << j))
2285                                 reta.bytes[j] =
2286                                         (uint8_t)reta_conf[idx].reta[shift + j];
2287                         else
2288                                 reta.bytes[j] = reg.bytes[j];
2289                 }
2290                 IGC_WRITE_REG_LE_VALUE(hw,
2291                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE), reta.dword);
2292         }
2293
2294         return 0;
2295 }
2296
2297 static int
2298 eth_igc_rss_reta_query(struct rte_eth_dev *dev,
2299                        struct rte_eth_rss_reta_entry64 *reta_conf,
2300                        uint16_t reta_size)
2301 {
2302         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2303         uint16_t i;
2304
2305         if (reta_size != RTE_ETH_RSS_RETA_SIZE_128) {
2306                 PMD_DRV_LOG(ERR,
2307                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2308                         reta_size, RTE_ETH_RSS_RETA_SIZE_128);
2309                 return -EINVAL;
2310         }
2311
2312         RTE_BUILD_BUG_ON(RTE_ETH_RSS_RETA_SIZE_128 % IGC_RSS_RDT_REG_SIZE);
2313
2314         /* read redirection table */
2315         for (i = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2316                 union igc_rss_reta_reg reta;
2317                 uint16_t idx, shift;
2318                 uint8_t j, mask;
2319
2320                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2321                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2322                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2323                                 IGC_RSS_RDT_REG_SIZE_MASK);
2324
2325                 /* if no need to read register */
2326                 if (!mask ||
2327                     shift > (RTE_ETH_RETA_GROUP_SIZE - IGC_RSS_RDT_REG_SIZE))
2328                         continue;
2329
2330                 /* read register and get the queue index */
2331                 RTE_BUILD_BUG_ON(sizeof(reta.bytes) != IGC_RSS_RDT_REG_SIZE);
2332                 reta.dword = IGC_READ_REG_LE_VALUE(hw,
2333                                 IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2334                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2335                         if (mask & (1u << j))
2336                                 reta_conf[idx].reta[shift + j] = reta.bytes[j];
2337                 }
2338         }
2339
2340         return 0;
2341 }
2342
2343 static int
2344 eth_igc_rss_hash_update(struct rte_eth_dev *dev,
2345                         struct rte_eth_rss_conf *rss_conf)
2346 {
2347         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2348         igc_hw_rss_hash_set(hw, rss_conf);
2349         return 0;
2350 }
2351
2352 static int
2353 eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
2354                         struct rte_eth_rss_conf *rss_conf)
2355 {
2356         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2357         uint32_t *hash_key = (uint32_t *)rss_conf->rss_key;
2358         uint32_t mrqc;
2359         uint64_t rss_hf;
2360
2361         if (hash_key != NULL) {
2362                 int i;
2363
2364                 /* if not enough space for store hash key */
2365                 if (rss_conf->rss_key_len != IGC_HKEY_SIZE) {
2366                         PMD_DRV_LOG(ERR,
2367                                 "RSS hash key size %u in parameter doesn't match the hardware hash key size %u",
2368                                 rss_conf->rss_key_len, IGC_HKEY_SIZE);
2369                         return -EINVAL;
2370                 }
2371
2372                 /* read RSS key from register */
2373                 for (i = 0; i < IGC_HKEY_MAX_INDEX; i++)
2374                         hash_key[i] = IGC_READ_REG_LE_VALUE(hw, IGC_RSSRK(i));
2375         }
2376
2377         /* get RSS functions configured in MRQC register */
2378         mrqc = IGC_READ_REG(hw, IGC_MRQC);
2379         if ((mrqc & IGC_MRQC_ENABLE_RSS_4Q) == 0)
2380                 return 0;
2381
2382         rss_hf = 0;
2383         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4)
2384                 rss_hf |= RTE_ETH_RSS_IPV4;
2385         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_TCP)
2386                 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
2387         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6)
2388                 rss_hf |= RTE_ETH_RSS_IPV6;
2389         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_EX)
2390                 rss_hf |= RTE_ETH_RSS_IPV6_EX;
2391         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP)
2392                 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
2393         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP_EX)
2394                 rss_hf |= RTE_ETH_RSS_IPV6_TCP_EX;
2395         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_UDP)
2396                 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
2397         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP)
2398                 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
2399         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP_EX)
2400                 rss_hf |= RTE_ETH_RSS_IPV6_UDP_EX;
2401
2402         rss_conf->rss_hf |= rss_hf;
2403         return 0;
2404 }
2405
2406 static int
2407 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2408 {
2409         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2410         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2411         uint32_t vfta;
2412         uint32_t vid_idx;
2413         uint32_t vid_bit;
2414
2415         vid_idx = (vlan_id >> IGC_VFTA_ENTRY_SHIFT) & IGC_VFTA_ENTRY_MASK;
2416         vid_bit = 1u << (vlan_id & IGC_VFTA_ENTRY_BIT_SHIFT_MASK);
2417         vfta = shadow_vfta->vfta[vid_idx];
2418         if (on)
2419                 vfta |= vid_bit;
2420         else
2421                 vfta &= ~vid_bit;
2422         IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, vid_idx, vfta);
2423
2424         /* update local VFTA copy */
2425         shadow_vfta->vfta[vid_idx] = vfta;
2426
2427         return 0;
2428 }
2429
2430 static void
2431 igc_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2432 {
2433         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2434         igc_read_reg_check_clear_bits(hw, IGC_RCTL,
2435                         IGC_RCTL_CFIEN | IGC_RCTL_VFE);
2436 }
2437
2438 static void
2439 igc_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2440 {
2441         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2442         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2443         uint32_t reg_val;
2444         int i;
2445
2446         /* Filter Table Enable, CFI not used for packet acceptance */
2447         reg_val = IGC_READ_REG(hw, IGC_RCTL);
2448         reg_val &= ~IGC_RCTL_CFIEN;
2449         reg_val |= IGC_RCTL_VFE;
2450         IGC_WRITE_REG(hw, IGC_RCTL, reg_val);
2451
2452         /* restore VFTA table */
2453         for (i = 0; i < IGC_VFTA_SIZE; i++)
2454                 IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, i, shadow_vfta->vfta[i]);
2455 }
2456
2457 static void
2458 igc_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2459 {
2460         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2461
2462         igc_read_reg_check_clear_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2463 }
2464
2465 static void
2466 igc_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2467 {
2468         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2469
2470         igc_read_reg_check_set_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2471 }
2472
2473 static int
2474 igc_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2475 {
2476         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2477         uint32_t frame_size = dev->data->mtu + IGC_ETH_OVERHEAD;
2478         uint32_t ctrl_ext;
2479
2480         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2481
2482         /* if extend vlan hasn't been enabled */
2483         if ((ctrl_ext & IGC_CTRL_EXT_EXT_VLAN) == 0)
2484                 return 0;
2485
2486         /* Update maximum packet length */
2487         if (frame_size < RTE_ETHER_MIN_MTU + VLAN_TAG_SIZE) {
2488                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, min is %u",
2489                         frame_size, VLAN_TAG_SIZE + RTE_ETHER_MIN_MTU);
2490                 return -EINVAL;
2491         }
2492         IGC_WRITE_REG(hw, IGC_RLPML, frame_size - VLAN_TAG_SIZE);
2493
2494         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_EXT_VLAN);
2495         return 0;
2496 }
2497
2498 static int
2499 igc_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2500 {
2501         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2502         uint32_t frame_size = dev->data->mtu + IGC_ETH_OVERHEAD;
2503         uint32_t ctrl_ext;
2504
2505         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2506
2507         /* if extend vlan has been enabled */
2508         if (ctrl_ext & IGC_CTRL_EXT_EXT_VLAN)
2509                 return 0;
2510
2511         /* Update maximum packet length */
2512         if (frame_size > MAX_RX_JUMBO_FRAME_SIZE) {
2513                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, max is %u",
2514                         frame_size, MAX_RX_JUMBO_FRAME_SIZE);
2515                 return -EINVAL;
2516         }
2517         IGC_WRITE_REG(hw, IGC_RLPML, frame_size);
2518
2519         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_EXT_VLAN);
2520         return 0;
2521 }
2522
2523 static int
2524 eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2525 {
2526         struct rte_eth_rxmode *rxmode;
2527
2528         rxmode = &dev->data->dev_conf.rxmode;
2529         if (mask & RTE_ETH_VLAN_STRIP_MASK) {
2530                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
2531                         igc_vlan_hw_strip_enable(dev);
2532                 else
2533                         igc_vlan_hw_strip_disable(dev);
2534         }
2535
2536         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
2537                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER)
2538                         igc_vlan_hw_filter_enable(dev);
2539                 else
2540                         igc_vlan_hw_filter_disable(dev);
2541         }
2542
2543         if (mask & RTE_ETH_VLAN_EXTEND_MASK) {
2544                 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_EXTEND)
2545                         return igc_vlan_hw_extend_enable(dev);
2546                 else
2547                         return igc_vlan_hw_extend_disable(dev);
2548         }
2549
2550         return 0;
2551 }
2552
2553 static int
2554 eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
2555                       enum rte_vlan_type vlan_type,
2556                       uint16_t tpid)
2557 {
2558         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2559         uint32_t reg_val;
2560
2561         /* only outer TPID of double VLAN can be configured*/
2562         if (vlan_type == RTE_ETH_VLAN_TYPE_OUTER) {
2563                 reg_val = IGC_READ_REG(hw, IGC_VET);
2564                 reg_val = (reg_val & (~IGC_VET_EXT)) |
2565                         ((uint32_t)tpid << IGC_VET_EXT_SHIFT);
2566                 IGC_WRITE_REG(hw, IGC_VET, reg_val);
2567
2568                 return 0;
2569         }
2570
2571         /* all other TPID values are read-only*/
2572         PMD_DRV_LOG(ERR, "Not supported");
2573         return -ENOTSUP;
2574 }
2575
2576 static int
2577 eth_igc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2578         struct rte_pci_device *pci_dev)
2579 {
2580         PMD_INIT_FUNC_TRACE();
2581         return rte_eth_dev_pci_generic_probe(pci_dev,
2582                 sizeof(struct igc_adapter), eth_igc_dev_init);
2583 }
2584
2585 static int
2586 eth_igc_pci_remove(struct rte_pci_device *pci_dev)
2587 {
2588         PMD_INIT_FUNC_TRACE();
2589         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igc_dev_uninit);
2590 }
2591
2592 static struct rte_pci_driver rte_igc_pmd = {
2593         .id_table = pci_id_igc_map,
2594         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2595         .probe = eth_igc_pci_probe,
2596         .remove = eth_igc_pci_remove,
2597 };
2598
2599 RTE_PMD_REGISTER_PCI(net_igc, rte_igc_pmd);
2600 RTE_PMD_REGISTER_PCI_TABLE(net_igc, pci_id_igc_map);
2601 RTE_PMD_REGISTER_KMOD_DEP(net_igc, "* igb_uio | uio_pci_generic | vfio-pci");