drivers/net/igc/igc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <string.h>
   7
   8 #include <rte_string_fns.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_malloc.h>
  14 #include <rte_alarm.h>
  15
  16 #include "igc_logs.h"
  17 #include "igc_txrx.h"
  18
  19 #define IGC_INTEL_VENDOR_ID             0x8086
  20
  21 /*
  22  * The overhead from MTU to max frame size.
  23  * Considering VLAN so tag needs to be counted.
  24  */
  25 #define IGC_ETH_OVERHEAD                (RTE_ETHER_HDR_LEN + \
  26                                         RTE_ETHER_CRC_LEN + VLAN_TAG_SIZE)
  27
  28 #define IGC_FC_PAUSE_TIME               0x0680
  29 #define IGC_LINK_UPDATE_CHECK_TIMEOUT   90  /* 9s */
  30 #define IGC_LINK_UPDATE_CHECK_INTERVAL  100 /* ms */
  31
  32 #define IGC_MISC_VEC_ID                 RTE_INTR_VEC_ZERO_OFFSET
  33 #define IGC_RX_VEC_START                RTE_INTR_VEC_RXTX_OFFSET
  34 #define IGC_MSIX_OTHER_INTR_VEC         0   /* MSI-X other interrupt vector */
  35 #define IGC_FLAG_NEED_LINK_UPDATE       (1u << 0)       /* need update link */
  36
  37 #define IGC_DEFAULT_RX_FREE_THRESH      32
  38
  39 #define IGC_DEFAULT_RX_PTHRESH          8
  40 #define IGC_DEFAULT_RX_HTHRESH          8
  41 #define IGC_DEFAULT_RX_WTHRESH          4
  42
  43 #define IGC_DEFAULT_TX_PTHRESH          8
  44 #define IGC_DEFAULT_TX_HTHRESH          1
  45 #define IGC_DEFAULT_TX_WTHRESH          16
  46
  47 /* MSI-X other interrupt vector */
  48 #define IGC_MSIX_OTHER_INTR_VEC         0
  49
  50 /* External VLAN Enable bit mask */
  51 #define IGC_CTRL_EXT_EXT_VLAN           (1u << 26)
  52
  53 /* Speed select */
  54 #define IGC_CTRL_SPEED_MASK             (7u << 8)
  55 #define IGC_CTRL_SPEED_2500             (6u << 8)
  56
  57 /* External VLAN Ether Type bit mask and shift */
  58 #define IGC_VET_EXT                     0xFFFF0000
  59 #define IGC_VET_EXT_SHIFT               16
  60
  61 /* Force EEE Auto-negotiation */
  62 #define IGC_EEER_EEE_FRC_AN             (1u << 28)
  63
  64 /* Per Queue Good Packets Received Count */
  65 #define IGC_PQGPRC(idx)         (0x10010 + 0x100 * (idx))
  66 /* Per Queue Good Octets Received Count */
  67 #define IGC_PQGORC(idx)         (0x10018 + 0x100 * (idx))
  68 /* Per Queue Good Octets Transmitted Count */
  69 #define IGC_PQGOTC(idx)         (0x10034 + 0x100 * (idx))
  70 /* Per Queue Multicast Packets Received Count */
  71 #define IGC_PQMPRC(idx)         (0x10038 + 0x100 * (idx))
  72 /* Transmit Queue Drop Packet Count */
  73 #define IGC_TQDPC(idx)          (0xe030 + 0x40 * (idx))
  74
  75 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
  76 #define U32_0_IN_U64            0       /* lower bytes of u64 */
  77 #define U32_1_IN_U64            1       /* higher bytes of u64 */
  78 #else
  79 #define U32_0_IN_U64            1
  80 #define U32_1_IN_U64            0
  81 #endif
  82
  83 #define IGC_ALARM_INTERVAL      8000000u
  84 /* us, about 13.6s some per-queue registers will wrap around back to 0. */
  85
  86 static const struct rte_eth_desc_lim rx_desc_lim = {
  87         .nb_max = IGC_MAX_RXD,
  88         .nb_min = IGC_MIN_RXD,
  89         .nb_align = IGC_RXD_ALIGN,
  90 };
  91
  92 static const struct rte_eth_desc_lim tx_desc_lim = {
  93         .nb_max = IGC_MAX_TXD,
  94         .nb_min = IGC_MIN_TXD,
  95         .nb_align = IGC_TXD_ALIGN,
  96         .nb_seg_max = IGC_TX_MAX_SEG,
  97         .nb_mtu_seg_max = IGC_TX_MAX_MTU_SEG,
  98 };
  99
 100 static const struct rte_pci_id pci_id_igc_map[] = {
 101         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_LM) },
 102         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_V)  },
 103         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_I)  },
 104         { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_K)  },
 105         { .vendor_id = 0, /* sentinel */ },
 106 };
 107
 108 /* store statistics names and its offset in stats structure */
 109 struct rte_igc_xstats_name_off {
 110         char name[RTE_ETH_XSTATS_NAME_SIZE];
 111         unsigned int offset;
 112 };
 113
 114 static const struct rte_igc_xstats_name_off rte_igc_stats_strings[] = {
 115         {"rx_crc_errors", offsetof(struct igc_hw_stats, crcerrs)},
 116         {"rx_align_errors", offsetof(struct igc_hw_stats, algnerrc)},
 117         {"rx_errors", offsetof(struct igc_hw_stats, rxerrc)},
 118         {"rx_missed_packets", offsetof(struct igc_hw_stats, mpc)},
 119         {"tx_single_collision_packets", offsetof(struct igc_hw_stats, scc)},
 120         {"tx_multiple_collision_packets", offsetof(struct igc_hw_stats, mcc)},
 121         {"tx_excessive_collision_packets", offsetof(struct igc_hw_stats,
 122                 ecol)},
 123         {"tx_late_collisions", offsetof(struct igc_hw_stats, latecol)},
 124         {"tx_total_collisions", offsetof(struct igc_hw_stats, colc)},
 125         {"tx_deferred_packets", offsetof(struct igc_hw_stats, dc)},
 126         {"tx_no_carrier_sense_packets", offsetof(struct igc_hw_stats, tncrs)},
 127         {"tx_discarded_packets", offsetof(struct igc_hw_stats, htdpmc)},
 128         {"rx_length_errors", offsetof(struct igc_hw_stats, rlec)},
 129         {"rx_xon_packets", offsetof(struct igc_hw_stats, xonrxc)},
 130         {"tx_xon_packets", offsetof(struct igc_hw_stats, xontxc)},
 131         {"rx_xoff_packets", offsetof(struct igc_hw_stats, xoffrxc)},
 132         {"tx_xoff_packets", offsetof(struct igc_hw_stats, xofftxc)},
 133         {"rx_flow_control_unsupported_packets", offsetof(struct igc_hw_stats,
 134                 fcruc)},
 135         {"rx_size_64_packets", offsetof(struct igc_hw_stats, prc64)},
 136         {"rx_size_65_to_127_packets", offsetof(struct igc_hw_stats, prc127)},
 137         {"rx_size_128_to_255_packets", offsetof(struct igc_hw_stats, prc255)},
 138         {"rx_size_256_to_511_packets", offsetof(struct igc_hw_stats, prc511)},
 139         {"rx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 140                 prc1023)},
 141         {"rx_size_1024_to_max_packets", offsetof(struct igc_hw_stats,
 142                 prc1522)},
 143         {"rx_broadcast_packets", offsetof(struct igc_hw_stats, bprc)},
 144         {"rx_multicast_packets", offsetof(struct igc_hw_stats, mprc)},
 145         {"rx_undersize_errors", offsetof(struct igc_hw_stats, ruc)},
 146         {"rx_fragment_errors", offsetof(struct igc_hw_stats, rfc)},
 147         {"rx_oversize_errors", offsetof(struct igc_hw_stats, roc)},
 148         {"rx_jabber_errors", offsetof(struct igc_hw_stats, rjc)},
 149         {"rx_no_buffers", offsetof(struct igc_hw_stats, rnbc)},
 150         {"rx_management_packets", offsetof(struct igc_hw_stats, mgprc)},
 151         {"rx_management_dropped", offsetof(struct igc_hw_stats, mgpdc)},
 152         {"tx_management_packets", offsetof(struct igc_hw_stats, mgptc)},
 153         {"rx_total_packets", offsetof(struct igc_hw_stats, tpr)},
 154         {"tx_total_packets", offsetof(struct igc_hw_stats, tpt)},
 155         {"rx_total_bytes", offsetof(struct igc_hw_stats, tor)},
 156         {"tx_total_bytes", offsetof(struct igc_hw_stats, tot)},
 157         {"tx_size_64_packets", offsetof(struct igc_hw_stats, ptc64)},
 158         {"tx_size_65_to_127_packets", offsetof(struct igc_hw_stats, ptc127)},
 159         {"tx_size_128_to_255_packets", offsetof(struct igc_hw_stats, ptc255)},
 160         {"tx_size_256_to_511_packets", offsetof(struct igc_hw_stats, ptc511)},
 161         {"tx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
 162                 ptc1023)},
 163         {"tx_size_1023_to_max_packets", offsetof(struct igc_hw_stats,
 164                 ptc1522)},
 165         {"tx_multicast_packets", offsetof(struct igc_hw_stats, mptc)},
 166         {"tx_broadcast_packets", offsetof(struct igc_hw_stats, bptc)},
 167         {"tx_tso_packets", offsetof(struct igc_hw_stats, tsctc)},
 168         {"rx_sent_to_host_packets", offsetof(struct igc_hw_stats, rpthc)},
 169         {"tx_sent_by_host_packets", offsetof(struct igc_hw_stats, hgptc)},
 170         {"interrupt_assert_count", offsetof(struct igc_hw_stats, iac)},
 171         {"rx_descriptor_lower_threshold",
 172                 offsetof(struct igc_hw_stats, icrxdmtc)},
 173 };
 174
 175 #define IGC_NB_XSTATS (sizeof(rte_igc_stats_strings) / \
 176                 sizeof(rte_igc_stats_strings[0]))
 177
 178 static int eth_igc_configure(struct rte_eth_dev *dev);
 179 static int eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
 180 static void eth_igc_stop(struct rte_eth_dev *dev);
 181 static int eth_igc_start(struct rte_eth_dev *dev);
 182 static int eth_igc_set_link_up(struct rte_eth_dev *dev);
 183 static int eth_igc_set_link_down(struct rte_eth_dev *dev);
 184 static void eth_igc_close(struct rte_eth_dev *dev);
 185 static int eth_igc_reset(struct rte_eth_dev *dev);
 186 static int eth_igc_promiscuous_enable(struct rte_eth_dev *dev);
 187 static int eth_igc_promiscuous_disable(struct rte_eth_dev *dev);
 188 static int eth_igc_fw_version_get(struct rte_eth_dev *dev,
 189                                 char *fw_version, size_t fw_size);
 190 static int eth_igc_infos_get(struct rte_eth_dev *dev,
 191                         struct rte_eth_dev_info *dev_info);
 192 static int eth_igc_led_on(struct rte_eth_dev *dev);
 193 static int eth_igc_led_off(struct rte_eth_dev *dev);
 194 static const uint32_t *eth_igc_supported_ptypes_get(struct rte_eth_dev *dev);
 195 static int eth_igc_rar_set(struct rte_eth_dev *dev,
 196                 struct rte_ether_addr *mac_addr, uint32_t index, uint32_t pool);
 197 static void eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 198 static int eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
 199                         struct rte_ether_addr *addr);
 200 static int eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
 201                          struct rte_ether_addr *mc_addr_set,
 202                          uint32_t nb_mc_addr);
 203 static int eth_igc_allmulticast_enable(struct rte_eth_dev *dev);
 204 static int eth_igc_allmulticast_disable(struct rte_eth_dev *dev);
 205 static int eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 206 static int eth_igc_stats_get(struct rte_eth_dev *dev,
 207                         struct rte_eth_stats *rte_stats);
 208 static int eth_igc_xstats_get(struct rte_eth_dev *dev,
 209                         struct rte_eth_xstat *xstats, unsigned int n);
 210 static int eth_igc_xstats_get_by_id(struct rte_eth_dev *dev,
 211                                 const uint64_t *ids,
 212                                 uint64_t *values, unsigned int n);
 213 static int eth_igc_xstats_get_names(struct rte_eth_dev *dev,
 214                                 struct rte_eth_xstat_name *xstats_names,
 215                                 unsigned int size);
 216 static int eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 217                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
 218                 unsigned int limit);
 219 static int eth_igc_xstats_reset(struct rte_eth_dev *dev);
 220 static int
 221 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
 222         uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx);
 223 static int
 224 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 225 static int
 226 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
 227 static int
 228 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 229 static int
 230 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf);
 231 static int eth_igc_rss_reta_update(struct rte_eth_dev *dev,
 232                         struct rte_eth_rss_reta_entry64 *reta_conf,
 233                         uint16_t reta_size);
 234 static int eth_igc_rss_reta_query(struct rte_eth_dev *dev,
 235                        struct rte_eth_rss_reta_entry64 *reta_conf,
 236                        uint16_t reta_size);
 237 static int eth_igc_rss_hash_update(struct rte_eth_dev *dev,
 238                         struct rte_eth_rss_conf *rss_conf);
 239 static int eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
 240                         struct rte_eth_rss_conf *rss_conf);
 241 static int
 242 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 243 static int eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 244 static int eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
 245                       enum rte_vlan_type vlan_type, uint16_t tpid);
 246
 247 static const struct eth_dev_ops eth_igc_ops = {
 248         .dev_configure          = eth_igc_configure,
 249         .link_update            = eth_igc_link_update,
 250         .dev_stop               = eth_igc_stop,
 251         .dev_start              = eth_igc_start,
 252         .dev_close              = eth_igc_close,
 253         .dev_reset              = eth_igc_reset,
 254         .dev_set_link_up        = eth_igc_set_link_up,
 255         .dev_set_link_down      = eth_igc_set_link_down,
 256         .promiscuous_enable     = eth_igc_promiscuous_enable,
 257         .promiscuous_disable    = eth_igc_promiscuous_disable,
 258         .allmulticast_enable    = eth_igc_allmulticast_enable,
 259         .allmulticast_disable   = eth_igc_allmulticast_disable,
 260         .fw_version_get         = eth_igc_fw_version_get,
 261         .dev_infos_get          = eth_igc_infos_get,
 262         .dev_led_on             = eth_igc_led_on,
 263         .dev_led_off            = eth_igc_led_off,
 264         .dev_supported_ptypes_get = eth_igc_supported_ptypes_get,
 265         .mtu_set                = eth_igc_mtu_set,
 266         .mac_addr_add           = eth_igc_rar_set,
 267         .mac_addr_remove        = eth_igc_rar_clear,
 268         .mac_addr_set           = eth_igc_default_mac_addr_set,
 269         .set_mc_addr_list       = eth_igc_set_mc_addr_list,
 270
 271         .rx_queue_setup         = eth_igc_rx_queue_setup,
 272         .rx_queue_release       = eth_igc_rx_queue_release,
 273         .rx_queue_count         = eth_igc_rx_queue_count,
 274         .rx_descriptor_done     = eth_igc_rx_descriptor_done,
 275         .rx_descriptor_status   = eth_igc_rx_descriptor_status,
 276         .tx_descriptor_status   = eth_igc_tx_descriptor_status,
 277         .tx_queue_setup         = eth_igc_tx_queue_setup,
 278         .tx_queue_release       = eth_igc_tx_queue_release,
 279         .tx_done_cleanup        = eth_igc_tx_done_cleanup,
 280         .rxq_info_get           = eth_igc_rxq_info_get,
 281         .txq_info_get           = eth_igc_txq_info_get,
 282         .stats_get              = eth_igc_stats_get,
 283         .xstats_get             = eth_igc_xstats_get,
 284         .xstats_get_by_id       = eth_igc_xstats_get_by_id,
 285         .xstats_get_names_by_id = eth_igc_xstats_get_names_by_id,
 286         .xstats_get_names       = eth_igc_xstats_get_names,
 287         .stats_reset            = eth_igc_xstats_reset,
 288         .xstats_reset           = eth_igc_xstats_reset,
 289         .queue_stats_mapping_set = eth_igc_queue_stats_mapping_set,
 290         .rx_queue_intr_enable   = eth_igc_rx_queue_intr_enable,
 291         .rx_queue_intr_disable  = eth_igc_rx_queue_intr_disable,
 292         .flow_ctrl_get          = eth_igc_flow_ctrl_get,
 293         .flow_ctrl_set          = eth_igc_flow_ctrl_set,
 294         .reta_update            = eth_igc_rss_reta_update,
 295         .reta_query             = eth_igc_rss_reta_query,
 296         .rss_hash_update        = eth_igc_rss_hash_update,
 297         .rss_hash_conf_get      = eth_igc_rss_hash_conf_get,
 298         .vlan_filter_set        = eth_igc_vlan_filter_set,
 299         .vlan_offload_set       = eth_igc_vlan_offload_set,
 300         .vlan_tpid_set          = eth_igc_vlan_tpid_set,
 301         .vlan_strip_queue_set   = eth_igc_vlan_strip_queue_set,
 302 };
 303
 304 /*
 305  * multiple queue mode checking
 306  */
 307 static int
 308 igc_check_mq_mode(struct rte_eth_dev *dev)
 309 {
 310         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 311         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
 312
 313         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 314                 PMD_INIT_LOG(ERR, "SRIOV is not supported.");
 315                 return -EINVAL;
 316         }
 317
 318         if (rx_mq_mode != ETH_MQ_RX_NONE &&
 319                 rx_mq_mode != ETH_MQ_RX_RSS) {
 320                 /* RSS together with VMDq not supported*/
 321                 PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
 322                                 rx_mq_mode);
 323                 return -EINVAL;
 324         }
 325
 326         /* To no break software that set invalid mode, only display
 327          * warning if invalid mode is used.
 328          */
 329         if (tx_mq_mode != ETH_MQ_TX_NONE)
 330                 PMD_INIT_LOG(WARNING,
 331                         "TX mode %d is not supported. Due to meaningless in this driver, just ignore",
 332                         tx_mq_mode);
 333
 334         return 0;
 335 }
 336
 337 static int
 338 eth_igc_configure(struct rte_eth_dev *dev)
 339 {
 340         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 341         int ret;
 342
 343         PMD_INIT_FUNC_TRACE();
 344
 345         ret  = igc_check_mq_mode(dev);
 346         if (ret != 0)
 347                 return ret;
 348
 349         intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 350         return 0;
 351 }
 352
 353 static int
 354 eth_igc_set_link_up(struct rte_eth_dev *dev)
 355 {
 356         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 357
 358         if (hw->phy.media_type == igc_media_type_copper)
 359                 igc_power_up_phy(hw);
 360         else
 361                 igc_power_up_fiber_serdes_link(hw);
 362         return 0;
 363 }
 364
 365 static int
 366 eth_igc_set_link_down(struct rte_eth_dev *dev)
 367 {
 368         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 369
 370         if (hw->phy.media_type == igc_media_type_copper)
 371                 igc_power_down_phy(hw);
 372         else
 373                 igc_shutdown_fiber_serdes_link(hw);
 374         return 0;
 375 }
 376
 377 /*
 378  * disable other interrupt
 379  */
 380 static void
 381 igc_intr_other_disable(struct rte_eth_dev *dev)
 382 {
 383         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 384         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 385         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 386
 387         if (rte_intr_allow_others(intr_handle) &&
 388                 dev->data->dev_conf.intr_conf.lsc) {
 389                 IGC_WRITE_REG(hw, IGC_EIMC, 1u << IGC_MSIX_OTHER_INTR_VEC);
 390         }
 391
 392         IGC_WRITE_REG(hw, IGC_IMC, ~0);
 393         IGC_WRITE_FLUSH(hw);
 394 }
 395
 396 /*
 397  * enable other interrupt
 398  */
 399 static inline void
 400 igc_intr_other_enable(struct rte_eth_dev *dev)
 401 {
 402         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 403         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 404         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 405         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 406
 407         if (rte_intr_allow_others(intr_handle) &&
 408                 dev->data->dev_conf.intr_conf.lsc) {
 409                 IGC_WRITE_REG(hw, IGC_EIMS, 1u << IGC_MSIX_OTHER_INTR_VEC);
 410         }
 411
 412         IGC_WRITE_REG(hw, IGC_IMS, intr->mask);
 413         IGC_WRITE_FLUSH(hw);
 414 }
 415
 416 /*
 417  * It reads ICR and gets interrupt causes, check it and set a bit flag
 418  * to update link status.
 419  */
 420 static void
 421 eth_igc_interrupt_get_status(struct rte_eth_dev *dev)
 422 {
 423         uint32_t icr;
 424         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 425         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 426
 427         /* read-on-clear nic registers here */
 428         icr = IGC_READ_REG(hw, IGC_ICR);
 429
 430         intr->flags = 0;
 431         if (icr & IGC_ICR_LSC)
 432                 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
 433 }
 434
 435 /* return 0 means link status changed, -1 means not changed */
 436 static int
 437 eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 438 {
 439         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 440         struct rte_eth_link link;
 441         int link_check, count;
 442
 443         link_check = 0;
 444         hw->mac.get_link_status = 1;
 445
 446         /* possible wait-to-complete in up to 9 seconds */
 447         for (count = 0; count < IGC_LINK_UPDATE_CHECK_TIMEOUT; count++) {
 448                 /* Read the real link status */
 449                 switch (hw->phy.media_type) {
 450                 case igc_media_type_copper:
 451                         /* Do the work to read phy */
 452                         igc_check_for_link(hw);
 453                         link_check = !hw->mac.get_link_status;
 454                         break;
 455
 456                 case igc_media_type_fiber:
 457                         igc_check_for_link(hw);
 458                         link_check = (IGC_READ_REG(hw, IGC_STATUS) &
 459                                       IGC_STATUS_LU);
 460                         break;
 461
 462                 case igc_media_type_internal_serdes:
 463                         igc_check_for_link(hw);
 464                         link_check = hw->mac.serdes_has_link;
 465                         break;
 466
 467                 default:
 468                         break;
 469                 }
 470                 if (link_check || wait_to_complete == 0)
 471                         break;
 472                 rte_delay_ms(IGC_LINK_UPDATE_CHECK_INTERVAL);
 473         }
 474         memset(&link, 0, sizeof(link));
 475
 476         /* Now we check if a transition has happened */
 477         if (link_check) {
 478                 uint16_t duplex, speed;
 479                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
 480                 link.link_duplex = (duplex == FULL_DUPLEX) ?
 481                                 ETH_LINK_FULL_DUPLEX :
 482                                 ETH_LINK_HALF_DUPLEX;
 483                 link.link_speed = speed;
 484                 link.link_status = ETH_LINK_UP;
 485                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 486                                 ETH_LINK_SPEED_FIXED);
 487
 488                 if (speed == SPEED_2500) {
 489                         uint32_t tipg = IGC_READ_REG(hw, IGC_TIPG);
 490                         if ((tipg & IGC_TIPG_IPGT_MASK) != 0x0b) {
 491                                 tipg &= ~IGC_TIPG_IPGT_MASK;
 492                                 tipg |= 0x0b;
 493                                 IGC_WRITE_REG(hw, IGC_TIPG, tipg);
 494                         }
 495                 }
 496         } else {
 497                 link.link_speed = 0;
 498                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
 499                 link.link_status = ETH_LINK_DOWN;
 500                 link.link_autoneg = ETH_LINK_FIXED;
 501         }
 502
 503         return rte_eth_linkstatus_set(dev, &link);
 504 }
 505
 506 /*
 507  * It executes link_update after knowing an interrupt is present.
 508  */
 509 static void
 510 eth_igc_interrupt_action(struct rte_eth_dev *dev)
 511 {
 512         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 513         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 514         struct rte_eth_link link;
 515         int ret;
 516
 517         if (intr->flags & IGC_FLAG_NEED_LINK_UPDATE) {
 518                 intr->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
 519
 520                 /* set get_link_status to check register later */
 521                 ret = eth_igc_link_update(dev, 0);
 522
 523                 /* check if link has changed */
 524                 if (ret < 0)
 525                         return;
 526
 527                 rte_eth_linkstatus_get(dev, &link);
 528                 if (link.link_status)
 529                         PMD_DRV_LOG(INFO,
 530                                 " Port %d: Link Up - speed %u Mbps - %s",
 531                                 dev->data->port_id,
 532                                 (unsigned int)link.link_speed,
 533                                 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
 534                                 "full-duplex" : "half-duplex");
 535                 else
 536                         PMD_DRV_LOG(INFO, " Port %d: Link Down",
 537                                 dev->data->port_id);
 538
 539                 PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
 540                                 pci_dev->addr.domain,
 541                                 pci_dev->addr.bus,
 542                                 pci_dev->addr.devid,
 543                                 pci_dev->addr.function);
 544                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
 545                                 NULL);
 546         }
 547 }
 548
 549 /*
 550  * Interrupt handler which shall be registered at first.
 551  *
 552  * @handle
 553  *  Pointer to interrupt handle.
 554  * @param
 555  *  The address of parameter (struct rte_eth_dev *) registered before.
 556  */
 557 static void
 558 eth_igc_interrupt_handler(void *param)
 559 {
 560         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
 561
 562         eth_igc_interrupt_get_status(dev);
 563         eth_igc_interrupt_action(dev);
 564 }
 565
 566 static void igc_read_queue_stats_register(struct rte_eth_dev *dev);
 567
 568 /*
 569  * Update the queue status every IGC_ALARM_INTERVAL time.
 570  * @param
 571  *  The address of parameter (struct rte_eth_dev *) registered before.
 572  */
 573 static void
 574 igc_update_queue_stats_handler(void *param)
 575 {
 576         struct rte_eth_dev *dev = param;
 577         igc_read_queue_stats_register(dev);
 578         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
 579                         igc_update_queue_stats_handler, dev);
 580 }
 581
 582 /*
 583  * rx,tx enable/disable
 584  */
 585 static void
 586 eth_igc_rxtx_control(struct rte_eth_dev *dev, bool enable)
 587 {
 588         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 589         uint32_t tctl, rctl;
 590
 591         tctl = IGC_READ_REG(hw, IGC_TCTL);
 592         rctl = IGC_READ_REG(hw, IGC_RCTL);
 593
 594         if (enable) {
 595                 /* enable Tx/Rx */
 596                 tctl |= IGC_TCTL_EN;
 597                 rctl |= IGC_RCTL_EN;
 598         } else {
 599                 /* disable Tx/Rx */
 600                 tctl &= ~IGC_TCTL_EN;
 601                 rctl &= ~IGC_RCTL_EN;
 602         }
 603         IGC_WRITE_REG(hw, IGC_TCTL, tctl);
 604         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
 605         IGC_WRITE_FLUSH(hw);
 606 }
 607
 608 /*
 609  *  This routine disables all traffic on the adapter by issuing a
 610  *  global reset on the MAC.
 611  */
 612 static void
 613 eth_igc_stop(struct rte_eth_dev *dev)
 614 {
 615         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 616         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 617         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 618         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 619         struct rte_eth_link link;
 620
 621         adapter->stopped = 1;
 622
 623         /* disable receive and transmit */
 624         eth_igc_rxtx_control(dev, false);
 625
 626         /* disable all MSI-X interrupts */
 627         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 628         IGC_WRITE_FLUSH(hw);
 629
 630         /* clear all MSI-X interrupts */
 631         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 632
 633         igc_intr_other_disable(dev);
 634
 635         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
 636
 637         /* disable intr eventfd mapping */
 638         rte_intr_disable(intr_handle);
 639
 640         igc_reset_hw(hw);
 641
 642         /* disable all wake up */
 643         IGC_WRITE_REG(hw, IGC_WUC, 0);
 644
 645         /* disable checking EEE operation in MAC loopback mode */
 646         igc_read_reg_check_clear_bits(hw, IGC_EEER, IGC_EEER_EEE_FRC_AN);
 647
 648         /* Set bit for Go Link disconnect */
 649         igc_read_reg_check_set_bits(hw, IGC_82580_PHY_POWER_MGMT,
 650                         IGC_82580_PM_GO_LINKD);
 651
 652         /* Power down the phy. Needed to make the link go Down */
 653         eth_igc_set_link_down(dev);
 654
 655         igc_dev_clear_queues(dev);
 656
 657         /* clear the recorded link status */
 658         memset(&link, 0, sizeof(link));
 659         rte_eth_linkstatus_set(dev, &link);
 660
 661         if (!rte_intr_allow_others(intr_handle))
 662                 /* resume to the default handler */
 663                 rte_intr_callback_register(intr_handle,
 664                                            eth_igc_interrupt_handler,
 665                                            (void *)dev);
 666
 667         /* Clean datapath event and queue/vec mapping */
 668         rte_intr_efd_disable(intr_handle);
 669         if (intr_handle->intr_vec != NULL) {
 670                 rte_free(intr_handle->intr_vec);
 671                 intr_handle->intr_vec = NULL;
 672         }
 673 }
 674
 675 /*
 676  * write interrupt vector allocation register
 677  * @hw
 678  *  board private structure
 679  * @queue_index
 680  *  queue index, valid 0,1,2,3
 681  * @tx
 682  *  tx:1, rx:0
 683  * @msix_vector
 684  *  msix-vector, valid 0,1,2,3,4
 685  */
 686 static void
 687 igc_write_ivar(struct igc_hw *hw, uint8_t queue_index,
 688                 bool tx, uint8_t msix_vector)
 689 {
 690         uint8_t offset = 0;
 691         uint8_t reg_index = queue_index >> 1;
 692         uint32_t val;
 693
 694         /*
 695          * IVAR(0)
 696          * bit31...24   bit23...16      bit15...8       bit7...0
 697          * TX1          RX1             TX0             RX0
 698          *
 699          * IVAR(1)
 700          * bit31...24   bit23...16      bit15...8       bit7...0
 701          * TX3          RX3             TX2             RX2
 702          */
 703
 704         if (tx)
 705                 offset = 8;
 706
 707         if (queue_index & 1)
 708                 offset += 16;
 709
 710         val = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, reg_index);
 711
 712         /* clear bits */
 713         val &= ~((uint32_t)0xFF << offset);
 714
 715         /* write vector and valid bit */
 716         val |= (uint32_t)(msix_vector | IGC_IVAR_VALID) << offset;
 717
 718         IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, reg_index, val);
 719 }
 720
 721 /* Sets up the hardware to generate MSI-X interrupts properly
 722  * @hw
 723  *  board private structure
 724  */
 725 static void
 726 igc_configure_msix_intr(struct rte_eth_dev *dev)
 727 {
 728         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 729         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 730         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 731
 732         uint32_t intr_mask;
 733         uint32_t vec = IGC_MISC_VEC_ID;
 734         uint32_t base = IGC_MISC_VEC_ID;
 735         uint32_t misc_shift = 0;
 736         int i;
 737
 738         /* won't configure msix register if no mapping is done
 739          * between intr vector and event fd
 740          */
 741         if (!rte_intr_dp_is_en(intr_handle))
 742                 return;
 743
 744         if (rte_intr_allow_others(intr_handle)) {
 745                 base = IGC_RX_VEC_START;
 746                 vec = base;
 747                 misc_shift = 1;
 748         }
 749
 750         /* turn on MSI-X capability first */
 751         IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE |
 752                                 IGC_GPIE_PBA | IGC_GPIE_EIAME |
 753                                 IGC_GPIE_NSICR);
 754         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
 755                 misc_shift;
 756
 757         if (dev->data->dev_conf.intr_conf.lsc)
 758                 intr_mask |= (1u << IGC_MSIX_OTHER_INTR_VEC);
 759
 760         /* enable msix auto-clear */
 761         igc_read_reg_check_set_bits(hw, IGC_EIAC, intr_mask);
 762
 763         /* set other cause interrupt vector */
 764         igc_read_reg_check_set_bits(hw, IGC_IVAR_MISC,
 765                 (uint32_t)(IGC_MSIX_OTHER_INTR_VEC | IGC_IVAR_VALID) << 8);
 766
 767         /* enable auto-mask */
 768         igc_read_reg_check_set_bits(hw, IGC_EIAM, intr_mask);
 769
 770         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 771                 igc_write_ivar(hw, i, 0, vec);
 772                 intr_handle->intr_vec[i] = vec;
 773                 if (vec < base + intr_handle->nb_efd - 1)
 774                         vec++;
 775         }
 776
 777         IGC_WRITE_FLUSH(hw);
 778 }
 779
 780 /**
 781  * It enables the interrupt mask and then enable the interrupt.
 782  *
 783  * @dev
 784  *  Pointer to struct rte_eth_dev.
 785  * @on
 786  *  Enable or Disable
 787  */
 788 static void
 789 igc_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
 790 {
 791         struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
 792
 793         if (on)
 794                 intr->mask |= IGC_ICR_LSC;
 795         else
 796                 intr->mask &= ~IGC_ICR_LSC;
 797 }
 798
 799 /*
 800  * It enables the interrupt.
 801  * It will be called once only during nic initialized.
 802  */
 803 static void
 804 igc_rxq_interrupt_setup(struct rte_eth_dev *dev)
 805 {
 806         uint32_t mask;
 807         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 808         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 809         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 810         int misc_shift = rte_intr_allow_others(intr_handle) ? 1 : 0;
 811
 812         /* won't configure msix register if no mapping is done
 813          * between intr vector and event fd
 814          */
 815         if (!rte_intr_dp_is_en(intr_handle))
 816                 return;
 817
 818         mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) << misc_shift;
 819         IGC_WRITE_REG(hw, IGC_EIMS, mask);
 820 }
 821
 822 /*
 823  *  Get hardware rx-buffer size.
 824  */
 825 static inline int
 826 igc_get_rx_buffer_size(struct igc_hw *hw)
 827 {
 828         return (IGC_READ_REG(hw, IGC_RXPBS) & 0x3f) << 10;
 829 }
 830
 831 /*
 832  * igc_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
 833  * For ASF and Pass Through versions of f/w this means
 834  * that the driver is loaded.
 835  */
 836 static void
 837 igc_hw_control_acquire(struct igc_hw *hw)
 838 {
 839         uint32_t ctrl_ext;
 840
 841         /* Let firmware know the driver has taken over */
 842         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 843         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 844 }
 845
 846 /*
 847  * igc_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
 848  * For ASF and Pass Through versions of f/w this means that the
 849  * driver is no longer loaded.
 850  */
 851 static void
 852 igc_hw_control_release(struct igc_hw *hw)
 853 {
 854         uint32_t ctrl_ext;
 855
 856         /* Let firmware taken over control of h/w */
 857         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
 858         IGC_WRITE_REG(hw, IGC_CTRL_EXT,
 859                         ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 860 }
 861
 862 static int
 863 igc_hardware_init(struct igc_hw *hw)
 864 {
 865         uint32_t rx_buf_size;
 866         int diag;
 867
 868         /* Let the firmware know the OS is in control */
 869         igc_hw_control_acquire(hw);
 870
 871         /* Issue a global reset */
 872         igc_reset_hw(hw);
 873
 874         /* disable all wake up */
 875         IGC_WRITE_REG(hw, IGC_WUC, 0);
 876
 877         /*
 878          * Hardware flow control
 879          * - High water mark should allow for at least two standard size (1518)
 880          *   frames to be received after sending an XOFF.
 881          * - Low water mark works best when it is very near the high water mark.
 882          *   This allows the receiver to restart by sending XON when it has
 883          *   drained a bit. Here we use an arbitrary value of 1500 which will
 884          *   restart after one full frame is pulled from the buffer. There
 885          *   could be several smaller frames in the buffer and if so they will
 886          *   not trigger the XON until their total number reduces the buffer
 887          *   by 1500.
 888          */
 889         rx_buf_size = igc_get_rx_buffer_size(hw);
 890         hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2);
 891         hw->fc.low_water = hw->fc.high_water - 1500;
 892         hw->fc.pause_time = IGC_FC_PAUSE_TIME;
 893         hw->fc.send_xon = 1;
 894         hw->fc.requested_mode = igc_fc_full;
 895
 896         diag = igc_init_hw(hw);
 897         if (diag < 0)
 898                 return diag;
 899
 900         igc_get_phy_info(hw);
 901         igc_check_for_link(hw);
 902
 903         return 0;
 904 }
 905
 906 static int
 907 eth_igc_start(struct rte_eth_dev *dev)
 908 {
 909         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
 910         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
 911         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 912         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 913         uint32_t *speeds;
 914         int ret;
 915
 916         PMD_INIT_FUNC_TRACE();
 917
 918         /* disable all MSI-X interrupts */
 919         IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
 920         IGC_WRITE_FLUSH(hw);
 921
 922         /* clear all MSI-X interrupts */
 923         IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
 924
 925         /* disable uio/vfio intr/eventfd mapping */
 926         if (!adapter->stopped)
 927                 rte_intr_disable(intr_handle);
 928
 929         /* Power up the phy. Needed to make the link go Up */
 930         eth_igc_set_link_up(dev);
 931
 932         /* Put the address into the Receive Address Array */
 933         igc_rar_set(hw, hw->mac.addr, 0);
 934
 935         /* Initialize the hardware */
 936         if (igc_hardware_init(hw)) {
 937                 PMD_DRV_LOG(ERR, "Unable to initialize the hardware");
 938                 return -EIO;
 939         }
 940         adapter->stopped = 0;
 941
 942         /* check and configure queue intr-vector mapping */
 943         if (rte_intr_cap_multiple(intr_handle) &&
 944                 dev->data->dev_conf.intr_conf.rxq) {
 945                 uint32_t intr_vector = dev->data->nb_rx_queues;
 946                 if (rte_intr_efd_enable(intr_handle, intr_vector))
 947                         return -1;
 948         }
 949
 950         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 951                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
 952                         dev->data->nb_rx_queues * sizeof(int), 0);
 953                 if (intr_handle->intr_vec == NULL) {
 954                         PMD_DRV_LOG(ERR,
 955                                 "Failed to allocate %d rx_queues intr_vec",
 956                                 dev->data->nb_rx_queues);
 957                         return -ENOMEM;
 958                 }
 959         }
 960
 961         /* configure msix for rx interrupt */
 962         igc_configure_msix_intr(dev);
 963
 964         igc_tx_init(dev);
 965
 966         /* This can fail when allocating mbufs for descriptor rings */
 967         ret = igc_rx_init(dev);
 968         if (ret) {
 969                 PMD_DRV_LOG(ERR, "Unable to initialize RX hardware");
 970                 igc_dev_clear_queues(dev);
 971                 return ret;
 972         }
 973
 974         igc_clear_hw_cntrs_base_generic(hw);
 975
 976         /* VLAN Offload Settings */
 977         eth_igc_vlan_offload_set(dev,
 978                 ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
 979                 ETH_VLAN_EXTEND_MASK);
 980
 981         /* Setup link speed and duplex */
 982         speeds = &dev->data->dev_conf.link_speeds;
 983         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
 984                 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
 985                 hw->mac.autoneg = 1;
 986         } else {
 987                 int num_speeds = 0;
 988                 bool autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
 989
 990                 /* Reset */
 991                 hw->phy.autoneg_advertised = 0;
 992
 993                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
 994                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
 995                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
 996                                 ETH_LINK_SPEED_FIXED)) {
 997                         num_speeds = -1;
 998                         goto error_invalid_config;
 999                 }
1000                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
1001                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
1002                         num_speeds++;
1003                 }
1004                 if (*speeds & ETH_LINK_SPEED_10M) {
1005                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
1006                         num_speeds++;
1007                 }
1008                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
1009                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1010                         num_speeds++;
1011                 }
1012                 if (*speeds & ETH_LINK_SPEED_100M) {
1013                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1014                         num_speeds++;
1015                 }
1016                 if (*speeds & ETH_LINK_SPEED_1G) {
1017                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1018                         num_speeds++;
1019                 }
1020                 if (*speeds & ETH_LINK_SPEED_2_5G) {
1021                         hw->phy.autoneg_advertised |= ADVERTISE_2500_FULL;
1022                         num_speeds++;
1023                 }
1024                 if (num_speeds == 0 || (!autoneg && num_speeds > 1))
1025                         goto error_invalid_config;
1026
1027                 /* Set/reset the mac.autoneg based on the link speed,
1028                  * fixed or not
1029                  */
1030                 if (!autoneg) {
1031                         hw->mac.autoneg = 0;
1032                         hw->mac.forced_speed_duplex =
1033                                         hw->phy.autoneg_advertised;
1034                 } else {
1035                         hw->mac.autoneg = 1;
1036                 }
1037         }
1038
1039         igc_setup_link(hw);
1040
1041         if (rte_intr_allow_others(intr_handle)) {
1042                 /* check if lsc interrupt is enabled */
1043                 if (dev->data->dev_conf.intr_conf.lsc)
1044                         igc_lsc_interrupt_setup(dev, 1);
1045                 else
1046                         igc_lsc_interrupt_setup(dev, 0);
1047         } else {
1048                 rte_intr_callback_unregister(intr_handle,
1049                                              eth_igc_interrupt_handler,
1050                                              (void *)dev);
1051                 if (dev->data->dev_conf.intr_conf.lsc)
1052                         PMD_DRV_LOG(INFO,
1053                                 "LSC won't enable because of no intr multiplex");
1054         }
1055
1056         /* enable uio/vfio intr/eventfd mapping */
1057         rte_intr_enable(intr_handle);
1058
1059         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1060                         igc_update_queue_stats_handler, dev);
1061
1062         /* check if rxq interrupt is enabled */
1063         if (dev->data->dev_conf.intr_conf.rxq &&
1064                         rte_intr_dp_is_en(intr_handle))
1065                 igc_rxq_interrupt_setup(dev);
1066
1067         /* resume enabled intr since hw reset */
1068         igc_intr_other_enable(dev);
1069
1070         eth_igc_rxtx_control(dev, true);
1071         eth_igc_link_update(dev, 0);
1072
1073         /* configure MAC-loopback mode */
1074         if (dev->data->dev_conf.lpbk_mode == 1) {
1075                 uint32_t reg_val;
1076
1077                 reg_val = IGC_READ_REG(hw, IGC_CTRL);
1078                 reg_val &= ~IGC_CTRL_SPEED_MASK;
1079                 reg_val |= IGC_CTRL_SLU | IGC_CTRL_FRCSPD |
1080                         IGC_CTRL_FRCDPX | IGC_CTRL_FD | IGC_CTRL_SPEED_2500;
1081                 IGC_WRITE_REG(hw, IGC_CTRL, reg_val);
1082
1083                 igc_read_reg_check_set_bits(hw, IGC_EEER, IGC_EEER_EEE_FRC_AN);
1084         }
1085
1086         return 0;
1087
1088 error_invalid_config:
1089         PMD_DRV_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1090                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1091         igc_dev_clear_queues(dev);
1092         return -EINVAL;
1093 }
1094
1095 static int
1096 igc_reset_swfw_lock(struct igc_hw *hw)
1097 {
1098         int ret_val;
1099
1100         /*
1101          * Do mac ops initialization manually here, since we will need
1102          * some function pointers set by this call.
1103          */
1104         ret_val = igc_init_mac_params(hw);
1105         if (ret_val)
1106                 return ret_val;
1107
1108         /*
1109          * SMBI lock should not fail in this early stage. If this is the case,
1110          * it is due to an improper exit of the application.
1111          * So force the release of the faulty lock.
1112          */
1113         if (igc_get_hw_semaphore_generic(hw) < 0)
1114                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
1115
1116         igc_put_hw_semaphore_generic(hw);
1117
1118         if (hw->mac.ops.acquire_swfw_sync != NULL) {
1119                 uint16_t mask;
1120
1121                 /*
1122                  * Phy lock should not fail in this early stage.
1123                  * If this is the case, it is due to an improper exit of the
1124                  * application. So force the release of the faulty lock.
1125                  */
1126                 mask = IGC_SWFW_PHY0_SM;
1127                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
1128                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
1129                                     hw->bus.func);
1130                 }
1131                 hw->mac.ops.release_swfw_sync(hw, mask);
1132
1133                 /*
1134                  * This one is more tricky since it is common to all ports; but
1135                  * swfw_sync retries last long enough (1s) to be almost sure
1136                  * that if lock can not be taken it is due to an improper lock
1137                  * of the semaphore.
1138                  */
1139                 mask = IGC_SWFW_EEP_SM;
1140                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0)
1141                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
1142
1143                 hw->mac.ops.release_swfw_sync(hw, mask);
1144         }
1145
1146         return IGC_SUCCESS;
1147 }
1148
1149 /*
1150  * free all rx/tx queues.
1151  */
1152 static void
1153 igc_dev_free_queues(struct rte_eth_dev *dev)
1154 {
1155         uint16_t i;
1156
1157         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1158                 eth_igc_rx_queue_release(dev->data->rx_queues[i]);
1159                 dev->data->rx_queues[i] = NULL;
1160         }
1161         dev->data->nb_rx_queues = 0;
1162
1163         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1164                 eth_igc_tx_queue_release(dev->data->tx_queues[i]);
1165                 dev->data->tx_queues[i] = NULL;
1166         }
1167         dev->data->nb_tx_queues = 0;
1168 }
1169
1170 static void
1171 eth_igc_close(struct rte_eth_dev *dev)
1172 {
1173         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1174         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1175         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1176         struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
1177         int retry = 0;
1178
1179         PMD_INIT_FUNC_TRACE();
1180
1181         if (!adapter->stopped)
1182                 eth_igc_stop(dev);
1183
1184         igc_intr_other_disable(dev);
1185         do {
1186                 int ret = rte_intr_callback_unregister(intr_handle,
1187                                 eth_igc_interrupt_handler, dev);
1188                 if (ret >= 0 || ret == -ENOENT || ret == -EINVAL)
1189                         break;
1190
1191                 PMD_DRV_LOG(ERR, "intr callback unregister failed: %d", ret);
1192                 DELAY(200 * 1000); /* delay 200ms */
1193         } while (retry++ < 5);
1194
1195         igc_phy_hw_reset(hw);
1196         igc_hw_control_release(hw);
1197         igc_dev_free_queues(dev);
1198
1199         /* Reset any pending lock */
1200         igc_reset_swfw_lock(hw);
1201 }
1202
1203 static void
1204 igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
1205 {
1206         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1207
1208         hw->vendor_id = pci_dev->id.vendor_id;
1209         hw->device_id = pci_dev->id.device_id;
1210         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1211         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1212 }
1213
1214 static int
1215 eth_igc_dev_init(struct rte_eth_dev *dev)
1216 {
1217         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1218         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1219         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1220         int i, error = 0;
1221
1222         PMD_INIT_FUNC_TRACE();
1223         dev->dev_ops = &eth_igc_ops;
1224
1225         /*
1226          * for secondary processes, we don't initialize any further as primary
1227          * has already done this work. Only check we don't need a different
1228          * RX function.
1229          */
1230         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1231                 return 0;
1232
1233         rte_eth_copy_pci_info(dev, pci_dev);
1234
1235         hw->back = pci_dev;
1236         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1237
1238         igc_identify_hardware(dev, pci_dev);
1239         if (igc_setup_init_funcs(hw, false) != IGC_SUCCESS) {
1240                 error = -EIO;
1241                 goto err_late;
1242         }
1243
1244         igc_get_bus_info(hw);
1245
1246         /* Reset any pending lock */
1247         if (igc_reset_swfw_lock(hw) != IGC_SUCCESS) {
1248                 error = -EIO;
1249                 goto err_late;
1250         }
1251
1252         /* Finish initialization */
1253         if (igc_setup_init_funcs(hw, true) != IGC_SUCCESS) {
1254                 error = -EIO;
1255                 goto err_late;
1256         }
1257
1258         hw->mac.autoneg = 1;
1259         hw->phy.autoneg_wait_to_complete = 0;
1260         hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
1261
1262         /* Copper options */
1263         if (hw->phy.media_type == igc_media_type_copper) {
1264                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1265                 hw->phy.disable_polarity_correction = 0;
1266                 hw->phy.ms_type = igc_ms_hw_default;
1267         }
1268
1269         /*
1270          * Start from a known state, this is important in reading the nvm
1271          * and mac from that.
1272          */
1273         igc_reset_hw(hw);
1274
1275         /* Make sure we have a good EEPROM before we read from it */
1276         if (igc_validate_nvm_checksum(hw) < 0) {
1277                 /*
1278                  * Some PCI-E parts fail the first check due to
1279                  * the link being in sleep state, call it again,
1280                  * if it fails a second time its a real issue.
1281                  */
1282                 if (igc_validate_nvm_checksum(hw) < 0) {
1283                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
1284                         error = -EIO;
1285                         goto err_late;
1286                 }
1287         }
1288
1289         /* Read the permanent MAC address out of the EEPROM */
1290         if (igc_read_mac_addr(hw) != 0) {
1291                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
1292                 error = -EIO;
1293                 goto err_late;
1294         }
1295
1296         /* Allocate memory for storing MAC addresses */
1297         dev->data->mac_addrs = rte_zmalloc("igc",
1298                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
1299         if (dev->data->mac_addrs == NULL) {
1300                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes for storing MAC",
1301                                 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1302                 error = -ENOMEM;
1303                 goto err_late;
1304         }
1305
1306         /* Copy the permanent MAC address */
1307         rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
1308                         &dev->data->mac_addrs[0]);
1309
1310         /* Now initialize the hardware */
1311         if (igc_hardware_init(hw) != 0) {
1312                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
1313                 rte_free(dev->data->mac_addrs);
1314                 dev->data->mac_addrs = NULL;
1315                 error = -ENODEV;
1316                 goto err_late;
1317         }
1318
1319         /* Pass the information to the rte_eth_dev_close() that it should also
1320          * release the private port resources.
1321          */
1322         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1323
1324         hw->mac.get_link_status = 1;
1325         igc->stopped = 0;
1326
1327         /* Indicate SOL/IDER usage */
1328         if (igc_check_reset_block(hw) < 0)
1329                 PMD_INIT_LOG(ERR,
1330                         "PHY reset is blocked due to SOL/IDER session.");
1331
1332         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
1333                         dev->data->port_id, pci_dev->id.vendor_id,
1334                         pci_dev->id.device_id);
1335
1336         rte_intr_callback_register(&pci_dev->intr_handle,
1337                         eth_igc_interrupt_handler, (void *)dev);
1338
1339         /* enable uio/vfio intr/eventfd mapping */
1340         rte_intr_enable(&pci_dev->intr_handle);
1341
1342         /* enable support intr */
1343         igc_intr_other_enable(dev);
1344
1345         /* initiate queue status */
1346         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1347                 igc->txq_stats_map[i] = -1;
1348                 igc->rxq_stats_map[i] = -1;
1349         }
1350
1351         return 0;
1352
1353 err_late:
1354         igc_hw_control_release(hw);
1355         return error;
1356 }
1357
1358 static int
1359 eth_igc_dev_uninit(__rte_unused struct rte_eth_dev *eth_dev)
1360 {
1361         PMD_INIT_FUNC_TRACE();
1362
1363         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1364                 return 0;
1365
1366         eth_igc_close(eth_dev);
1367         return 0;
1368 }
1369
1370 static int
1371 eth_igc_reset(struct rte_eth_dev *dev)
1372 {
1373         int ret;
1374
1375         PMD_INIT_FUNC_TRACE();
1376
1377         ret = eth_igc_dev_uninit(dev);
1378         if (ret)
1379                 return ret;
1380
1381         return eth_igc_dev_init(dev);
1382 }
1383
1384 static int
1385 eth_igc_promiscuous_enable(struct rte_eth_dev *dev)
1386 {
1387         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1388         uint32_t rctl;
1389
1390         rctl = IGC_READ_REG(hw, IGC_RCTL);
1391         rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1392         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1393         return 0;
1394 }
1395
1396 static int
1397 eth_igc_promiscuous_disable(struct rte_eth_dev *dev)
1398 {
1399         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1400         uint32_t rctl;
1401
1402         rctl = IGC_READ_REG(hw, IGC_RCTL);
1403         rctl &= (~IGC_RCTL_UPE);
1404         if (dev->data->all_multicast == 1)
1405                 rctl |= IGC_RCTL_MPE;
1406         else
1407                 rctl &= (~IGC_RCTL_MPE);
1408         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1409         return 0;
1410 }
1411
1412 static int
1413 eth_igc_allmulticast_enable(struct rte_eth_dev *dev)
1414 {
1415         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1416         uint32_t rctl;
1417
1418         rctl = IGC_READ_REG(hw, IGC_RCTL);
1419         rctl |= IGC_RCTL_MPE;
1420         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1421         return 0;
1422 }
1423
1424 static int
1425 eth_igc_allmulticast_disable(struct rte_eth_dev *dev)
1426 {
1427         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1428         uint32_t rctl;
1429
1430         if (dev->data->promiscuous == 1)
1431                 return 0;       /* must remain in all_multicast mode */
1432
1433         rctl = IGC_READ_REG(hw, IGC_RCTL);
1434         rctl &= (~IGC_RCTL_MPE);
1435         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1436         return 0;
1437 }
1438
1439 static int
1440 eth_igc_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
1441                        size_t fw_size)
1442 {
1443         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1444         struct igc_fw_version fw;
1445         int ret;
1446
1447         igc_get_fw_version(hw, &fw);
1448
1449         /* if option rom is valid, display its version too */
1450         if (fw.or_valid) {
1451                 ret = snprintf(fw_version, fw_size,
1452                          "%d.%d, 0x%08x, %d.%d.%d",
1453                          fw.eep_major, fw.eep_minor, fw.etrack_id,
1454                          fw.or_major, fw.or_build, fw.or_patch);
1455         /* no option rom */
1456         } else {
1457                 if (fw.etrack_id != 0X0000) {
1458                         ret = snprintf(fw_version, fw_size,
1459                                  "%d.%d, 0x%08x",
1460                                  fw.eep_major, fw.eep_minor,
1461                                  fw.etrack_id);
1462                 } else {
1463                         ret = snprintf(fw_version, fw_size,
1464                                  "%d.%d.%d",
1465                                  fw.eep_major, fw.eep_minor,
1466                                  fw.eep_build);
1467                 }
1468         }
1469
1470         ret += 1; /* add the size of '\0' */
1471         if (fw_size < (u32)ret)
1472                 return ret;
1473         else
1474                 return 0;
1475 }
1476
1477 static int
1478 eth_igc_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1479 {
1480         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1481
1482         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1483         dev_info->max_rx_pktlen = MAX_RX_JUMBO_FRAME_SIZE;
1484         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1485         dev_info->rx_offload_capa = IGC_RX_OFFLOAD_ALL;
1486         dev_info->tx_offload_capa = IGC_TX_OFFLOAD_ALL;
1487         dev_info->rx_queue_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1488
1489         dev_info->max_rx_queues = IGC_QUEUE_PAIRS_NUM;
1490         dev_info->max_tx_queues = IGC_QUEUE_PAIRS_NUM;
1491         dev_info->max_vmdq_pools = 0;
1492
1493         dev_info->hash_key_size = IGC_HKEY_MAX_INDEX * sizeof(uint32_t);
1494         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1495         dev_info->flow_type_rss_offloads = IGC_RSS_OFFLOAD_ALL;
1496
1497         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1498                 .rx_thresh = {
1499                         .pthresh = IGC_DEFAULT_RX_PTHRESH,
1500                         .hthresh = IGC_DEFAULT_RX_HTHRESH,
1501                         .wthresh = IGC_DEFAULT_RX_WTHRESH,
1502                 },
1503                 .rx_free_thresh = IGC_DEFAULT_RX_FREE_THRESH,
1504                 .rx_drop_en = 0,
1505                 .offloads = 0,
1506         };
1507
1508         dev_info->default_txconf = (struct rte_eth_txconf) {
1509                 .tx_thresh = {
1510                         .pthresh = IGC_DEFAULT_TX_PTHRESH,
1511                         .hthresh = IGC_DEFAULT_TX_HTHRESH,
1512                         .wthresh = IGC_DEFAULT_TX_WTHRESH,
1513                 },
1514                 .offloads = 0,
1515         };
1516
1517         dev_info->rx_desc_lim = rx_desc_lim;
1518         dev_info->tx_desc_lim = tx_desc_lim;
1519
1520         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1521                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1522                         ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G;
1523
1524         dev_info->max_mtu = dev_info->max_rx_pktlen - IGC_ETH_OVERHEAD;
1525         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
1526         return 0;
1527 }
1528
1529 static int
1530 eth_igc_led_on(struct rte_eth_dev *dev)
1531 {
1532         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1533
1534         return igc_led_on(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1535 }
1536
1537 static int
1538 eth_igc_led_off(struct rte_eth_dev *dev)
1539 {
1540         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1541
1542         return igc_led_off(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1543 }
1544
1545 static const uint32_t *
1546 eth_igc_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev)
1547 {
1548         static const uint32_t ptypes[] = {
1549                 /* refers to rx_desc_pkt_info_to_pkt_type() */
1550                 RTE_PTYPE_L2_ETHER,
1551                 RTE_PTYPE_L3_IPV4,
1552                 RTE_PTYPE_L3_IPV4_EXT,
1553                 RTE_PTYPE_L3_IPV6,
1554                 RTE_PTYPE_L3_IPV6_EXT,
1555                 RTE_PTYPE_L4_TCP,
1556                 RTE_PTYPE_L4_UDP,
1557                 RTE_PTYPE_L4_SCTP,
1558                 RTE_PTYPE_TUNNEL_IP,
1559                 RTE_PTYPE_INNER_L3_IPV6,
1560                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1561                 RTE_PTYPE_INNER_L4_TCP,
1562                 RTE_PTYPE_INNER_L4_UDP,
1563                 RTE_PTYPE_UNKNOWN
1564         };
1565
1566         return ptypes;
1567 }
1568
1569 static int
1570 eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1571 {
1572         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1573         uint32_t frame_size = mtu + IGC_ETH_OVERHEAD;
1574         uint32_t rctl;
1575
1576         /* if extend vlan has been enabled */
1577         if (IGC_READ_REG(hw, IGC_CTRL_EXT) & IGC_CTRL_EXT_EXT_VLAN)
1578                 frame_size += VLAN_TAG_SIZE;
1579
1580         /* check that mtu is within the allowed range */
1581         if (mtu < RTE_ETHER_MIN_MTU ||
1582                 frame_size > MAX_RX_JUMBO_FRAME_SIZE)
1583                 return -EINVAL;
1584
1585         /*
1586          * refuse mtu that requires the support of scattered packets when
1587          * this feature has not been enabled before.
1588          */
1589         if (!dev->data->scattered_rx &&
1590             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
1591                 return -EINVAL;
1592
1593         rctl = IGC_READ_REG(hw, IGC_RCTL);
1594
1595         /* switch to jumbo mode if needed */
1596         if (mtu > RTE_ETHER_MTU) {
1597                 dev->data->dev_conf.rxmode.offloads |=
1598                         DEV_RX_OFFLOAD_JUMBO_FRAME;
1599                 rctl |= IGC_RCTL_LPE;
1600         } else {
1601                 dev->data->dev_conf.rxmode.offloads &=
1602                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1603                 rctl &= ~IGC_RCTL_LPE;
1604         }
1605         IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1606
1607         /* update max frame size */
1608         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1609
1610         IGC_WRITE_REG(hw, IGC_RLPML,
1611                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1612
1613         return 0;
1614 }
1615
1616 static int
1617 eth_igc_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1618                 uint32_t index, uint32_t pool)
1619 {
1620         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1621
1622         igc_rar_set(hw, mac_addr->addr_bytes, index);
1623         RTE_SET_USED(pool);
1624         return 0;
1625 }
1626
1627 static void
1628 eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1629 {
1630         uint8_t addr[RTE_ETHER_ADDR_LEN];
1631         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1632
1633         memset(addr, 0, sizeof(addr));
1634         igc_rar_set(hw, addr, index);
1635 }
1636
1637 static int
1638 eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
1639                         struct rte_ether_addr *addr)
1640 {
1641         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1642         igc_rar_set(hw, addr->addr_bytes, 0);
1643         return 0;
1644 }
1645
1646 static int
1647 eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
1648                          struct rte_ether_addr *mc_addr_set,
1649                          uint32_t nb_mc_addr)
1650 {
1651         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1652         igc_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
1653         return 0;
1654 }
1655
1656 /*
1657  * Read hardware registers
1658  */
1659 static void
1660 igc_read_stats_registers(struct igc_hw *hw, struct igc_hw_stats *stats)
1661 {
1662         int pause_frames;
1663
1664         uint64_t old_gprc  = stats->gprc;
1665         uint64_t old_gptc  = stats->gptc;
1666         uint64_t old_tpr   = stats->tpr;
1667         uint64_t old_tpt   = stats->tpt;
1668         uint64_t old_rpthc = stats->rpthc;
1669         uint64_t old_hgptc = stats->hgptc;
1670
1671         stats->crcerrs += IGC_READ_REG(hw, IGC_CRCERRS);
1672         stats->algnerrc += IGC_READ_REG(hw, IGC_ALGNERRC);
1673         stats->rxerrc += IGC_READ_REG(hw, IGC_RXERRC);
1674         stats->mpc += IGC_READ_REG(hw, IGC_MPC);
1675         stats->scc += IGC_READ_REG(hw, IGC_SCC);
1676         stats->ecol += IGC_READ_REG(hw, IGC_ECOL);
1677
1678         stats->mcc += IGC_READ_REG(hw, IGC_MCC);
1679         stats->latecol += IGC_READ_REG(hw, IGC_LATECOL);
1680         stats->colc += IGC_READ_REG(hw, IGC_COLC);
1681
1682         stats->dc += IGC_READ_REG(hw, IGC_DC);
1683         stats->tncrs += IGC_READ_REG(hw, IGC_TNCRS);
1684         stats->htdpmc += IGC_READ_REG(hw, IGC_HTDPMC);
1685         stats->rlec += IGC_READ_REG(hw, IGC_RLEC);
1686         stats->xonrxc += IGC_READ_REG(hw, IGC_XONRXC);
1687         stats->xontxc += IGC_READ_REG(hw, IGC_XONTXC);
1688
1689         /*
1690          * For watchdog management we need to know if we have been
1691          * paused during the last interval, so capture that here.
1692          */
1693         pause_frames = IGC_READ_REG(hw, IGC_XOFFRXC);
1694         stats->xoffrxc += pause_frames;
1695         stats->xofftxc += IGC_READ_REG(hw, IGC_XOFFTXC);
1696         stats->fcruc += IGC_READ_REG(hw, IGC_FCRUC);
1697         stats->prc64 += IGC_READ_REG(hw, IGC_PRC64);
1698         stats->prc127 += IGC_READ_REG(hw, IGC_PRC127);
1699         stats->prc255 += IGC_READ_REG(hw, IGC_PRC255);
1700         stats->prc511 += IGC_READ_REG(hw, IGC_PRC511);
1701         stats->prc1023 += IGC_READ_REG(hw, IGC_PRC1023);
1702         stats->prc1522 += IGC_READ_REG(hw, IGC_PRC1522);
1703         stats->gprc += IGC_READ_REG(hw, IGC_GPRC);
1704         stats->bprc += IGC_READ_REG(hw, IGC_BPRC);
1705         stats->mprc += IGC_READ_REG(hw, IGC_MPRC);
1706         stats->gptc += IGC_READ_REG(hw, IGC_GPTC);
1707
1708         /* For the 64-bit byte counters the low dword must be read first. */
1709         /* Both registers clear on the read of the high dword */
1710
1711         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1712         stats->gorc += IGC_READ_REG(hw, IGC_GORCL);
1713         stats->gorc += ((uint64_t)IGC_READ_REG(hw, IGC_GORCH) << 32);
1714         stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN;
1715         stats->gotc += IGC_READ_REG(hw, IGC_GOTCL);
1716         stats->gotc += ((uint64_t)IGC_READ_REG(hw, IGC_GOTCH) << 32);
1717         stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN;
1718
1719         stats->rnbc += IGC_READ_REG(hw, IGC_RNBC);
1720         stats->ruc += IGC_READ_REG(hw, IGC_RUC);
1721         stats->rfc += IGC_READ_REG(hw, IGC_RFC);
1722         stats->roc += IGC_READ_REG(hw, IGC_ROC);
1723         stats->rjc += IGC_READ_REG(hw, IGC_RJC);
1724
1725         stats->mgprc += IGC_READ_REG(hw, IGC_MGTPRC);
1726         stats->mgpdc += IGC_READ_REG(hw, IGC_MGTPDC);
1727         stats->mgptc += IGC_READ_REG(hw, IGC_MGTPTC);
1728         stats->b2ospc += IGC_READ_REG(hw, IGC_B2OSPC);
1729         stats->b2ogprc += IGC_READ_REG(hw, IGC_B2OGPRC);
1730         stats->o2bgptc += IGC_READ_REG(hw, IGC_O2BGPTC);
1731         stats->o2bspc += IGC_READ_REG(hw, IGC_O2BSPC);
1732
1733         stats->tpr += IGC_READ_REG(hw, IGC_TPR);
1734         stats->tpt += IGC_READ_REG(hw, IGC_TPT);
1735
1736         stats->tor += IGC_READ_REG(hw, IGC_TORL);
1737         stats->tor += ((uint64_t)IGC_READ_REG(hw, IGC_TORH) << 32);
1738         stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN;
1739         stats->tot += IGC_READ_REG(hw, IGC_TOTL);
1740         stats->tot += ((uint64_t)IGC_READ_REG(hw, IGC_TOTH) << 32);
1741         stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN;
1742
1743         stats->ptc64 += IGC_READ_REG(hw, IGC_PTC64);
1744         stats->ptc127 += IGC_READ_REG(hw, IGC_PTC127);
1745         stats->ptc255 += IGC_READ_REG(hw, IGC_PTC255);
1746         stats->ptc511 += IGC_READ_REG(hw, IGC_PTC511);
1747         stats->ptc1023 += IGC_READ_REG(hw, IGC_PTC1023);
1748         stats->ptc1522 += IGC_READ_REG(hw, IGC_PTC1522);
1749         stats->mptc += IGC_READ_REG(hw, IGC_MPTC);
1750         stats->bptc += IGC_READ_REG(hw, IGC_BPTC);
1751         stats->tsctc += IGC_READ_REG(hw, IGC_TSCTC);
1752
1753         stats->iac += IGC_READ_REG(hw, IGC_IAC);
1754         stats->rpthc += IGC_READ_REG(hw, IGC_RPTHC);
1755         stats->hgptc += IGC_READ_REG(hw, IGC_HGPTC);
1756         stats->icrxdmtc += IGC_READ_REG(hw, IGC_ICRXDMTC);
1757
1758         /* Host to Card Statistics */
1759         stats->hgorc += IGC_READ_REG(hw, IGC_HGORCL);
1760         stats->hgorc += ((uint64_t)IGC_READ_REG(hw, IGC_HGORCH) << 32);
1761         stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN;
1762         stats->hgotc += IGC_READ_REG(hw, IGC_HGOTCL);
1763         stats->hgotc += ((uint64_t)IGC_READ_REG(hw, IGC_HGOTCH) << 32);
1764         stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN;
1765         stats->lenerrs += IGC_READ_REG(hw, IGC_LENERRS);
1766 }
1767
1768 /*
1769  * Write 0 to all queue status registers
1770  */
1771 static void
1772 igc_reset_queue_stats_register(struct igc_hw *hw)
1773 {
1774         int i;
1775
1776         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1777                 IGC_WRITE_REG(hw, IGC_PQGPRC(i), 0);
1778                 IGC_WRITE_REG(hw, IGC_PQGPTC(i), 0);
1779                 IGC_WRITE_REG(hw, IGC_PQGORC(i), 0);
1780                 IGC_WRITE_REG(hw, IGC_PQGOTC(i), 0);
1781                 IGC_WRITE_REG(hw, IGC_PQMPRC(i), 0);
1782                 IGC_WRITE_REG(hw, IGC_RQDPC(i), 0);
1783                 IGC_WRITE_REG(hw, IGC_TQDPC(i), 0);
1784         }
1785 }
1786
1787 /*
1788  * Read all hardware queue status registers
1789  */
1790 static void
1791 igc_read_queue_stats_register(struct rte_eth_dev *dev)
1792 {
1793         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1794         struct igc_hw_queue_stats *queue_stats =
1795                                 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1796         int i;
1797
1798         /*
1799          * This register is not cleared on read. Furthermore, the register wraps
1800          * around back to 0x00000000 on the next increment when reaching a value
1801          * of 0xFFFFFFFF and then continues normal count operation.
1802          */
1803         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1804                 union {
1805                         u64 ddword;
1806                         u32 dword[2];
1807                 } value;
1808                 u32 tmp;
1809
1810                 /*
1811                  * Read the register first, if the value is smaller than that
1812                  * previous read, that mean the register has been overflowed,
1813                  * then we add the high 4 bytes by 1 and replace the low 4
1814                  * bytes by the new value.
1815                  */
1816                 tmp = IGC_READ_REG(hw, IGC_PQGPRC(i));
1817                 value.ddword = queue_stats->pqgprc[i];
1818                 if (value.dword[U32_0_IN_U64] > tmp)
1819                         value.dword[U32_1_IN_U64]++;
1820                 value.dword[U32_0_IN_U64] = tmp;
1821                 queue_stats->pqgprc[i] = value.ddword;
1822
1823                 tmp = IGC_READ_REG(hw, IGC_PQGPTC(i));
1824                 value.ddword = queue_stats->pqgptc[i];
1825                 if (value.dword[U32_0_IN_U64] > tmp)
1826                         value.dword[U32_1_IN_U64]++;
1827                 value.dword[U32_0_IN_U64] = tmp;
1828                 queue_stats->pqgptc[i] = value.ddword;
1829
1830                 tmp = IGC_READ_REG(hw, IGC_PQGORC(i));
1831                 value.ddword = queue_stats->pqgorc[i];
1832                 if (value.dword[U32_0_IN_U64] > tmp)
1833                         value.dword[U32_1_IN_U64]++;
1834                 value.dword[U32_0_IN_U64] = tmp;
1835                 queue_stats->pqgorc[i] = value.ddword;
1836
1837                 tmp = IGC_READ_REG(hw, IGC_PQGOTC(i));
1838                 value.ddword = queue_stats->pqgotc[i];
1839                 if (value.dword[U32_0_IN_U64] > tmp)
1840                         value.dword[U32_1_IN_U64]++;
1841                 value.dword[U32_0_IN_U64] = tmp;
1842                 queue_stats->pqgotc[i] = value.ddword;
1843
1844                 tmp = IGC_READ_REG(hw, IGC_PQMPRC(i));
1845                 value.ddword = queue_stats->pqmprc[i];
1846                 if (value.dword[U32_0_IN_U64] > tmp)
1847                         value.dword[U32_1_IN_U64]++;
1848                 value.dword[U32_0_IN_U64] = tmp;
1849                 queue_stats->pqmprc[i] = value.ddword;
1850
1851                 tmp = IGC_READ_REG(hw, IGC_RQDPC(i));
1852                 value.ddword = queue_stats->rqdpc[i];
1853                 if (value.dword[U32_0_IN_U64] > tmp)
1854                         value.dword[U32_1_IN_U64]++;
1855                 value.dword[U32_0_IN_U64] = tmp;
1856                 queue_stats->rqdpc[i] = value.ddword;
1857
1858                 tmp = IGC_READ_REG(hw, IGC_TQDPC(i));
1859                 value.ddword = queue_stats->tqdpc[i];
1860                 if (value.dword[U32_0_IN_U64] > tmp)
1861                         value.dword[U32_1_IN_U64]++;
1862                 value.dword[U32_0_IN_U64] = tmp;
1863                 queue_stats->tqdpc[i] = value.ddword;
1864         }
1865 }
1866
1867 static int
1868 eth_igc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1869 {
1870         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1871         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1872         struct igc_hw_stats *stats = IGC_DEV_PRIVATE_STATS(dev);
1873         struct igc_hw_queue_stats *queue_stats =
1874                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1875         int i;
1876
1877         /*
1878          * Cancel status handler since it will read the queue status registers
1879          */
1880         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1881
1882         /* Read status register */
1883         igc_read_queue_stats_register(dev);
1884         igc_read_stats_registers(hw, stats);
1885
1886         if (rte_stats == NULL) {
1887                 /* Restart queue status handler */
1888                 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1889                                 igc_update_queue_stats_handler, dev);
1890                 return -EINVAL;
1891         }
1892
1893         /* Rx Errors */
1894         rte_stats->imissed = stats->mpc;
1895         rte_stats->ierrors = stats->crcerrs +
1896                         stats->rlec + stats->ruc + stats->roc +
1897                         stats->rxerrc + stats->algnerrc;
1898
1899         /* Tx Errors */
1900         rte_stats->oerrors = stats->ecol + stats->latecol;
1901
1902         rte_stats->ipackets = stats->gprc;
1903         rte_stats->opackets = stats->gptc;
1904         rte_stats->ibytes   = stats->gorc;
1905         rte_stats->obytes   = stats->gotc;
1906
1907         /* Get per-queue statuses */
1908         for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1909                 /* GET TX queue statuses */
1910                 int map_id = igc->txq_stats_map[i];
1911                 if (map_id >= 0) {
1912                         rte_stats->q_opackets[map_id] += queue_stats->pqgptc[i];
1913                         rte_stats->q_obytes[map_id] += queue_stats->pqgotc[i];
1914                 }
1915                 /* Get RX queue statuses */
1916                 map_id = igc->rxq_stats_map[i];
1917                 if (map_id >= 0) {
1918                         rte_stats->q_ipackets[map_id] += queue_stats->pqgprc[i];
1919                         rte_stats->q_ibytes[map_id] += queue_stats->pqgorc[i];
1920                         rte_stats->q_errors[map_id] += queue_stats->rqdpc[i];
1921                 }
1922         }
1923
1924         /* Restart queue status handler */
1925         rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1926                         igc_update_queue_stats_handler, dev);
1927         return 0;
1928 }
1929
1930 static int
1931 eth_igc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1932                    unsigned int n)
1933 {
1934         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1935         struct igc_hw_stats *hw_stats =
1936                         IGC_DEV_PRIVATE_STATS(dev);
1937         unsigned int i;
1938
1939         igc_read_stats_registers(hw, hw_stats);
1940
1941         if (n < IGC_NB_XSTATS)
1942                 return IGC_NB_XSTATS;
1943
1944         /* If this is a reset xstats is NULL, and we have cleared the
1945          * registers by reading them.
1946          */
1947         if (!xstats)
1948                 return 0;
1949
1950         /* Extended stats */
1951         for (i = 0; i < IGC_NB_XSTATS; i++) {
1952                 xstats[i].id = i;
1953                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1954                         rte_igc_stats_strings[i].offset);
1955         }
1956
1957         return IGC_NB_XSTATS;
1958 }
1959
1960 static int
1961 eth_igc_xstats_reset(struct rte_eth_dev *dev)
1962 {
1963         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1964         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1965         struct igc_hw_queue_stats *queue_stats =
1966                         IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1967
1968         /* Cancel queue status handler for avoid conflict */
1969         rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1970
1971         /* HW registers are cleared on read */
1972         igc_reset_queue_stats_register(hw);
1973         igc_read_stats_registers(hw, hw_stats);
1974
1975         /* Reset software totals */
1976         memset(hw_stats, 0, sizeof(*hw_stats));
1977         memset(queue_stats, 0, sizeof(*queue_stats));
1978
1979         /* Restart the queue status handler */
1980         rte_eal_alarm_set(IGC_ALARM_INTERVAL, igc_update_queue_stats_handler,
1981                         dev);
1982
1983         return 0;
1984 }
1985
1986 static int
1987 eth_igc_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1988         struct rte_eth_xstat_name *xstats_names, unsigned int size)
1989 {
1990         unsigned int i;
1991
1992         if (xstats_names == NULL)
1993                 return IGC_NB_XSTATS;
1994
1995         if (size < IGC_NB_XSTATS) {
1996                 PMD_DRV_LOG(ERR, "not enough buffers!");
1997                 return IGC_NB_XSTATS;
1998         }
1999
2000         for (i = 0; i < IGC_NB_XSTATS; i++)
2001                 strlcpy(xstats_names[i].name, rte_igc_stats_strings[i].name,
2002                         sizeof(xstats_names[i].name));
2003
2004         return IGC_NB_XSTATS;
2005 }
2006
2007 static int
2008 eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
2009                 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
2010                 unsigned int limit)
2011 {
2012         unsigned int i;
2013
2014         if (!ids)
2015                 return eth_igc_xstats_get_names(dev, xstats_names, limit);
2016
2017         for (i = 0; i < limit; i++) {
2018                 if (ids[i] >= IGC_NB_XSTATS) {
2019                         PMD_DRV_LOG(ERR, "id value isn't valid");
2020                         return -EINVAL;
2021                 }
2022                 strlcpy(xstats_names[i].name,
2023                         rte_igc_stats_strings[ids[i]].name,
2024                         sizeof(xstats_names[i].name));
2025         }
2026         return limit;
2027 }
2028
2029 static int
2030 eth_igc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
2031                 uint64_t *values, unsigned int n)
2032 {
2033         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2034         struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
2035         unsigned int i;
2036
2037         igc_read_stats_registers(hw, hw_stats);
2038
2039         if (!ids) {
2040                 if (n < IGC_NB_XSTATS)
2041                         return IGC_NB_XSTATS;
2042
2043                 /* If this is a reset xstats is NULL, and we have cleared the
2044                  * registers by reading them.
2045                  */
2046                 if (!values)
2047                         return 0;
2048
2049                 /* Extended stats */
2050                 for (i = 0; i < IGC_NB_XSTATS; i++)
2051                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2052                                         rte_igc_stats_strings[i].offset);
2053
2054                 return IGC_NB_XSTATS;
2055
2056         } else {
2057                 for (i = 0; i < n; i++) {
2058                         if (ids[i] >= IGC_NB_XSTATS) {
2059                                 PMD_DRV_LOG(ERR, "id value isn't valid");
2060                                 return -EINVAL;
2061                         }
2062                         values[i] = *(uint64_t *)(((char *)hw_stats) +
2063                                         rte_igc_stats_strings[ids[i]].offset);
2064                 }
2065                 return n;
2066         }
2067 }
2068
2069 static int
2070 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
2071                 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx)
2072 {
2073         struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
2074
2075         /* check queue id is valid */
2076         if (queue_id >= IGC_QUEUE_PAIRS_NUM) {
2077                 PMD_DRV_LOG(ERR, "queue id(%u) error, max is %u",
2078                         queue_id, IGC_QUEUE_PAIRS_NUM - 1);
2079                 return -EINVAL;
2080         }
2081
2082         /* store the mapping status id */
2083         if (is_rx)
2084                 igc->rxq_stats_map[queue_id] = stat_idx;
2085         else
2086                 igc->txq_stats_map[queue_id] = stat_idx;
2087
2088         return 0;
2089 }
2090
2091 static int
2092 eth_igc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2093 {
2094         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2095         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2096         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2097         uint32_t vec = IGC_MISC_VEC_ID;
2098
2099         if (rte_intr_allow_others(intr_handle))
2100                 vec = IGC_RX_VEC_START;
2101
2102         uint32_t mask = 1u << (queue_id + vec);
2103
2104         IGC_WRITE_REG(hw, IGC_EIMC, mask);
2105         IGC_WRITE_FLUSH(hw);
2106
2107         return 0;
2108 }
2109
2110 static int
2111 eth_igc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2112 {
2113         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2114         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2115         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2116         uint32_t vec = IGC_MISC_VEC_ID;
2117
2118         if (rte_intr_allow_others(intr_handle))
2119                 vec = IGC_RX_VEC_START;
2120
2121         uint32_t mask = 1u << (queue_id + vec);
2122
2123         IGC_WRITE_REG(hw, IGC_EIMS, mask);
2124         IGC_WRITE_FLUSH(hw);
2125
2126         rte_intr_enable(intr_handle);
2127
2128         return 0;
2129 }
2130
2131 static int
2132 eth_igc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2133 {
2134         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2135         uint32_t ctrl;
2136         int tx_pause;
2137         int rx_pause;
2138
2139         fc_conf->pause_time = hw->fc.pause_time;
2140         fc_conf->high_water = hw->fc.high_water;
2141         fc_conf->low_water = hw->fc.low_water;
2142         fc_conf->send_xon = hw->fc.send_xon;
2143         fc_conf->autoneg = hw->mac.autoneg;
2144
2145         /*
2146          * Return rx_pause and tx_pause status according to actual setting of
2147          * the TFCE and RFCE bits in the CTRL register.
2148          */
2149         ctrl = IGC_READ_REG(hw, IGC_CTRL);
2150         if (ctrl & IGC_CTRL_TFCE)
2151                 tx_pause = 1;
2152         else
2153                 tx_pause = 0;
2154
2155         if (ctrl & IGC_CTRL_RFCE)
2156                 rx_pause = 1;
2157         else
2158                 rx_pause = 0;
2159
2160         if (rx_pause && tx_pause)
2161                 fc_conf->mode = RTE_FC_FULL;
2162         else if (rx_pause)
2163                 fc_conf->mode = RTE_FC_RX_PAUSE;
2164         else if (tx_pause)
2165                 fc_conf->mode = RTE_FC_TX_PAUSE;
2166         else
2167                 fc_conf->mode = RTE_FC_NONE;
2168
2169         return 0;
2170 }
2171
2172 static int
2173 eth_igc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2174 {
2175         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2176         uint32_t rx_buf_size;
2177         uint32_t max_high_water;
2178         uint32_t rctl;
2179         int err;
2180
2181         if (fc_conf->autoneg != hw->mac.autoneg)
2182                 return -ENOTSUP;
2183
2184         rx_buf_size = igc_get_rx_buffer_size(hw);
2185         PMD_DRV_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2186
2187         /* At least reserve one Ethernet frame for watermark */
2188         max_high_water = rx_buf_size - RTE_ETHER_MAX_LEN;
2189         if (fc_conf->high_water > max_high_water ||
2190                 fc_conf->high_water < fc_conf->low_water) {
2191                 PMD_DRV_LOG(ERR,
2192                         "Incorrect high(%u)/low(%u) water value, max is %u",
2193                         fc_conf->high_water, fc_conf->low_water,
2194                         max_high_water);
2195                 return -EINVAL;
2196         }
2197
2198         switch (fc_conf->mode) {
2199         case RTE_FC_NONE:
2200                 hw->fc.requested_mode = igc_fc_none;
2201                 break;
2202         case RTE_FC_RX_PAUSE:
2203                 hw->fc.requested_mode = igc_fc_rx_pause;
2204                 break;
2205         case RTE_FC_TX_PAUSE:
2206                 hw->fc.requested_mode = igc_fc_tx_pause;
2207                 break;
2208         case RTE_FC_FULL:
2209                 hw->fc.requested_mode = igc_fc_full;
2210                 break;
2211         default:
2212                 PMD_DRV_LOG(ERR, "unsupported fc mode: %u", fc_conf->mode);
2213                 return -EINVAL;
2214         }
2215
2216         hw->fc.pause_time     = fc_conf->pause_time;
2217         hw->fc.high_water     = fc_conf->high_water;
2218         hw->fc.low_water      = fc_conf->low_water;
2219         hw->fc.send_xon       = fc_conf->send_xon;
2220
2221         err = igc_setup_link_generic(hw);
2222         if (err == IGC_SUCCESS) {
2223                 /**
2224                  * check if we want to forward MAC frames - driver doesn't have
2225                  * native capability to do that, so we'll write the registers
2226                  * ourselves
2227                  **/
2228                 rctl = IGC_READ_REG(hw, IGC_RCTL);
2229
2230                 /* set or clear MFLCN.PMCF bit depending on configuration */
2231                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2232                         rctl |= IGC_RCTL_PMCF;
2233                 else
2234                         rctl &= ~IGC_RCTL_PMCF;
2235
2236                 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
2237                 IGC_WRITE_FLUSH(hw);
2238
2239                 return 0;
2240         }
2241
2242         PMD_DRV_LOG(ERR, "igc_setup_link_generic = 0x%x", err);
2243         return -EIO;
2244 }
2245
2246 static int
2247 eth_igc_rss_reta_update(struct rte_eth_dev *dev,
2248                         struct rte_eth_rss_reta_entry64 *reta_conf,
2249                         uint16_t reta_size)
2250 {
2251         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2252         uint16_t i;
2253
2254         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2255                 PMD_DRV_LOG(ERR,
2256                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2257                         reta_size, ETH_RSS_RETA_SIZE_128);
2258                 return -EINVAL;
2259         }
2260
2261         /* set redirection table */
2262         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2263                 union igc_rss_reta_reg reta, reg;
2264                 uint16_t idx, shift;
2265                 uint8_t j, mask;
2266
2267                 idx = i / RTE_RETA_GROUP_SIZE;
2268                 shift = i % RTE_RETA_GROUP_SIZE;
2269                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2270                                 IGC_RSS_RDT_REG_SIZE_MASK);
2271
2272                 /* if no need to update the register */
2273                 if (!mask)
2274                         continue;
2275
2276                 /* check mask whether need to read the register value first */
2277                 if (mask == IGC_RSS_RDT_REG_SIZE_MASK)
2278                         reg.dword = 0;
2279                 else
2280                         reg.dword = IGC_READ_REG_LE_VALUE(hw,
2281                                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2282
2283                 /* update the register */
2284                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2285                         if (mask & (1u << j))
2286                                 reta.bytes[j] =
2287                                         (uint8_t)reta_conf[idx].reta[shift + j];
2288                         else
2289                                 reta.bytes[j] = reg.bytes[j];
2290                 }
2291                 IGC_WRITE_REG_LE_VALUE(hw,
2292                         IGC_RETA(i / IGC_RSS_RDT_REG_SIZE), reta.dword);
2293         }
2294
2295         return 0;
2296 }
2297
2298 static int
2299 eth_igc_rss_reta_query(struct rte_eth_dev *dev,
2300                        struct rte_eth_rss_reta_entry64 *reta_conf,
2301                        uint16_t reta_size)
2302 {
2303         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2304         uint16_t i;
2305
2306         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2307                 PMD_DRV_LOG(ERR,
2308                         "The size of RSS redirection table configured(%d) doesn't match the number hardware can supported(%d)",
2309                         reta_size, ETH_RSS_RETA_SIZE_128);
2310                 return -EINVAL;
2311         }
2312
2313         /* read redirection table */
2314         for (i = 0; i < ETH_RSS_RETA_SIZE_128; i += IGC_RSS_RDT_REG_SIZE) {
2315                 union igc_rss_reta_reg reta;
2316                 uint16_t idx, shift;
2317                 uint8_t j, mask;
2318
2319                 idx = i / RTE_RETA_GROUP_SIZE;
2320                 shift = i % RTE_RETA_GROUP_SIZE;
2321                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2322                                 IGC_RSS_RDT_REG_SIZE_MASK);
2323
2324                 /* if no need to read register */
2325                 if (!mask)
2326                         continue;
2327
2328                 /* read register and get the queue index */
2329                 reta.dword = IGC_READ_REG_LE_VALUE(hw,
2330                                 IGC_RETA(i / IGC_RSS_RDT_REG_SIZE));
2331                 for (j = 0; j < IGC_RSS_RDT_REG_SIZE; j++) {
2332                         if (mask & (1u << j))
2333                                 reta_conf[idx].reta[shift + j] = reta.bytes[j];
2334                 }
2335         }
2336
2337         return 0;
2338 }
2339
2340 static int
2341 eth_igc_rss_hash_update(struct rte_eth_dev *dev,
2342                         struct rte_eth_rss_conf *rss_conf)
2343 {
2344         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2345         igc_hw_rss_hash_set(hw, rss_conf);
2346         return 0;
2347 }
2348
2349 static int
2350 eth_igc_rss_hash_conf_get(struct rte_eth_dev *dev,
2351                         struct rte_eth_rss_conf *rss_conf)
2352 {
2353         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2354         uint32_t *hash_key = (uint32_t *)rss_conf->rss_key;
2355         uint32_t mrqc;
2356         uint64_t rss_hf;
2357
2358         if (hash_key != NULL) {
2359                 int i;
2360
2361                 /* if not enough space for store hash key */
2362                 if (rss_conf->rss_key_len != IGC_HKEY_SIZE) {
2363                         PMD_DRV_LOG(ERR,
2364                                 "RSS hash key size %u in parameter doesn't match the hardware hash key size %u",
2365                                 rss_conf->rss_key_len, IGC_HKEY_SIZE);
2366                         return -EINVAL;
2367                 }
2368
2369                 /* read RSS key from register */
2370                 for (i = 0; i < IGC_HKEY_MAX_INDEX; i++)
2371                         hash_key[i] = IGC_READ_REG_LE_VALUE(hw, IGC_RSSRK(i));
2372         }
2373
2374         /* get RSS functions configured in MRQC register */
2375         mrqc = IGC_READ_REG(hw, IGC_MRQC);
2376         if ((mrqc & IGC_MRQC_ENABLE_RSS_4Q) == 0)
2377                 return 0;
2378
2379         rss_hf = 0;
2380         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4)
2381                 rss_hf |= ETH_RSS_IPV4;
2382         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_TCP)
2383                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2384         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6)
2385                 rss_hf |= ETH_RSS_IPV6;
2386         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_EX)
2387                 rss_hf |= ETH_RSS_IPV6_EX;
2388         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP)
2389                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
2390         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_TCP_EX)
2391                 rss_hf |= ETH_RSS_IPV6_TCP_EX;
2392         if (mrqc & IGC_MRQC_RSS_FIELD_IPV4_UDP)
2393                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2394         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP)
2395                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
2396         if (mrqc & IGC_MRQC_RSS_FIELD_IPV6_UDP_EX)
2397                 rss_hf |= ETH_RSS_IPV6_UDP_EX;
2398
2399         rss_conf->rss_hf |= rss_hf;
2400         return 0;
2401 }
2402
2403 static int
2404 eth_igc_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2405 {
2406         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2407         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2408         uint32_t vfta;
2409         uint32_t vid_idx;
2410         uint32_t vid_bit;
2411
2412         vid_idx = (vlan_id >> IGC_VFTA_ENTRY_SHIFT) & IGC_VFTA_ENTRY_MASK;
2413         vid_bit = 1u << (vlan_id & IGC_VFTA_ENTRY_BIT_SHIFT_MASK);
2414         vfta = shadow_vfta->vfta[vid_idx];
2415         if (on)
2416                 vfta |= vid_bit;
2417         else
2418                 vfta &= ~vid_bit;
2419         IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, vid_idx, vfta);
2420
2421         /* update local VFTA copy */
2422         shadow_vfta->vfta[vid_idx] = vfta;
2423
2424         return 0;
2425 }
2426
2427 static void
2428 igc_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2429 {
2430         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2431         igc_read_reg_check_clear_bits(hw, IGC_RCTL,
2432                         IGC_RCTL_CFIEN | IGC_RCTL_VFE);
2433 }
2434
2435 static void
2436 igc_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2437 {
2438         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2439         struct igc_vfta *shadow_vfta = IGC_DEV_PRIVATE_VFTA(dev);
2440         uint32_t reg_val;
2441         int i;
2442
2443         /* Filter Table Enable, CFI not used for packet acceptance */
2444         reg_val = IGC_READ_REG(hw, IGC_RCTL);
2445         reg_val &= ~IGC_RCTL_CFIEN;
2446         reg_val |= IGC_RCTL_VFE;
2447         IGC_WRITE_REG(hw, IGC_RCTL, reg_val);
2448
2449         /* restore VFTA table */
2450         for (i = 0; i < IGC_VFTA_SIZE; i++)
2451                 IGC_WRITE_REG_ARRAY(hw, IGC_VFTA, i, shadow_vfta->vfta[i]);
2452 }
2453
2454 static void
2455 igc_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2456 {
2457         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2458
2459         igc_read_reg_check_clear_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2460 }
2461
2462 static void
2463 igc_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2464 {
2465         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2466
2467         igc_read_reg_check_set_bits(hw, IGC_CTRL, IGC_CTRL_VME);
2468 }
2469
2470 static int
2471 igc_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2472 {
2473         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2474         uint32_t ctrl_ext;
2475
2476         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2477
2478         /* if extend vlan hasn't been enabled */
2479         if ((ctrl_ext & IGC_CTRL_EXT_EXT_VLAN) == 0)
2480                 return 0;
2481
2482         if ((dev->data->dev_conf.rxmode.offloads &
2483                         DEV_RX_OFFLOAD_JUMBO_FRAME) == 0)
2484                 goto write_ext_vlan;
2485
2486         /* Update maximum packet length */
2487         if (dev->data->dev_conf.rxmode.max_rx_pkt_len <
2488                 RTE_ETHER_MIN_MTU + VLAN_TAG_SIZE) {
2489                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, min is %u",
2490                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
2491                         VLAN_TAG_SIZE + RTE_ETHER_MIN_MTU);
2492                 return -EINVAL;
2493         }
2494         dev->data->dev_conf.rxmode.max_rx_pkt_len -= VLAN_TAG_SIZE;
2495         IGC_WRITE_REG(hw, IGC_RLPML,
2496                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
2497
2498 write_ext_vlan:
2499         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_EXT_VLAN);
2500         return 0;
2501 }
2502
2503 static int
2504 igc_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2505 {
2506         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2507         uint32_t ctrl_ext;
2508
2509         ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
2510
2511         /* if extend vlan has been enabled */
2512         if (ctrl_ext & IGC_CTRL_EXT_EXT_VLAN)
2513                 return 0;
2514
2515         if ((dev->data->dev_conf.rxmode.offloads &
2516                         DEV_RX_OFFLOAD_JUMBO_FRAME) == 0)
2517                 goto write_ext_vlan;
2518
2519         /* Update maximum packet length */
2520         if (dev->data->dev_conf.rxmode.max_rx_pkt_len >
2521                 MAX_RX_JUMBO_FRAME_SIZE - VLAN_TAG_SIZE) {
2522                 PMD_DRV_LOG(ERR, "Maximum packet length %u error, max is %u",
2523                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2524                         VLAN_TAG_SIZE, MAX_RX_JUMBO_FRAME_SIZE);
2525                 return -EINVAL;
2526         }
2527         dev->data->dev_conf.rxmode.max_rx_pkt_len += VLAN_TAG_SIZE;
2528         IGC_WRITE_REG(hw, IGC_RLPML,
2529                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
2530
2531 write_ext_vlan:
2532         IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_EXT_VLAN);
2533         return 0;
2534 }
2535
2536 static int
2537 eth_igc_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2538 {
2539         struct rte_eth_rxmode *rxmode;
2540
2541         rxmode = &dev->data->dev_conf.rxmode;
2542         if (mask & ETH_VLAN_STRIP_MASK) {
2543                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
2544                         igc_vlan_hw_strip_enable(dev);
2545                 else
2546                         igc_vlan_hw_strip_disable(dev);
2547         }
2548
2549         if (mask & ETH_VLAN_FILTER_MASK) {
2550                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
2551                         igc_vlan_hw_filter_enable(dev);
2552                 else
2553                         igc_vlan_hw_filter_disable(dev);
2554         }
2555
2556         if (mask & ETH_VLAN_EXTEND_MASK) {
2557                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
2558                         return igc_vlan_hw_extend_enable(dev);
2559                 else
2560                         return igc_vlan_hw_extend_disable(dev);
2561         }
2562
2563         return 0;
2564 }
2565
2566 static int
2567 eth_igc_vlan_tpid_set(struct rte_eth_dev *dev,
2568                       enum rte_vlan_type vlan_type,
2569                       uint16_t tpid)
2570 {
2571         struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
2572         uint32_t reg_val;
2573
2574         /* only outer TPID of double VLAN can be configured*/
2575         if (vlan_type == ETH_VLAN_TYPE_OUTER) {
2576                 reg_val = IGC_READ_REG(hw, IGC_VET);
2577                 reg_val = (reg_val & (~IGC_VET_EXT)) |
2578                         ((uint32_t)tpid << IGC_VET_EXT_SHIFT);
2579                 IGC_WRITE_REG(hw, IGC_VET, reg_val);
2580
2581                 return 0;
2582         }
2583
2584         /* all other TPID values are read-only*/
2585         PMD_DRV_LOG(ERR, "Not supported");
2586         return -ENOTSUP;
2587 }
2588
2589 static int
2590 eth_igc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2591         struct rte_pci_device *pci_dev)
2592 {
2593         PMD_INIT_FUNC_TRACE();
2594         return rte_eth_dev_pci_generic_probe(pci_dev,
2595                 sizeof(struct igc_adapter), eth_igc_dev_init);
2596 }
2597
2598 static int
2599 eth_igc_pci_remove(struct rte_pci_device *pci_dev)
2600 {
2601         PMD_INIT_FUNC_TRACE();
2602         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igc_dev_uninit);
2603 }
2604
2605 static struct rte_pci_driver rte_igc_pmd = {
2606         .id_table = pci_id_igc_map,
2607         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2608         .probe = eth_igc_pci_probe,
2609         .remove = eth_igc_pci_remove,
2610 };
2611
2612 RTE_PMD_REGISTER_PCI(net_igc, rte_igc_pmd);
2613 RTE_PMD_REGISTER_PCI_TABLE(net_igc, pci_id_igc_map);
2614 RTE_PMD_REGISTER_KMOD_DEP(net_igc, "* igb_uio | uio_pci_generic | vfio-pci");