drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 #define IGB_HKEY_MAX_INDEX 10
  73
  74 /* Bit shift and mask */
  75 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  76 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  77 #define IGB_8_BIT_WIDTH  CHAR_BIT
  78 #define IGB_8_BIT_MASK   UINT8_MAX
  79
  80 /* Additional timesync values. */
  81 #define E1000_CYCLECOUNTER_MASK      0xffffffffffffffffULL
  82 #define E1000_ETQF_FILTER_1588       3
  83 #define IGB_82576_TSYNC_SHIFT        16
  84 #define E1000_INCPERIOD_82576        (1 << E1000_TIMINCA_16NS_SHIFT)
  85 #define E1000_INCVALUE_82576         (16 << IGB_82576_TSYNC_SHIFT)
  86 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  87
  88 static int  eth_igb_configure(struct rte_eth_dev *dev);
  89 static int  eth_igb_start(struct rte_eth_dev *dev);
  90 static void eth_igb_stop(struct rte_eth_dev *dev);
  91 static void eth_igb_close(struct rte_eth_dev *dev);
  92 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  93 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  94 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  95 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  96 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  97                                 int wait_to_complete);
  98 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  99                                 struct rte_eth_stats *rte_stats);
 100 static int eth_igb_xstats_get(struct rte_eth_dev *dev,
 101                               struct rte_eth_xstats *xstats, unsigned n);
 102 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 103 static void eth_igb_xstats_reset(struct rte_eth_dev *dev);
 104 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 105                               struct rte_eth_dev_info *dev_info);
 106 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 107                                 struct rte_eth_dev_info *dev_info);
 108 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 109                                 struct rte_eth_fc_conf *fc_conf);
 110 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 111                                 struct rte_eth_fc_conf *fc_conf);
 112 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 113 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 114 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 115 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 116 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 117                                                         void *param);
 118 static int  igb_hardware_init(struct e1000_hw *hw);
 119 static void igb_hw_control_acquire(struct e1000_hw *hw);
 120 static void igb_hw_control_release(struct e1000_hw *hw);
 121 static void igb_init_manageability(struct e1000_hw *hw);
 122 static void igb_release_manageability(struct e1000_hw *hw);
 123
 124 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 125
 126 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 127                 uint16_t vlan_id, int on);
 128 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
 129                                  enum rte_vlan_type vlan_type,
 130                                  uint16_t tpid_id);
 131 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 132
 133 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 134 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 135 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 136 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 137 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 138 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 139
 140 static int eth_igb_led_on(struct rte_eth_dev *dev);
 141 static int eth_igb_led_off(struct rte_eth_dev *dev);
 142
 143 static void igb_intr_disable(struct e1000_hw *hw);
 144 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 145 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 146                 struct ether_addr *mac_addr,
 147                 uint32_t index, uint32_t pool);
 148 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 149 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 150                 struct ether_addr *addr);
 151
 152 static void igbvf_intr_disable(struct e1000_hw *hw);
 153 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 154 static int igbvf_dev_start(struct rte_eth_dev *dev);
 155 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 156 static void igbvf_dev_close(struct rte_eth_dev *dev);
 157 static int eth_igbvf_link_update(struct e1000_hw *hw);
 158 static void eth_igbvf_stats_get(struct rte_eth_dev *dev,
 159                                 struct rte_eth_stats *rte_stats);
 160 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev,
 161                                 struct rte_eth_xstats *xstats, unsigned n);
 162 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 163 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 164                 uint16_t vlan_id, int on);
 165 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 166 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 167 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 168                 struct ether_addr *addr);
 169 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 170 static int igbvf_get_regs(struct rte_eth_dev *dev,
 171                 struct rte_dev_reg_info *regs);
 172
 173 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 174                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 175                                    uint16_t reta_size);
 176 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 177                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 178                                   uint16_t reta_size);
 179
 180 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 181                         struct rte_eth_syn_filter *filter,
 182                         bool add);
 183 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 184                         struct rte_eth_syn_filter *filter);
 185 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 186                         enum rte_filter_op filter_op,
 187                         void *arg);
 188 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 189                         struct rte_eth_ntuple_filter *ntuple_filter);
 190 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 191                         struct rte_eth_ntuple_filter *ntuple_filter);
 192 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 193                         struct rte_eth_flex_filter *filter,
 194                         bool add);
 195 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 196                         struct rte_eth_flex_filter *filter);
 197 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 198                         enum rte_filter_op filter_op,
 199                         void *arg);
 200 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 201                         struct rte_eth_ntuple_filter *ntuple_filter);
 202 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 203                         struct rte_eth_ntuple_filter *ntuple_filter);
 204 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 205                         struct rte_eth_ntuple_filter *filter,
 206                         bool add);
 207 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 208                         struct rte_eth_ntuple_filter *filter);
 209 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 210                                 enum rte_filter_op filter_op,
 211                                 void *arg);
 212 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 213                         struct rte_eth_ethertype_filter *filter,
 214                         bool add);
 215 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 216                                 enum rte_filter_op filter_op,
 217                                 void *arg);
 218 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 219                         struct rte_eth_ethertype_filter *filter);
 220 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 221                      enum rte_filter_type filter_type,
 222                      enum rte_filter_op filter_op,
 223                      void *arg);
 224 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 225 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 226                 struct rte_dev_reg_info *regs);
 227 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 228 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 229                 struct rte_dev_eeprom_info *eeprom);
 230 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 231                 struct rte_dev_eeprom_info *eeprom);
 232 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 233                                     struct ether_addr *mc_addr_set,
 234                                     uint32_t nb_mc_addr);
 235 static int igb_timesync_enable(struct rte_eth_dev *dev);
 236 static int igb_timesync_disable(struct rte_eth_dev *dev);
 237 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 238                                           struct timespec *timestamp,
 239                                           uint32_t flags);
 240 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 241                                           struct timespec *timestamp);
 242 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 243 static int igb_timesync_read_time(struct rte_eth_dev *dev,
 244                                   struct timespec *timestamp);
 245 static int igb_timesync_write_time(struct rte_eth_dev *dev,
 246                                    const struct timespec *timestamp);
 247 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 248                                         uint16_t queue_id);
 249 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 250                                          uint16_t queue_id);
 251 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 252                                        uint8_t queue, uint8_t msix_vector);
 253 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 254                                uint8_t index, uint8_t offset);
 255 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 256
 257 /*
 258  * Define VF Stats MACRO for Non "cleared on read" register
 259  */
 260 #define UPDATE_VF_STAT(reg, last, cur)            \
 261 {                                                 \
 262         u32 latest = E1000_READ_REG(hw, reg);     \
 263         cur += (latest - last) & UINT_MAX;        \
 264         last = latest;                            \
 265 }
 266
 267 #define IGB_FC_PAUSE_TIME 0x0680
 268 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 269 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 270
 271 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 272
 273 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 274
 275 /*
 276  * The set of PCI devices this driver supports
 277  */
 278 static const struct rte_pci_id pci_id_igb_map[] = {
 279
 280 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 281 #include "rte_pci_dev_ids.h"
 282
 283 {0},
 284 };
 285
 286 /*
 287  * The set of PCI devices this driver supports (for 82576&I350 VF)
 288  */
 289 static const struct rte_pci_id pci_id_igbvf_map[] = {
 290
 291 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 292 #include "rte_pci_dev_ids.h"
 293
 294 {0},
 295 };
 296
 297 static const struct rte_eth_desc_lim rx_desc_lim = {
 298         .nb_max = E1000_MAX_RING_DESC,
 299         .nb_min = E1000_MIN_RING_DESC,
 300         .nb_align = IGB_RXD_ALIGN,
 301 };
 302
 303 static const struct rte_eth_desc_lim tx_desc_lim = {
 304         .nb_max = E1000_MAX_RING_DESC,
 305         .nb_min = E1000_MIN_RING_DESC,
 306         .nb_align = IGB_RXD_ALIGN,
 307 };
 308
 309 static const struct eth_dev_ops eth_igb_ops = {
 310         .dev_configure        = eth_igb_configure,
 311         .dev_start            = eth_igb_start,
 312         .dev_stop             = eth_igb_stop,
 313         .dev_close            = eth_igb_close,
 314         .promiscuous_enable   = eth_igb_promiscuous_enable,
 315         .promiscuous_disable  = eth_igb_promiscuous_disable,
 316         .allmulticast_enable  = eth_igb_allmulticast_enable,
 317         .allmulticast_disable = eth_igb_allmulticast_disable,
 318         .link_update          = eth_igb_link_update,
 319         .stats_get            = eth_igb_stats_get,
 320         .xstats_get           = eth_igb_xstats_get,
 321         .stats_reset          = eth_igb_stats_reset,
 322         .xstats_reset         = eth_igb_xstats_reset,
 323         .dev_infos_get        = eth_igb_infos_get,
 324         .mtu_set              = eth_igb_mtu_set,
 325         .vlan_filter_set      = eth_igb_vlan_filter_set,
 326         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 327         .vlan_offload_set     = eth_igb_vlan_offload_set,
 328         .rx_queue_setup       = eth_igb_rx_queue_setup,
 329         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 330         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 331         .rx_queue_release     = eth_igb_rx_queue_release,
 332         .rx_queue_count       = eth_igb_rx_queue_count,
 333         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 334         .tx_queue_setup       = eth_igb_tx_queue_setup,
 335         .tx_queue_release     = eth_igb_tx_queue_release,
 336         .dev_led_on           = eth_igb_led_on,
 337         .dev_led_off          = eth_igb_led_off,
 338         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 339         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 340         .mac_addr_add         = eth_igb_rar_set,
 341         .mac_addr_remove      = eth_igb_rar_clear,
 342         .mac_addr_set         = eth_igb_default_mac_addr_set,
 343         .reta_update          = eth_igb_rss_reta_update,
 344         .reta_query           = eth_igb_rss_reta_query,
 345         .rss_hash_update      = eth_igb_rss_hash_update,
 346         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 347         .filter_ctrl          = eth_igb_filter_ctrl,
 348         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 349         .rxq_info_get         = igb_rxq_info_get,
 350         .txq_info_get         = igb_txq_info_get,
 351         .timesync_enable      = igb_timesync_enable,
 352         .timesync_disable     = igb_timesync_disable,
 353         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 354         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 355         .get_reg_length       = eth_igb_get_reg_length,
 356         .get_reg              = eth_igb_get_regs,
 357         .get_eeprom_length    = eth_igb_get_eeprom_length,
 358         .get_eeprom           = eth_igb_get_eeprom,
 359         .set_eeprom           = eth_igb_set_eeprom,
 360         .timesync_adjust_time = igb_timesync_adjust_time,
 361         .timesync_read_time   = igb_timesync_read_time,
 362         .timesync_write_time  = igb_timesync_write_time,
 363 };
 364
 365 /*
 366  * dev_ops for virtual function, bare necessities for basic vf
 367  * operation have been implemented
 368  */
 369 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 370         .dev_configure        = igbvf_dev_configure,
 371         .dev_start            = igbvf_dev_start,
 372         .dev_stop             = igbvf_dev_stop,
 373         .dev_close            = igbvf_dev_close,
 374         .link_update          = eth_igb_link_update,
 375         .stats_get            = eth_igbvf_stats_get,
 376         .xstats_get           = eth_igbvf_xstats_get,
 377         .stats_reset          = eth_igbvf_stats_reset,
 378         .xstats_reset         = eth_igbvf_stats_reset,
 379         .vlan_filter_set      = igbvf_vlan_filter_set,
 380         .dev_infos_get        = eth_igbvf_infos_get,
 381         .rx_queue_setup       = eth_igb_rx_queue_setup,
 382         .rx_queue_release     = eth_igb_rx_queue_release,
 383         .tx_queue_setup       = eth_igb_tx_queue_setup,
 384         .tx_queue_release     = eth_igb_tx_queue_release,
 385         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 386         .rxq_info_get         = igb_rxq_info_get,
 387         .txq_info_get         = igb_txq_info_get,
 388         .mac_addr_set         = igbvf_default_mac_addr_set,
 389         .get_reg_length       = igbvf_get_reg_length,
 390         .get_reg              = igbvf_get_regs,
 391 };
 392
 393 /* store statistics names and its offset in stats structure */
 394 struct rte_igb_xstats_name_off {
 395         char name[RTE_ETH_XSTATS_NAME_SIZE];
 396         unsigned offset;
 397 };
 398
 399 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = {
 400         {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)},
 401         {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)},
 402         {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)},
 403         {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)},
 404         {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)},
 405         {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)},
 406         {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats,
 407                 ecol)},
 408         {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)},
 409         {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)},
 410         {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)},
 411         {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)},
 412         {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)},
 413         {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)},
 414         {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)},
 415         {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)},
 416         {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)},
 417         {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)},
 418         {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats,
 419                 fcruc)},
 420         {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)},
 421         {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)},
 422         {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)},
 423         {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)},
 424         {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 425                 prc1023)},
 426         {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats,
 427                 prc1522)},
 428         {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)},
 429         {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)},
 430         {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)},
 431         {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)},
 432         {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)},
 433         {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)},
 434         {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)},
 435         {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)},
 436         {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)},
 437         {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)},
 438         {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)},
 439         {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)},
 440         {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)},
 441         {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)},
 442         {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)},
 443         {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)},
 444         {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)},
 445         {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 446                 ptc1023)},
 447         {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats,
 448                 ptc1522)},
 449         {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)},
 450         {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)},
 451         {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)},
 452         {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)},
 453         {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)},
 454         {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)},
 455         {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)},
 456
 457         {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)},
 458 };
 459
 460 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \
 461                 sizeof(rte_igb_stats_strings[0]))
 462
 463 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = {
 464         {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)},
 465         {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)},
 466         {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)},
 467         {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)},
 468         {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)},
 469 };
 470
 471 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \
 472                 sizeof(rte_igbvf_stats_strings[0]))
 473
 474 /**
 475  * Atomically reads the link status information from global
 476  * structure rte_eth_dev.
 477  *
 478  * @param dev
 479  *   - Pointer to the structure rte_eth_dev to read from.
 480  *   - Pointer to the buffer to be saved with the link status.
 481  *
 482  * @return
 483  *   - On success, zero.
 484  *   - On failure, negative value.
 485  */
 486 static inline int
 487 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 488                                 struct rte_eth_link *link)
 489 {
 490         struct rte_eth_link *dst = link;
 491         struct rte_eth_link *src = &(dev->data->dev_link);
 492
 493         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 494                                         *(uint64_t *)src) == 0)
 495                 return -1;
 496
 497         return 0;
 498 }
 499
 500 /**
 501  * Atomically writes the link status information into global
 502  * structure rte_eth_dev.
 503  *
 504  * @param dev
 505  *   - Pointer to the structure rte_eth_dev to read from.
 506  *   - Pointer to the buffer to be saved with the link status.
 507  *
 508  * @return
 509  *   - On success, zero.
 510  *   - On failure, negative value.
 511  */
 512 static inline int
 513 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 514                                 struct rte_eth_link *link)
 515 {
 516         struct rte_eth_link *dst = &(dev->data->dev_link);
 517         struct rte_eth_link *src = link;
 518
 519         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 520                                         *(uint64_t *)src) == 0)
 521                 return -1;
 522
 523         return 0;
 524 }
 525
 526 static inline void
 527 igb_intr_enable(struct rte_eth_dev *dev)
 528 {
 529         struct e1000_interrupt *intr =
 530                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 531         struct e1000_hw *hw =
 532                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 533
 534         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 535         E1000_WRITE_FLUSH(hw);
 536 }
 537
 538 static void
 539 igb_intr_disable(struct e1000_hw *hw)
 540 {
 541         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 542         E1000_WRITE_FLUSH(hw);
 543 }
 544
 545 static inline int32_t
 546 igb_pf_reset_hw(struct e1000_hw *hw)
 547 {
 548         uint32_t ctrl_ext;
 549         int32_t status;
 550
 551         status = e1000_reset_hw(hw);
 552
 553         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 554         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 555         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 556         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 557         E1000_WRITE_FLUSH(hw);
 558
 559         return status;
 560 }
 561
 562 static void
 563 igb_identify_hardware(struct rte_eth_dev *dev)
 564 {
 565         struct e1000_hw *hw =
 566                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 567
 568         hw->vendor_id = dev->pci_dev->id.vendor_id;
 569         hw->device_id = dev->pci_dev->id.device_id;
 570         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 571         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 572
 573         e1000_set_mac_type(hw);
 574
 575         /* need to check if it is a vf device below */
 576 }
 577
 578 static int
 579 igb_reset_swfw_lock(struct e1000_hw *hw)
 580 {
 581         int ret_val;
 582
 583         /*
 584          * Do mac ops initialization manually here, since we will need
 585          * some function pointers set by this call.
 586          */
 587         ret_val = e1000_init_mac_params(hw);
 588         if (ret_val)
 589                 return ret_val;
 590
 591         /*
 592          * SMBI lock should not fail in this early stage. If this is the case,
 593          * it is due to an improper exit of the application.
 594          * So force the release of the faulty lock.
 595          */
 596         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 597                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 598         }
 599         e1000_put_hw_semaphore_generic(hw);
 600
 601         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 602                 uint16_t mask;
 603
 604                 /*
 605                  * Phy lock should not fail in this early stage. If this is the case,
 606                  * it is due to an improper exit of the application.
 607                  * So force the release of the faulty lock.
 608                  */
 609                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 610                 if (hw->bus.func > E1000_FUNC_1)
 611                         mask <<= 2;
 612                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 613                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 614                                     hw->bus.func);
 615                 }
 616                 hw->mac.ops.release_swfw_sync(hw, mask);
 617
 618                 /*
 619                  * This one is more tricky since it is common to all ports; but
 620                  * swfw_sync retries last long enough (1s) to be almost sure that if
 621                  * lock can not be taken it is due to an improper lock of the
 622                  * semaphore.
 623                  */
 624                 mask = E1000_SWFW_EEP_SM;
 625                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 626                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 627                 }
 628                 hw->mac.ops.release_swfw_sync(hw, mask);
 629         }
 630
 631         return E1000_SUCCESS;
 632 }
 633
 634 static int
 635 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 636 {
 637         int error = 0;
 638         struct rte_pci_device *pci_dev;
 639         struct e1000_hw *hw =
 640                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 641         struct e1000_vfta * shadow_vfta =
 642                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 643         struct e1000_filter_info *filter_info =
 644                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 645         struct e1000_adapter *adapter =
 646                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 647
 648         uint32_t ctrl_ext;
 649
 650         pci_dev = eth_dev->pci_dev;
 651
 652         eth_dev->dev_ops = &eth_igb_ops;
 653         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 654         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 655
 656         /* for secondary processes, we don't initialise any further as primary
 657          * has already done this work. Only check we don't need a different
 658          * RX function */
 659         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 660                 if (eth_dev->data->scattered_rx)
 661                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 662                 return 0;
 663         }
 664
 665         rte_eth_copy_pci_info(eth_dev, pci_dev);
 666
 667         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 668
 669         igb_identify_hardware(eth_dev);
 670         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 671                 error = -EIO;
 672                 goto err_late;
 673         }
 674
 675         e1000_get_bus_info(hw);
 676
 677         /* Reset any pending lock */
 678         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 679                 error = -EIO;
 680                 goto err_late;
 681         }
 682
 683         /* Finish initialization */
 684         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 685                 error = -EIO;
 686                 goto err_late;
 687         }
 688
 689         hw->mac.autoneg = 1;
 690         hw->phy.autoneg_wait_to_complete = 0;
 691         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 692
 693         /* Copper options */
 694         if (hw->phy.media_type == e1000_media_type_copper) {
 695                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 696                 hw->phy.disable_polarity_correction = 0;
 697                 hw->phy.ms_type = e1000_ms_hw_default;
 698         }
 699
 700         /*
 701          * Start from a known state, this is important in reading the nvm
 702          * and mac from that.
 703          */
 704         igb_pf_reset_hw(hw);
 705
 706         /* Make sure we have a good EEPROM before we read from it */
 707         if (e1000_validate_nvm_checksum(hw) < 0) {
 708                 /*
 709                  * Some PCI-E parts fail the first check due to
 710                  * the link being in sleep state, call it again,
 711                  * if it fails a second time its a real issue.
 712                  */
 713                 if (e1000_validate_nvm_checksum(hw) < 0) {
 714                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 715                         error = -EIO;
 716                         goto err_late;
 717                 }
 718         }
 719
 720         /* Read the permanent MAC address out of the EEPROM */
 721         if (e1000_read_mac_addr(hw) != 0) {
 722                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 723                 error = -EIO;
 724                 goto err_late;
 725         }
 726
 727         /* Allocate memory for storing MAC addresses */
 728         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 729                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 730         if (eth_dev->data->mac_addrs == NULL) {
 731                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 732                                                 "store MAC addresses",
 733                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 734                 error = -ENOMEM;
 735                 goto err_late;
 736         }
 737
 738         /* Copy the permanent MAC address */
 739         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 740
 741         /* initialize the vfta */
 742         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 743
 744         /* Now initialize the hardware */
 745         if (igb_hardware_init(hw) != 0) {
 746                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 747                 rte_free(eth_dev->data->mac_addrs);
 748                 eth_dev->data->mac_addrs = NULL;
 749                 error = -ENODEV;
 750                 goto err_late;
 751         }
 752         hw->mac.get_link_status = 1;
 753         adapter->stopped = 0;
 754
 755         /* Indicate SOL/IDER usage */
 756         if (e1000_check_reset_block(hw) < 0) {
 757                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 758                                         "SOL/IDER session");
 759         }
 760
 761         /* initialize PF if max_vfs not zero */
 762         igb_pf_host_init(eth_dev);
 763
 764         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 765         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 766         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 767         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 768         E1000_WRITE_FLUSH(hw);
 769
 770         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 771                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 772                      pci_dev->id.device_id);
 773
 774         rte_intr_callback_register(&pci_dev->intr_handle,
 775                                    eth_igb_interrupt_handler,
 776                                    (void *)eth_dev);
 777
 778         /* enable uio/vfio intr/eventfd mapping */
 779         rte_intr_enable(&pci_dev->intr_handle);
 780
 781         /* enable support intr */
 782         igb_intr_enable(eth_dev);
 783
 784         TAILQ_INIT(&filter_info->flex_list);
 785         filter_info->flex_mask = 0;
 786         TAILQ_INIT(&filter_info->twotuple_list);
 787         filter_info->twotuple_mask = 0;
 788         TAILQ_INIT(&filter_info->fivetuple_list);
 789         filter_info->fivetuple_mask = 0;
 790
 791         return 0;
 792
 793 err_late:
 794         igb_hw_control_release(hw);
 795
 796         return error;
 797 }
 798
 799 static int
 800 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 801 {
 802         struct rte_pci_device *pci_dev;
 803         struct e1000_hw *hw;
 804         struct e1000_adapter *adapter =
 805                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 806
 807         PMD_INIT_FUNC_TRACE();
 808
 809         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 810                 return -EPERM;
 811
 812         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 813         pci_dev = eth_dev->pci_dev;
 814
 815         if (adapter->stopped == 0)
 816                 eth_igb_close(eth_dev);
 817
 818         eth_dev->dev_ops = NULL;
 819         eth_dev->rx_pkt_burst = NULL;
 820         eth_dev->tx_pkt_burst = NULL;
 821
 822         /* Reset any pending lock */
 823         igb_reset_swfw_lock(hw);
 824
 825         rte_free(eth_dev->data->mac_addrs);
 826         eth_dev->data->mac_addrs = NULL;
 827
 828         /* uninitialize PF if max_vfs not zero */
 829         igb_pf_host_uninit(eth_dev);
 830
 831         /* disable uio intr before callback unregister */
 832         rte_intr_disable(&(pci_dev->intr_handle));
 833         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 834                 eth_igb_interrupt_handler, (void *)eth_dev);
 835
 836         return 0;
 837 }
 838
 839 /*
 840  * Virtual Function device init
 841  */
 842 static int
 843 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 844 {
 845         struct rte_pci_device *pci_dev;
 846         struct e1000_adapter *adapter =
 847                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 848         struct e1000_hw *hw =
 849                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 850         int diag;
 851         struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr;
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 856         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 857         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 858
 859         /* for secondary processes, we don't initialise any further as primary
 860          * has already done this work. Only check we don't need a different
 861          * RX function */
 862         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 863                 if (eth_dev->data->scattered_rx)
 864                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 865                 return 0;
 866         }
 867
 868         pci_dev = eth_dev->pci_dev;
 869
 870         rte_eth_copy_pci_info(eth_dev, pci_dev);
 871
 872         hw->device_id = pci_dev->id.device_id;
 873         hw->vendor_id = pci_dev->id.vendor_id;
 874         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 875         adapter->stopped = 0;
 876
 877         /* Initialize the shared code (base driver) */
 878         diag = e1000_setup_init_funcs(hw, TRUE);
 879         if (diag != 0) {
 880                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 881                         diag);
 882                 return -EIO;
 883         }
 884
 885         /* init_mailbox_params */
 886         hw->mbx.ops.init_params(hw);
 887
 888         /* Disable the interrupts for VF */
 889         igbvf_intr_disable(hw);
 890
 891         diag = hw->mac.ops.reset_hw(hw);
 892
 893         /* Allocate memory for storing MAC addresses */
 894         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 895                 hw->mac.rar_entry_count, 0);
 896         if (eth_dev->data->mac_addrs == NULL) {
 897                 PMD_INIT_LOG(ERR,
 898                         "Failed to allocate %d bytes needed to store MAC "
 899                         "addresses",
 900                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 901                 return -ENOMEM;
 902         }
 903
 904         /* Generate a random MAC address, if none was assigned by PF. */
 905         if (is_zero_ether_addr(perm_addr)) {
 906                 eth_random_addr(perm_addr->addr_bytes);
 907                 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0);
 908                 if (diag) {
 909                         rte_free(eth_dev->data->mac_addrs);
 910                         eth_dev->data->mac_addrs = NULL;
 911                         return diag;
 912                 }
 913                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
 914                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
 915                              "%02x:%02x:%02x:%02x:%02x:%02x",
 916                              perm_addr->addr_bytes[0],
 917                              perm_addr->addr_bytes[1],
 918                              perm_addr->addr_bytes[2],
 919                              perm_addr->addr_bytes[3],
 920                              perm_addr->addr_bytes[4],
 921                              perm_addr->addr_bytes[5]);
 922         }
 923
 924         /* Copy the permanent MAC address */
 925         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 926                         &eth_dev->data->mac_addrs[0]);
 927
 928         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 929                      "mac.type=%s",
 930                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 931                      pci_dev->id.device_id, "igb_mac_82576_vf");
 932
 933         return 0;
 934 }
 935
 936 static int
 937 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 938 {
 939         struct e1000_adapter *adapter =
 940                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 941
 942         PMD_INIT_FUNC_TRACE();
 943
 944         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 945                 return -EPERM;
 946
 947         if (adapter->stopped == 0)
 948                 igbvf_dev_close(eth_dev);
 949
 950         eth_dev->dev_ops = NULL;
 951         eth_dev->rx_pkt_burst = NULL;
 952         eth_dev->tx_pkt_burst = NULL;
 953
 954         rte_free(eth_dev->data->mac_addrs);
 955         eth_dev->data->mac_addrs = NULL;
 956
 957         return 0;
 958 }
 959
 960 static struct eth_driver rte_igb_pmd = {
 961         .pci_drv = {
 962                 .name = "rte_igb_pmd",
 963                 .id_table = pci_id_igb_map,
 964                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 965                         RTE_PCI_DRV_DETACHABLE,
 966         },
 967         .eth_dev_init = eth_igb_dev_init,
 968         .eth_dev_uninit = eth_igb_dev_uninit,
 969         .dev_private_size = sizeof(struct e1000_adapter),
 970 };
 971
 972 /*
 973  * virtual function driver struct
 974  */
 975 static struct eth_driver rte_igbvf_pmd = {
 976         .pci_drv = {
 977                 .name = "rte_igbvf_pmd",
 978                 .id_table = pci_id_igbvf_map,
 979                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
 980         },
 981         .eth_dev_init = eth_igbvf_dev_init,
 982         .eth_dev_uninit = eth_igbvf_dev_uninit,
 983         .dev_private_size = sizeof(struct e1000_adapter),
 984 };
 985
 986 static int
 987 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 988 {
 989         rte_eth_driver_register(&rte_igb_pmd);
 990         return 0;
 991 }
 992
 993 static void
 994 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 995 {
 996         struct e1000_hw *hw =
 997                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 998         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 999         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
1000         rctl |= E1000_RCTL_VFE;
1001         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1002 }
1003
1004 /*
1005  * VF Driver initialization routine.
1006  * Invoked one at EAL init time.
1007  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
1008  */
1009 static int
1010 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1011 {
1012         PMD_INIT_FUNC_TRACE();
1013
1014         rte_eth_driver_register(&rte_igbvf_pmd);
1015         return 0;
1016 }
1017
1018 static int
1019 igb_check_mq_mode(struct rte_eth_dev *dev)
1020 {
1021         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1022         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
1023         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1024         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1025
1026         if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) ||
1027             tx_mq_mode == ETH_MQ_TX_DCB ||
1028             tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1029                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
1030                 return -EINVAL;
1031         }
1032         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1033                 /* Check multi-queue mode.
1034                  * To no break software we accept ETH_MQ_RX_NONE as this might
1035                  * be used to turn off VLAN filter.
1036                  */
1037
1038                 if (rx_mq_mode == ETH_MQ_RX_NONE ||
1039                     rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1040                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1041                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1042                 } else {
1043                         /* Only support one queue on VFs.
1044                          * RSS together with SRIOV is not supported.
1045                          */
1046                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1047                                         " wrong mq_mode rx %d.",
1048                                         rx_mq_mode);
1049                         return -EINVAL;
1050                 }
1051                 /* TX mode is not used here, so mode might be ignored.*/
1052                 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1053                         /* SRIOV only works in VMDq enable mode */
1054                         PMD_INIT_LOG(WARNING, "SRIOV is active,"
1055                                         " TX mode %d is not supported. "
1056                                         " Driver will behave as %d mode.",
1057                                         tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY);
1058                 }
1059
1060                 /* check valid queue number */
1061                 if ((nb_rx_q > 1) || (nb_tx_q > 1)) {
1062                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1063                                         " only support one queue on VFs.");
1064                         return -EINVAL;
1065                 }
1066         } else {
1067                 /* To no break software that set invalid mode, only display
1068                  * warning if invalid mode is used.
1069                  */
1070                 if (rx_mq_mode != ETH_MQ_RX_NONE &&
1071                     rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY &&
1072                     rx_mq_mode != ETH_MQ_RX_RSS) {
1073                         /* RSS together with VMDq not supported*/
1074                         PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
1075                                      rx_mq_mode);
1076                         return -EINVAL;
1077                 }
1078
1079                 if (tx_mq_mode != ETH_MQ_TX_NONE &&
1080                     tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1081                         PMD_INIT_LOG(WARNING, "TX mode %d is not supported."
1082                                         " Due to txmode is meaningless in this"
1083                                         " driver, just ignore.",
1084                                         tx_mq_mode);
1085                 }
1086         }
1087         return 0;
1088 }
1089
1090 static int
1091 eth_igb_configure(struct rte_eth_dev *dev)
1092 {
1093         struct e1000_interrupt *intr =
1094                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1095         int ret;
1096
1097         PMD_INIT_FUNC_TRACE();
1098
1099         /* multipe queue mode checking */
1100         ret  = igb_check_mq_mode(dev);
1101         if (ret != 0) {
1102                 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.",
1103                             ret);
1104                 return ret;
1105         }
1106
1107         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1108         PMD_INIT_FUNC_TRACE();
1109
1110         return 0;
1111 }
1112
1113 static int
1114 eth_igb_start(struct rte_eth_dev *dev)
1115 {
1116         struct e1000_hw *hw =
1117                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1118         struct e1000_adapter *adapter =
1119                 E1000_DEV_PRIVATE(dev->data->dev_private);
1120         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1121         int ret, mask;
1122         uint32_t intr_vector = 0;
1123         uint32_t ctrl_ext;
1124
1125         PMD_INIT_FUNC_TRACE();
1126
1127         /* disable uio/vfio intr/eventfd mapping */
1128         rte_intr_disable(intr_handle);
1129
1130         /* Power up the phy. Needed to make the link go Up */
1131         e1000_power_up_phy(hw);
1132
1133         /*
1134          * Packet Buffer Allocation (PBA)
1135          * Writing PBA sets the receive portion of the buffer
1136          * the remainder is used for the transmit buffer.
1137          */
1138         if (hw->mac.type == e1000_82575) {
1139                 uint32_t pba;
1140
1141                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1142                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1143         }
1144
1145         /* Put the address into the Receive Address Array */
1146         e1000_rar_set(hw, hw->mac.addr, 0);
1147
1148         /* Initialize the hardware */
1149         if (igb_hardware_init(hw)) {
1150                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
1151                 return -EIO;
1152         }
1153         adapter->stopped = 0;
1154
1155         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1156
1157         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1158         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1159         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
1160         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1161         E1000_WRITE_FLUSH(hw);
1162
1163         /* configure PF module if SRIOV enabled */
1164         igb_pf_host_configure(dev);
1165
1166         /* check and configure queue intr-vector mapping */
1167         if ((rte_intr_cap_multiple(intr_handle) ||
1168              !RTE_ETH_DEV_SRIOV(dev).active) &&
1169             dev->data->dev_conf.intr_conf.rxq != 0) {
1170                 intr_vector = dev->data->nb_rx_queues;
1171                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1172                         return -1;
1173         }
1174
1175         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1176                 intr_handle->intr_vec =
1177                         rte_zmalloc("intr_vec",
1178                                     dev->data->nb_rx_queues * sizeof(int), 0);
1179                 if (intr_handle->intr_vec == NULL) {
1180                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1181                                      " intr_vec\n", dev->data->nb_rx_queues);
1182                         return -ENOMEM;
1183                 }
1184         }
1185
1186         /* confiugre msix for rx interrupt */
1187         eth_igb_configure_msix_intr(dev);
1188
1189         /* Configure for OS presence */
1190         igb_init_manageability(hw);
1191
1192         eth_igb_tx_init(dev);
1193
1194         /* This can fail when allocating mbufs for descriptor rings */
1195         ret = eth_igb_rx_init(dev);
1196         if (ret) {
1197                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1198                 igb_dev_clear_queues(dev);
1199                 return ret;
1200         }
1201
1202         e1000_clear_hw_cntrs_base_generic(hw);
1203
1204         /*
1205          * VLAN Offload Settings
1206          */
1207         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1208                         ETH_VLAN_EXTEND_MASK;
1209         eth_igb_vlan_offload_set(dev, mask);
1210
1211         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1212                 /* Enable VLAN filter since VMDq always use VLAN filter */
1213                 igb_vmdq_vlan_hw_filter_enable(dev);
1214         }
1215
1216         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
1217                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
1218                 (hw->mac.type == e1000_i211)) {
1219                 /* Configure EITR with the maximum possible value (0xFFFF) */
1220                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
1221         }
1222
1223         /* Setup link speed and duplex */
1224         switch (dev->data->dev_conf.link_speed) {
1225         case ETH_LINK_SPEED_AUTONEG:
1226                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1227                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1228                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1229                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
1230                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1231                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
1232                 else
1233                         goto error_invalid_config;
1234                 break;
1235         case ETH_LINK_SPEED_10:
1236                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1237                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
1238                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1239                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
1240                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1241                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
1242                 else
1243                         goto error_invalid_config;
1244                 break;
1245         case ETH_LINK_SPEED_100:
1246                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1247                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
1248                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1249                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
1250                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1251                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
1252                 else
1253                         goto error_invalid_config;
1254                 break;
1255         case ETH_LINK_SPEED_1000:
1256                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
1257                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
1258                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
1259                 else
1260                         goto error_invalid_config;
1261                 break;
1262         case ETH_LINK_SPEED_10000:
1263         default:
1264                 goto error_invalid_config;
1265         }
1266         e1000_setup_link(hw);
1267
1268         if (rte_intr_allow_others(intr_handle)) {
1269                 /* check if lsc interrupt is enabled */
1270                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1271                         eth_igb_lsc_interrupt_setup(dev);
1272         } else {
1273                 rte_intr_callback_unregister(intr_handle,
1274                                              eth_igb_interrupt_handler,
1275                                              (void *)dev);
1276                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1277                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1278                                      " no intr multiplex\n");
1279         }
1280
1281         /* check if rxq interrupt is enabled */
1282         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1283             rte_intr_dp_is_en(intr_handle))
1284                 eth_igb_rxq_interrupt_setup(dev);
1285
1286         /* enable uio/vfio intr/eventfd mapping */
1287         rte_intr_enable(intr_handle);
1288
1289         /* resume enabled intr since hw reset */
1290         igb_intr_enable(dev);
1291
1292         PMD_INIT_LOG(DEBUG, "<<");
1293
1294         return 0;
1295
1296 error_invalid_config:
1297         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
1298                      dev->data->dev_conf.link_speed,
1299                      dev->data->dev_conf.link_duplex, dev->data->port_id);
1300         igb_dev_clear_queues(dev);
1301         return -EINVAL;
1302 }
1303
1304 /*********************************************************************
1305  *
1306  *  This routine disables all traffic on the adapter by issuing a
1307  *  global reset on the MAC.
1308  *
1309  **********************************************************************/
1310 static void
1311 eth_igb_stop(struct rte_eth_dev *dev)
1312 {
1313         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1314         struct e1000_filter_info *filter_info =
1315                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1316         struct rte_eth_link link;
1317         struct e1000_flex_filter *p_flex;
1318         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1319         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1320         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1321
1322         igb_intr_disable(hw);
1323
1324         /* disable intr eventfd mapping */
1325         rte_intr_disable(intr_handle);
1326
1327         igb_pf_reset_hw(hw);
1328         E1000_WRITE_REG(hw, E1000_WUC, 0);
1329
1330         /* Set bit for Go Link disconnect */
1331         if (hw->mac.type >= e1000_82580) {
1332                 uint32_t phpm_reg;
1333
1334                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1335                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1336                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1337         }
1338
1339         /* Power down the phy. Needed to make the link go Down */
1340         if (hw->phy.media_type == e1000_media_type_copper)
1341                 e1000_power_down_phy(hw);
1342         else
1343                 e1000_shutdown_fiber_serdes_link(hw);
1344
1345         igb_dev_clear_queues(dev);
1346
1347         /* clear the recorded link status */
1348         memset(&link, 0, sizeof(link));
1349         rte_igb_dev_atomic_write_link_status(dev, &link);
1350
1351         /* Remove all flex filters of the device */
1352         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1353                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1354                 rte_free(p_flex);
1355         }
1356         filter_info->flex_mask = 0;
1357
1358         /* Remove all ntuple filters of the device */
1359         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1360              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1361                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1362                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1363                              p_5tuple, entries);
1364                 rte_free(p_5tuple);
1365         }
1366         filter_info->fivetuple_mask = 0;
1367         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1368              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1369                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1370                 TAILQ_REMOVE(&filter_info->twotuple_list,
1371                              p_2tuple, entries);
1372                 rte_free(p_2tuple);
1373         }
1374         filter_info->twotuple_mask = 0;
1375
1376         if (!rte_intr_allow_others(intr_handle))
1377                 /* resume to the default handler */
1378                 rte_intr_callback_register(intr_handle,
1379                                            eth_igb_interrupt_handler,
1380                                            (void *)dev);
1381
1382         /* Clean datapath event and queue/vec mapping */
1383         rte_intr_efd_disable(intr_handle);
1384         if (intr_handle->intr_vec != NULL) {
1385                 rte_free(intr_handle->intr_vec);
1386                 intr_handle->intr_vec = NULL;
1387         }
1388 }
1389
1390 static void
1391 eth_igb_close(struct rte_eth_dev *dev)
1392 {
1393         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1394         struct e1000_adapter *adapter =
1395                 E1000_DEV_PRIVATE(dev->data->dev_private);
1396         struct rte_eth_link link;
1397         struct rte_pci_device *pci_dev;
1398
1399         eth_igb_stop(dev);
1400         adapter->stopped = 1;
1401
1402         e1000_phy_hw_reset(hw);
1403         igb_release_manageability(hw);
1404         igb_hw_control_release(hw);
1405
1406         /* Clear bit for Go Link disconnect */
1407         if (hw->mac.type >= e1000_82580) {
1408                 uint32_t phpm_reg;
1409
1410                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1411                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1412                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1413         }
1414
1415         igb_dev_free_queues(dev);
1416
1417         pci_dev = dev->pci_dev;
1418         if (pci_dev->intr_handle.intr_vec) {
1419                 rte_free(pci_dev->intr_handle.intr_vec);
1420                 pci_dev->intr_handle.intr_vec = NULL;
1421         }
1422
1423         memset(&link, 0, sizeof(link));
1424         rte_igb_dev_atomic_write_link_status(dev, &link);
1425 }
1426
1427 static int
1428 igb_get_rx_buffer_size(struct e1000_hw *hw)
1429 {
1430         uint32_t rx_buf_size;
1431         if (hw->mac.type == e1000_82576) {
1432                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1433         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1434                 /* PBS needs to be translated according to a lookup table */
1435                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1436                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1437                 rx_buf_size = (rx_buf_size << 10);
1438         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1439                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1440         } else {
1441                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1442         }
1443
1444         return rx_buf_size;
1445 }
1446
1447 /*********************************************************************
1448  *
1449  *  Initialize the hardware
1450  *
1451  **********************************************************************/
1452 static int
1453 igb_hardware_init(struct e1000_hw *hw)
1454 {
1455         uint32_t rx_buf_size;
1456         int diag;
1457
1458         /* Let the firmware know the OS is in control */
1459         igb_hw_control_acquire(hw);
1460
1461         /*
1462          * These parameters control the automatic generation (Tx) and
1463          * response (Rx) to Ethernet PAUSE frames.
1464          * - High water mark should allow for at least two standard size (1518)
1465          *   frames to be received after sending an XOFF.
1466          * - Low water mark works best when it is very near the high water mark.
1467          *   This allows the receiver to restart by sending XON when it has
1468          *   drained a bit. Here we use an arbitrary value of 1500 which will
1469          *   restart after one full frame is pulled from the buffer. There
1470          *   could be several smaller frames in the buffer and if so they will
1471          *   not trigger the XON until their total number reduces the buffer
1472          *   by 1500.
1473          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1474          */
1475         rx_buf_size = igb_get_rx_buffer_size(hw);
1476
1477         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1478         hw->fc.low_water = hw->fc.high_water - 1500;
1479         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1480         hw->fc.send_xon = 1;
1481
1482         /* Set Flow control, use the tunable location if sane */
1483         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1484                 hw->fc.requested_mode = igb_fc_setting;
1485         else
1486                 hw->fc.requested_mode = e1000_fc_none;
1487
1488         /* Issue a global reset */
1489         igb_pf_reset_hw(hw);
1490         E1000_WRITE_REG(hw, E1000_WUC, 0);
1491
1492         diag = e1000_init_hw(hw);
1493         if (diag < 0)
1494                 return diag;
1495
1496         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1497         e1000_get_phy_info(hw);
1498         e1000_check_for_link(hw);
1499
1500         return 0;
1501 }
1502
1503 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1504 static void
1505 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats)
1506 {
1507         int pause_frames;
1508
1509         uint64_t old_gprc  = stats->gprc;
1510         uint64_t old_gptc  = stats->gptc;
1511         uint64_t old_tpr   = stats->tpr;
1512         uint64_t old_tpt   = stats->tpt;
1513         uint64_t old_rpthc = stats->rpthc;
1514         uint64_t old_hgptc = stats->hgptc;
1515
1516         if(hw->phy.media_type == e1000_media_type_copper ||
1517             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1518                 stats->symerrs +=
1519                     E1000_READ_REG(hw,E1000_SYMERRS);
1520                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1521         }
1522
1523         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1524         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1525         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1526         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1527
1528         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1529         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1530         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1531         stats->dc += E1000_READ_REG(hw, E1000_DC);
1532         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1533         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1534         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1535         /*
1536         ** For watchdog management we need to know if we have been
1537         ** paused during the last interval, so capture that here.
1538         */
1539         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1540         stats->xoffrxc += pause_frames;
1541         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1542         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1543         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1544         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1545         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1546         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1547         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1548         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1549         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1550         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1551         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1552         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1553
1554         /* For the 64-bit byte counters the low dword must be read first. */
1555         /* Both registers clear on the read of the high dword */
1556
1557         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1558         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1559         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1560         stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN;
1561         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1562         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1563         stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN;
1564
1565         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1566         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1567         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1568         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1569         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1570
1571         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1572         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1573
1574         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1575         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1576         stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN;
1577         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1578         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1579         stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN;
1580
1581         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1582         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1583         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1584         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1585         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1586         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1587         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1588         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1589
1590         /* Interrupt Counts */
1591
1592         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1593         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1594         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1595         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1596         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1597         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1598         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1599         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1600         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1601
1602         /* Host to Card Statistics */
1603
1604         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1605         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1606         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1607         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1608         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1609         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1610         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1611         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1612         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1613         stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN;
1614         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1615         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1616         stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN;
1617         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1618         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1619         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1620
1621         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1622         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1623         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1624         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1625         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1626         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1627 }
1628
1629 static void
1630 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1631 {
1632         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1633         struct e1000_hw_stats *stats =
1634                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1635
1636         igb_read_stats_registers(hw, stats);
1637
1638         if (rte_stats == NULL)
1639                 return;
1640
1641         /* Rx Errors */
1642         rte_stats->imissed = stats->mpc;
1643         rte_stats->ierrors = stats->crcerrs +
1644                              stats->rlec + stats->ruc + stats->roc +
1645                              rte_stats->imissed +
1646                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1647
1648         /* Tx Errors */
1649         rte_stats->oerrors = stats->ecol + stats->latecol;
1650
1651         rte_stats->ipackets = stats->gprc;
1652         rte_stats->opackets = stats->gptc;
1653         rte_stats->ibytes   = stats->gorc;
1654         rte_stats->obytes   = stats->gotc;
1655 }
1656
1657 static void
1658 eth_igb_stats_reset(struct rte_eth_dev *dev)
1659 {
1660         struct e1000_hw_stats *hw_stats =
1661                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1662
1663         /* HW registers are cleared on read */
1664         eth_igb_stats_get(dev, NULL);
1665
1666         /* Reset software totals */
1667         memset(hw_stats, 0, sizeof(*hw_stats));
1668 }
1669
1670 static void
1671 eth_igb_xstats_reset(struct rte_eth_dev *dev)
1672 {
1673         struct e1000_hw_stats *stats =
1674                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1675
1676         /* HW registers are cleared on read */
1677         eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS);
1678
1679         /* Reset software totals */
1680         memset(stats, 0, sizeof(*stats));
1681 }
1682
1683 static int
1684 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1685                    unsigned n)
1686 {
1687         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1688         struct e1000_hw_stats *hw_stats =
1689                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1690         unsigned i;
1691
1692         if (n < IGB_NB_XSTATS)
1693                 return IGB_NB_XSTATS;
1694
1695         igb_read_stats_registers(hw, hw_stats);
1696
1697         /* If this is a reset xstats is NULL, and we have cleared the
1698          * registers by reading them.
1699          */
1700         if (!xstats)
1701                 return 0;
1702
1703         /* Extended stats */
1704         for (i = 0; i < IGB_NB_XSTATS; i++) {
1705                 snprintf(xstats[i].name, sizeof(xstats[i].name),
1706                          "%s", rte_igb_stats_strings[i].name);
1707                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1708                         rte_igb_stats_strings[i].offset);
1709         }
1710
1711         return IGB_NB_XSTATS;
1712 }
1713
1714 static void
1715 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats)
1716 {
1717         /* Good Rx packets, include VF loopback */
1718         UPDATE_VF_STAT(E1000_VFGPRC,
1719             hw_stats->last_gprc, hw_stats->gprc);
1720
1721         /* Good Rx octets, include VF loopback */
1722         UPDATE_VF_STAT(E1000_VFGORC,
1723             hw_stats->last_gorc, hw_stats->gorc);
1724
1725         /* Good Tx packets, include VF loopback */
1726         UPDATE_VF_STAT(E1000_VFGPTC,
1727             hw_stats->last_gptc, hw_stats->gptc);
1728
1729         /* Good Tx octets, include VF loopback */
1730         UPDATE_VF_STAT(E1000_VFGOTC,
1731             hw_stats->last_gotc, hw_stats->gotc);
1732
1733         /* Rx Multicst packets */
1734         UPDATE_VF_STAT(E1000_VFMPRC,
1735             hw_stats->last_mprc, hw_stats->mprc);
1736
1737         /* Good Rx loopback packets */
1738         UPDATE_VF_STAT(E1000_VFGPRLBC,
1739             hw_stats->last_gprlbc, hw_stats->gprlbc);
1740
1741         /* Good Rx loopback octets */
1742         UPDATE_VF_STAT(E1000_VFGORLBC,
1743             hw_stats->last_gorlbc, hw_stats->gorlbc);
1744
1745         /* Good Tx loopback packets */
1746         UPDATE_VF_STAT(E1000_VFGPTLBC,
1747             hw_stats->last_gptlbc, hw_stats->gptlbc);
1748
1749         /* Good Tx loopback octets */
1750         UPDATE_VF_STAT(E1000_VFGOTLBC,
1751             hw_stats->last_gotlbc, hw_stats->gotlbc);
1752 }
1753
1754 static int
1755 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1756                      unsigned n)
1757 {
1758         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1759         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1760                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1761         unsigned i;
1762
1763         if (n < IGBVF_NB_XSTATS)
1764                 return IGBVF_NB_XSTATS;
1765
1766         igbvf_read_stats_registers(hw, hw_stats);
1767
1768         if (!xstats)
1769                 return 0;
1770
1771         for (i = 0; i < IGBVF_NB_XSTATS; i++) {
1772                 snprintf(xstats[i].name, sizeof(xstats[i].name), "%s",
1773                          rte_igbvf_stats_strings[i].name);
1774                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1775                         rte_igbvf_stats_strings[i].offset);
1776         }
1777
1778         return IGBVF_NB_XSTATS;
1779 }
1780
1781 static void
1782 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1783 {
1784         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1785         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1786                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1787
1788         igbvf_read_stats_registers(hw, hw_stats);
1789
1790         if (rte_stats == NULL)
1791                 return;
1792
1793         rte_stats->ipackets = hw_stats->gprc;
1794         rte_stats->ibytes = hw_stats->gorc;
1795         rte_stats->opackets = hw_stats->gptc;
1796         rte_stats->obytes = hw_stats->gotc;
1797         rte_stats->imcasts = hw_stats->mprc;
1798         rte_stats->ilbpackets = hw_stats->gprlbc;
1799         rte_stats->ilbbytes = hw_stats->gorlbc;
1800         rte_stats->olbpackets = hw_stats->gptlbc;
1801         rte_stats->olbbytes = hw_stats->gotlbc;
1802 }
1803
1804 static void
1805 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1806 {
1807         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1808                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1809
1810         /* Sync HW register to the last stats */
1811         eth_igbvf_stats_get(dev, NULL);
1812
1813         /* reset HW current stats*/
1814         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1815                offsetof(struct e1000_vf_stats, gprc));
1816 }
1817
1818 static void
1819 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1820 {
1821         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1822
1823         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1824         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1825         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1826         dev_info->rx_offload_capa =
1827                 DEV_RX_OFFLOAD_VLAN_STRIP |
1828                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1829                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1830                 DEV_RX_OFFLOAD_TCP_CKSUM;
1831         dev_info->tx_offload_capa =
1832                 DEV_TX_OFFLOAD_VLAN_INSERT |
1833                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1834                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1835                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1836                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1837                 DEV_TX_OFFLOAD_TCP_TSO;
1838
1839         switch (hw->mac.type) {
1840         case e1000_82575:
1841                 dev_info->max_rx_queues = 4;
1842                 dev_info->max_tx_queues = 4;
1843                 dev_info->max_vmdq_pools = 0;
1844                 break;
1845
1846         case e1000_82576:
1847                 dev_info->max_rx_queues = 16;
1848                 dev_info->max_tx_queues = 16;
1849                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1850                 dev_info->vmdq_queue_num = 16;
1851                 break;
1852
1853         case e1000_82580:
1854                 dev_info->max_rx_queues = 8;
1855                 dev_info->max_tx_queues = 8;
1856                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1857                 dev_info->vmdq_queue_num = 8;
1858                 break;
1859
1860         case e1000_i350:
1861                 dev_info->max_rx_queues = 8;
1862                 dev_info->max_tx_queues = 8;
1863                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1864                 dev_info->vmdq_queue_num = 8;
1865                 break;
1866
1867         case e1000_i354:
1868                 dev_info->max_rx_queues = 8;
1869                 dev_info->max_tx_queues = 8;
1870                 break;
1871
1872         case e1000_i210:
1873                 dev_info->max_rx_queues = 4;
1874                 dev_info->max_tx_queues = 4;
1875                 dev_info->max_vmdq_pools = 0;
1876                 break;
1877
1878         case e1000_i211:
1879                 dev_info->max_rx_queues = 2;
1880                 dev_info->max_tx_queues = 2;
1881                 dev_info->max_vmdq_pools = 0;
1882                 break;
1883
1884         default:
1885                 /* Should not happen */
1886                 break;
1887         }
1888         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1889         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1890         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1891
1892         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1893                 .rx_thresh = {
1894                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1895                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1896                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1897                 },
1898                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1899                 .rx_drop_en = 0,
1900         };
1901
1902         dev_info->default_txconf = (struct rte_eth_txconf) {
1903                 .tx_thresh = {
1904                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1905                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1906                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1907                 },
1908                 .txq_flags = 0,
1909         };
1910
1911         dev_info->rx_desc_lim = rx_desc_lim;
1912         dev_info->tx_desc_lim = tx_desc_lim;
1913 }
1914
1915 static void
1916 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1917 {
1918         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1919
1920         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1921         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1922         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1923         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1924                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1925                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1926                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1927         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1928                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1929                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1930                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1931                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1932                                 DEV_TX_OFFLOAD_TCP_TSO;
1933         switch (hw->mac.type) {
1934         case e1000_vfadapt:
1935                 dev_info->max_rx_queues = 2;
1936                 dev_info->max_tx_queues = 2;
1937                 break;
1938         case e1000_vfadapt_i350:
1939                 dev_info->max_rx_queues = 1;
1940                 dev_info->max_tx_queues = 1;
1941                 break;
1942         default:
1943                 /* Should not happen */
1944                 break;
1945         }
1946
1947         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1948                 .rx_thresh = {
1949                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1950                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1951                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1952                 },
1953                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1954                 .rx_drop_en = 0,
1955         };
1956
1957         dev_info->default_txconf = (struct rte_eth_txconf) {
1958                 .tx_thresh = {
1959                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1960                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1961                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1962                 },
1963                 .txq_flags = 0,
1964         };
1965
1966         dev_info->rx_desc_lim = rx_desc_lim;
1967         dev_info->tx_desc_lim = tx_desc_lim;
1968 }
1969
1970 /* return 0 means link status changed, -1 means not changed */
1971 static int
1972 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1973 {
1974         struct e1000_hw *hw =
1975                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1976         struct rte_eth_link link, old;
1977         int link_check, count;
1978
1979         link_check = 0;
1980         hw->mac.get_link_status = 1;
1981
1982         /* possible wait-to-complete in up to 9 seconds */
1983         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1984                 /* Read the real link status */
1985                 switch (hw->phy.media_type) {
1986                 case e1000_media_type_copper:
1987                         /* Do the work to read phy */
1988                         e1000_check_for_link(hw);
1989                         link_check = !hw->mac.get_link_status;
1990                         break;
1991
1992                 case e1000_media_type_fiber:
1993                         e1000_check_for_link(hw);
1994                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1995                                       E1000_STATUS_LU);
1996                         break;
1997
1998                 case e1000_media_type_internal_serdes:
1999                         e1000_check_for_link(hw);
2000                         link_check = hw->mac.serdes_has_link;
2001                         break;
2002
2003                 /* VF device is type_unknown */
2004                 case e1000_media_type_unknown:
2005                         eth_igbvf_link_update(hw);
2006                         link_check = !hw->mac.get_link_status;
2007                         break;
2008
2009                 default:
2010                         break;
2011                 }
2012                 if (link_check || wait_to_complete == 0)
2013                         break;
2014                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
2015         }
2016         memset(&link, 0, sizeof(link));
2017         rte_igb_dev_atomic_read_link_status(dev, &link);
2018         old = link;
2019
2020         /* Now we check if a transition has happened */
2021         if (link_check) {
2022                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
2023                                           &link.link_duplex);
2024                 link.link_status = 1;
2025         } else if (!link_check) {
2026                 link.link_speed = 0;
2027                 link.link_duplex = 0;
2028                 link.link_status = 0;
2029         }
2030         rte_igb_dev_atomic_write_link_status(dev, &link);
2031
2032         /* not changed */
2033         if (old.link_status == link.link_status)
2034                 return -1;
2035
2036         /* changed */
2037         return 0;
2038 }
2039
2040 /*
2041  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
2042  * For ASF and Pass Through versions of f/w this means
2043  * that the driver is loaded.
2044  */
2045 static void
2046 igb_hw_control_acquire(struct e1000_hw *hw)
2047 {
2048         uint32_t ctrl_ext;
2049
2050         /* Let firmware know the driver has taken over */
2051         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2052         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2053 }
2054
2055 /*
2056  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
2057  * For ASF and Pass Through versions of f/w this means that the
2058  * driver is no longer loaded.
2059  */
2060 static void
2061 igb_hw_control_release(struct e1000_hw *hw)
2062 {
2063         uint32_t ctrl_ext;
2064
2065         /* Let firmware taken over control of h/w */
2066         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2067         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
2068                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2069 }
2070
2071 /*
2072  * Bit of a misnomer, what this really means is
2073  * to enable OS management of the system... aka
2074  * to disable special hardware management features.
2075  */
2076 static void
2077 igb_init_manageability(struct e1000_hw *hw)
2078 {
2079         if (e1000_enable_mng_pass_thru(hw)) {
2080                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
2081                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2082
2083                 /* disable hardware interception of ARP */
2084                 manc &= ~(E1000_MANC_ARP_EN);
2085
2086                 /* enable receiving management packets to the host */
2087                 manc |= E1000_MANC_EN_MNG2HOST;
2088                 manc2h |= 1 << 5;  /* Mng Port 623 */
2089                 manc2h |= 1 << 6;  /* Mng Port 664 */
2090                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
2091                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2092         }
2093 }
2094
2095 static void
2096 igb_release_manageability(struct e1000_hw *hw)
2097 {
2098         if (e1000_enable_mng_pass_thru(hw)) {
2099                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2100
2101                 manc |= E1000_MANC_ARP_EN;
2102                 manc &= ~E1000_MANC_EN_MNG2HOST;
2103
2104                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2105         }
2106 }
2107
2108 static void
2109 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
2110 {
2111         struct e1000_hw *hw =
2112                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2113         uint32_t rctl;
2114
2115         rctl = E1000_READ_REG(hw, E1000_RCTL);
2116         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2117         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2118 }
2119
2120 static void
2121 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
2122 {
2123         struct e1000_hw *hw =
2124                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2125         uint32_t rctl;
2126
2127         rctl = E1000_READ_REG(hw, E1000_RCTL);
2128         rctl &= (~E1000_RCTL_UPE);
2129         if (dev->data->all_multicast == 1)
2130                 rctl |= E1000_RCTL_MPE;
2131         else
2132                 rctl &= (~E1000_RCTL_MPE);
2133         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2134 }
2135
2136 static void
2137 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
2138 {
2139         struct e1000_hw *hw =
2140                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2141         uint32_t rctl;
2142
2143         rctl = E1000_READ_REG(hw, E1000_RCTL);
2144         rctl |= E1000_RCTL_MPE;
2145         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2146 }
2147
2148 static void
2149 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
2150 {
2151         struct e1000_hw *hw =
2152                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2153         uint32_t rctl;
2154
2155         if (dev->data->promiscuous == 1)
2156                 return; /* must remain in all_multicast mode */
2157         rctl = E1000_READ_REG(hw, E1000_RCTL);
2158         rctl &= (~E1000_RCTL_MPE);
2159         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2160 }
2161
2162 static int
2163 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2164 {
2165         struct e1000_hw *hw =
2166                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2167         struct e1000_vfta * shadow_vfta =
2168                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2169         uint32_t vfta;
2170         uint32_t vid_idx;
2171         uint32_t vid_bit;
2172
2173         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
2174                               E1000_VFTA_ENTRY_MASK);
2175         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
2176         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
2177         if (on)
2178                 vfta |= vid_bit;
2179         else
2180                 vfta &= ~vid_bit;
2181         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
2182
2183         /* update local VFTA copy */
2184         shadow_vfta->vfta[vid_idx] = vfta;
2185
2186         return 0;
2187 }
2188
2189 static int
2190 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
2191                       enum rte_vlan_type vlan_type,
2192                       uint16_t tpid)
2193 {
2194         struct e1000_hw *hw =
2195                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2196         uint32_t reg = ETHER_TYPE_VLAN;
2197         int ret = 0;
2198
2199         switch (vlan_type) {
2200         case ETH_VLAN_TYPE_INNER:
2201                 reg |= (tpid << 16);
2202                 E1000_WRITE_REG(hw, E1000_VET, reg);
2203                 break;
2204         default:
2205                 ret = -EINVAL;
2206                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d\n", vlan_type);
2207                 break;
2208         }
2209
2210         return ret;
2211 }
2212
2213 static void
2214 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2215 {
2216         struct e1000_hw *hw =
2217                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2218         uint32_t reg;
2219
2220         /* Filter Table Disable */
2221         reg = E1000_READ_REG(hw, E1000_RCTL);
2222         reg &= ~E1000_RCTL_CFIEN;
2223         reg &= ~E1000_RCTL_VFE;
2224         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2225 }
2226
2227 static void
2228 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2229 {
2230         struct e1000_hw *hw =
2231                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2232         struct e1000_vfta * shadow_vfta =
2233                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2234         uint32_t reg;
2235         int i;
2236
2237         /* Filter Table Enable, CFI not used for packet acceptance */
2238         reg = E1000_READ_REG(hw, E1000_RCTL);
2239         reg &= ~E1000_RCTL_CFIEN;
2240         reg |= E1000_RCTL_VFE;
2241         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2242
2243         /* restore VFTA table */
2244         for (i = 0; i < IGB_VFTA_SIZE; i++)
2245                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
2246 }
2247
2248 static void
2249 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2250 {
2251         struct e1000_hw *hw =
2252                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2253         uint32_t reg;
2254
2255         /* VLAN Mode Disable */
2256         reg = E1000_READ_REG(hw, E1000_CTRL);
2257         reg &= ~E1000_CTRL_VME;
2258         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2259 }
2260
2261 static void
2262 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2263 {
2264         struct e1000_hw *hw =
2265                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2266         uint32_t reg;
2267
2268         /* VLAN Mode Enable */
2269         reg = E1000_READ_REG(hw, E1000_CTRL);
2270         reg |= E1000_CTRL_VME;
2271         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2272 }
2273
2274 static void
2275 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2276 {
2277         struct e1000_hw *hw =
2278                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2279         uint32_t reg;
2280
2281         /* CTRL_EXT: Extended VLAN */
2282         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2283         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
2284         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2285
2286         /* Update maximum packet length */
2287         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2288                 E1000_WRITE_REG(hw, E1000_RLPML,
2289                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2290                                                 VLAN_TAG_SIZE);
2291 }
2292
2293 static void
2294 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2295 {
2296         struct e1000_hw *hw =
2297                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2298         uint32_t reg;
2299
2300         /* CTRL_EXT: Extended VLAN */
2301         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2302         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
2303         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2304
2305         /* Update maximum packet length */
2306         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2307                 E1000_WRITE_REG(hw, E1000_RLPML,
2308                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2309                                                 2 * VLAN_TAG_SIZE);
2310 }
2311
2312 static void
2313 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2314 {
2315         if(mask & ETH_VLAN_STRIP_MASK){
2316                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2317                         igb_vlan_hw_strip_enable(dev);
2318                 else
2319                         igb_vlan_hw_strip_disable(dev);
2320         }
2321
2322         if(mask & ETH_VLAN_FILTER_MASK){
2323                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2324                         igb_vlan_hw_filter_enable(dev);
2325                 else
2326                         igb_vlan_hw_filter_disable(dev);
2327         }
2328
2329         if(mask & ETH_VLAN_EXTEND_MASK){
2330                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2331                         igb_vlan_hw_extend_enable(dev);
2332                 else
2333                         igb_vlan_hw_extend_disable(dev);
2334         }
2335 }
2336
2337
2338 /**
2339  * It enables the interrupt mask and then enable the interrupt.
2340  *
2341  * @param dev
2342  *  Pointer to struct rte_eth_dev.
2343  *
2344  * @return
2345  *  - On success, zero.
2346  *  - On failure, a negative value.
2347  */
2348 static int
2349 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2350 {
2351         struct e1000_interrupt *intr =
2352                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2353
2354         intr->mask |= E1000_ICR_LSC;
2355
2356         return 0;
2357 }
2358
2359 /* It clears the interrupt causes and enables the interrupt.
2360  * It will be called once only during nic initialized.
2361  *
2362  * @param dev
2363  *  Pointer to struct rte_eth_dev.
2364  *
2365  * @return
2366  *  - On success, zero.
2367  *  - On failure, a negative value.
2368  */
2369 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2370 {
2371         uint32_t mask, regval;
2372         struct e1000_hw *hw =
2373                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2374         struct rte_eth_dev_info dev_info;
2375
2376         memset(&dev_info, 0, sizeof(dev_info));
2377         eth_igb_infos_get(dev, &dev_info);
2378
2379         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2380         regval = E1000_READ_REG(hw, E1000_EIMS);
2381         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2382
2383         return 0;
2384 }
2385
2386 /*
2387  * It reads ICR and gets interrupt causes, check it and set a bit flag
2388  * to update link status.
2389  *
2390  * @param dev
2391  *  Pointer to struct rte_eth_dev.
2392  *
2393  * @return
2394  *  - On success, zero.
2395  *  - On failure, a negative value.
2396  */
2397 static int
2398 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2399 {
2400         uint32_t icr;
2401         struct e1000_hw *hw =
2402                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2403         struct e1000_interrupt *intr =
2404                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2405
2406         igb_intr_disable(hw);
2407
2408         /* read-on-clear nic registers here */
2409         icr = E1000_READ_REG(hw, E1000_ICR);
2410
2411         intr->flags = 0;
2412         if (icr & E1000_ICR_LSC) {
2413                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2414         }
2415
2416         if (icr & E1000_ICR_VMMB)
2417                 intr->flags |= E1000_FLAG_MAILBOX;
2418
2419         return 0;
2420 }
2421
2422 /*
2423  * It executes link_update after knowing an interrupt is prsent.
2424  *
2425  * @param dev
2426  *  Pointer to struct rte_eth_dev.
2427  *
2428  * @return
2429  *  - On success, zero.
2430  *  - On failure, a negative value.
2431  */
2432 static int
2433 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2434 {
2435         struct e1000_hw *hw =
2436                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2437         struct e1000_interrupt *intr =
2438                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2439         uint32_t tctl, rctl;
2440         struct rte_eth_link link;
2441         int ret;
2442
2443         if (intr->flags & E1000_FLAG_MAILBOX) {
2444                 igb_pf_mbx_process(dev);
2445                 intr->flags &= ~E1000_FLAG_MAILBOX;
2446         }
2447
2448         igb_intr_enable(dev);
2449         rte_intr_enable(&(dev->pci_dev->intr_handle));
2450
2451         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2452                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2453
2454                 /* set get_link_status to check register later */
2455                 hw->mac.get_link_status = 1;
2456                 ret = eth_igb_link_update(dev, 0);
2457
2458                 /* check if link has changed */
2459                 if (ret < 0)
2460                         return 0;
2461
2462                 memset(&link, 0, sizeof(link));
2463                 rte_igb_dev_atomic_read_link_status(dev, &link);
2464                 if (link.link_status) {
2465                         PMD_INIT_LOG(INFO,
2466                                      " Port %d: Link Up - speed %u Mbps - %s",
2467                                      dev->data->port_id,
2468                                      (unsigned)link.link_speed,
2469                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2470                                      "full-duplex" : "half-duplex");
2471                 } else {
2472                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2473                                      dev->data->port_id);
2474                 }
2475
2476                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2477                              dev->pci_dev->addr.domain,
2478                              dev->pci_dev->addr.bus,
2479                              dev->pci_dev->addr.devid,
2480                              dev->pci_dev->addr.function);
2481                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2482                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2483                 if (link.link_status) {
2484                         /* enable Tx/Rx */
2485                         tctl |= E1000_TCTL_EN;
2486                         rctl |= E1000_RCTL_EN;
2487                 } else {
2488                         /* disable Tx/Rx */
2489                         tctl &= ~E1000_TCTL_EN;
2490                         rctl &= ~E1000_RCTL_EN;
2491                 }
2492                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2493                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2494                 E1000_WRITE_FLUSH(hw);
2495                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2496         }
2497
2498         return 0;
2499 }
2500
2501 /**
2502  * Interrupt handler which shall be registered at first.
2503  *
2504  * @param handle
2505  *  Pointer to interrupt handle.
2506  * @param param
2507  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2508  *
2509  * @return
2510  *  void
2511  */
2512 static void
2513 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2514                                                         void *param)
2515 {
2516         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2517
2518         eth_igb_interrupt_get_status(dev);
2519         eth_igb_interrupt_action(dev);
2520 }
2521
2522 static int
2523 eth_igb_led_on(struct rte_eth_dev *dev)
2524 {
2525         struct e1000_hw *hw;
2526
2527         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2528         return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2529 }
2530
2531 static int
2532 eth_igb_led_off(struct rte_eth_dev *dev)
2533 {
2534         struct e1000_hw *hw;
2535
2536         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2537         return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2538 }
2539
2540 static int
2541 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2542 {
2543         struct e1000_hw *hw;
2544         uint32_t ctrl;
2545         int tx_pause;
2546         int rx_pause;
2547
2548         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2549         fc_conf->pause_time = hw->fc.pause_time;
2550         fc_conf->high_water = hw->fc.high_water;
2551         fc_conf->low_water = hw->fc.low_water;
2552         fc_conf->send_xon = hw->fc.send_xon;
2553         fc_conf->autoneg = hw->mac.autoneg;
2554
2555         /*
2556          * Return rx_pause and tx_pause status according to actual setting of
2557          * the TFCE and RFCE bits in the CTRL register.
2558          */
2559         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2560         if (ctrl & E1000_CTRL_TFCE)
2561                 tx_pause = 1;
2562         else
2563                 tx_pause = 0;
2564
2565         if (ctrl & E1000_CTRL_RFCE)
2566                 rx_pause = 1;
2567         else
2568                 rx_pause = 0;
2569
2570         if (rx_pause && tx_pause)
2571                 fc_conf->mode = RTE_FC_FULL;
2572         else if (rx_pause)
2573                 fc_conf->mode = RTE_FC_RX_PAUSE;
2574         else if (tx_pause)
2575                 fc_conf->mode = RTE_FC_TX_PAUSE;
2576         else
2577                 fc_conf->mode = RTE_FC_NONE;
2578
2579         return 0;
2580 }
2581
2582 static int
2583 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2584 {
2585         struct e1000_hw *hw;
2586         int err;
2587         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2588                 e1000_fc_none,
2589                 e1000_fc_rx_pause,
2590                 e1000_fc_tx_pause,
2591                 e1000_fc_full
2592         };
2593         uint32_t rx_buf_size;
2594         uint32_t max_high_water;
2595         uint32_t rctl;
2596
2597         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2598         if (fc_conf->autoneg != hw->mac.autoneg)
2599                 return -ENOTSUP;
2600         rx_buf_size = igb_get_rx_buffer_size(hw);
2601         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2602
2603         /* At least reserve one Ethernet frame for watermark */
2604         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2605         if ((fc_conf->high_water > max_high_water) ||
2606             (fc_conf->high_water < fc_conf->low_water)) {
2607                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2608                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2609                 return -EINVAL;
2610         }
2611
2612         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2613         hw->fc.pause_time     = fc_conf->pause_time;
2614         hw->fc.high_water     = fc_conf->high_water;
2615         hw->fc.low_water      = fc_conf->low_water;
2616         hw->fc.send_xon       = fc_conf->send_xon;
2617
2618         err = e1000_setup_link_generic(hw);
2619         if (err == E1000_SUCCESS) {
2620
2621                 /* check if we want to forward MAC frames - driver doesn't have native
2622                  * capability to do that, so we'll write the registers ourselves */
2623
2624                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2625
2626                 /* set or clear MFLCN.PMCF bit depending on configuration */
2627                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2628                         rctl |= E1000_RCTL_PMCF;
2629                 else
2630                         rctl &= ~E1000_RCTL_PMCF;
2631
2632                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2633                 E1000_WRITE_FLUSH(hw);
2634
2635                 return 0;
2636         }
2637
2638         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2639         return -EIO;
2640 }
2641
2642 #define E1000_RAH_POOLSEL_SHIFT      (18)
2643 static void
2644 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2645                 uint32_t index, __rte_unused uint32_t pool)
2646 {
2647         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2648         uint32_t rah;
2649
2650         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2651         rah = E1000_READ_REG(hw, E1000_RAH(index));
2652         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2653         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2654 }
2655
2656 static void
2657 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2658 {
2659         uint8_t addr[ETHER_ADDR_LEN];
2660         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2661
2662         memset(addr, 0, sizeof(addr));
2663
2664         e1000_rar_set(hw, addr, index);
2665 }
2666
2667 static void
2668 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2669                                 struct ether_addr *addr)
2670 {
2671         eth_igb_rar_clear(dev, 0);
2672
2673         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2674 }
2675 /*
2676  * Virtual Function operations
2677  */
2678 static void
2679 igbvf_intr_disable(struct e1000_hw *hw)
2680 {
2681         PMD_INIT_FUNC_TRACE();
2682
2683         /* Clear interrupt mask to stop from interrupts being generated */
2684         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2685
2686         E1000_WRITE_FLUSH(hw);
2687 }
2688
2689 static void
2690 igbvf_stop_adapter(struct rte_eth_dev *dev)
2691 {
2692         u32 reg_val;
2693         u16 i;
2694         struct rte_eth_dev_info dev_info;
2695         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2696
2697         memset(&dev_info, 0, sizeof(dev_info));
2698         eth_igbvf_infos_get(dev, &dev_info);
2699
2700         /* Clear interrupt mask to stop from interrupts being generated */
2701         igbvf_intr_disable(hw);
2702
2703         /* Clear any pending interrupts, flush previous writes */
2704         E1000_READ_REG(hw, E1000_EICR);
2705
2706         /* Disable the transmit unit.  Each queue must be disabled. */
2707         for (i = 0; i < dev_info.max_tx_queues; i++)
2708                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2709
2710         /* Disable the receive unit by stopping each queue */
2711         for (i = 0; i < dev_info.max_rx_queues; i++) {
2712                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2713                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2714                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2715                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2716                         ;
2717         }
2718
2719         /* flush all queues disables */
2720         E1000_WRITE_FLUSH(hw);
2721         msec_delay(2);
2722 }
2723
2724 static int eth_igbvf_link_update(struct e1000_hw *hw)
2725 {
2726         struct e1000_mbx_info *mbx = &hw->mbx;
2727         struct e1000_mac_info *mac = &hw->mac;
2728         int ret_val = E1000_SUCCESS;
2729
2730         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2731
2732         /*
2733          * We only want to run this if there has been a rst asserted.
2734          * in this case that could mean a link change, device reset,
2735          * or a virtual function reset
2736          */
2737
2738         /* If we were hit with a reset or timeout drop the link */
2739         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2740                 mac->get_link_status = TRUE;
2741
2742         if (!mac->get_link_status)
2743                 goto out;
2744
2745         /* if link status is down no point in checking to see if pf is up */
2746         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2747                 goto out;
2748
2749         /* if we passed all the tests above then the link is up and we no
2750          * longer need to check for link */
2751         mac->get_link_status = FALSE;
2752
2753 out:
2754         return ret_val;
2755 }
2756
2757
2758 static int
2759 igbvf_dev_configure(struct rte_eth_dev *dev)
2760 {
2761         struct rte_eth_conf* conf = &dev->data->dev_conf;
2762
2763         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2764                      dev->data->port_id);
2765
2766         /*
2767          * VF has no ability to enable/disable HW CRC
2768          * Keep the persistent behavior the same as Host PF
2769          */
2770 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2771         if (!conf->rxmode.hw_strip_crc) {
2772                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2773                 conf->rxmode.hw_strip_crc = 1;
2774         }
2775 #else
2776         if (conf->rxmode.hw_strip_crc) {
2777                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2778                 conf->rxmode.hw_strip_crc = 0;
2779         }
2780 #endif
2781
2782         return 0;
2783 }
2784
2785 static int
2786 igbvf_dev_start(struct rte_eth_dev *dev)
2787 {
2788         struct e1000_hw *hw =
2789                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2790         struct e1000_adapter *adapter =
2791                 E1000_DEV_PRIVATE(dev->data->dev_private);
2792         int ret;
2793
2794         PMD_INIT_FUNC_TRACE();
2795
2796         hw->mac.ops.reset_hw(hw);
2797         adapter->stopped = 0;
2798
2799         /* Set all vfta */
2800         igbvf_set_vfta_all(dev,1);
2801
2802         eth_igbvf_tx_init(dev);
2803
2804         /* This can fail when allocating mbufs for descriptor rings */
2805         ret = eth_igbvf_rx_init(dev);
2806         if (ret) {
2807                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2808                 igb_dev_clear_queues(dev);
2809                 return ret;
2810         }
2811
2812         return 0;
2813 }
2814
2815 static void
2816 igbvf_dev_stop(struct rte_eth_dev *dev)
2817 {
2818         PMD_INIT_FUNC_TRACE();
2819
2820         igbvf_stop_adapter(dev);
2821
2822         /*
2823           * Clear what we set, but we still keep shadow_vfta to
2824           * restore after device starts
2825           */
2826         igbvf_set_vfta_all(dev,0);
2827
2828         igb_dev_clear_queues(dev);
2829 }
2830
2831 static void
2832 igbvf_dev_close(struct rte_eth_dev *dev)
2833 {
2834         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2835         struct e1000_adapter *adapter =
2836                 E1000_DEV_PRIVATE(dev->data->dev_private);
2837
2838         PMD_INIT_FUNC_TRACE();
2839
2840         e1000_reset_hw(hw);
2841
2842         igbvf_dev_stop(dev);
2843         adapter->stopped = 1;
2844         igb_dev_free_queues(dev);
2845 }
2846
2847 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2848 {
2849         struct e1000_mbx_info *mbx = &hw->mbx;
2850         uint32_t msgbuf[2];
2851         s32 err;
2852
2853         /* After set vlan, vlan strip will also be enabled in igb driver*/
2854         msgbuf[0] = E1000_VF_SET_VLAN;
2855         msgbuf[1] = vid;
2856         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2857         if (on)
2858                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2859
2860         err = mbx->ops.write_posted(hw, msgbuf, 2, 0);
2861         if (err)
2862                 goto mbx_err;
2863
2864         err = mbx->ops.read_posted(hw, msgbuf, 2, 0);
2865         if (err)
2866                 goto mbx_err;
2867
2868         msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS;
2869         if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK))
2870                 err = -EINVAL;
2871
2872 mbx_err:
2873         return err;
2874 }
2875
2876 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2877 {
2878         struct e1000_hw *hw =
2879                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2880         struct e1000_vfta * shadow_vfta =
2881                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2882         int i = 0, j = 0, vfta = 0, mask = 1;
2883
2884         for (i = 0; i < IGB_VFTA_SIZE; i++){
2885                 vfta = shadow_vfta->vfta[i];
2886                 if(vfta){
2887                         mask = 1;
2888                         for (j = 0; j < 32; j++){
2889                                 if(vfta & mask)
2890                                         igbvf_set_vfta(hw,
2891                                                 (uint16_t)((i<<5)+j), on);
2892                                 mask<<=1;
2893                         }
2894                 }
2895         }
2896
2897 }
2898
2899 static int
2900 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2901 {
2902         struct e1000_hw *hw =
2903                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2904         struct e1000_vfta * shadow_vfta =
2905                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2906         uint32_t vid_idx = 0;
2907         uint32_t vid_bit = 0;
2908         int ret = 0;
2909
2910         PMD_INIT_FUNC_TRACE();
2911
2912         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2913         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2914         if(ret){
2915                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2916                 return ret;
2917         }
2918         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2919         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2920
2921         /*Save what we set and retore it after device reset*/
2922         if (on)
2923                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2924         else
2925                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2926
2927         return 0;
2928 }
2929
2930 static void
2931 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
2932 {
2933         struct e1000_hw *hw =
2934                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2935
2936         /* index is not used by rar_set() */
2937         hw->mac.ops.rar_set(hw, (void *)addr, 0);
2938 }
2939
2940
2941 static int
2942 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2943                         struct rte_eth_rss_reta_entry64 *reta_conf,
2944                         uint16_t reta_size)
2945 {
2946         uint8_t i, j, mask;
2947         uint32_t reta, r;
2948         uint16_t idx, shift;
2949         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2950
2951         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2952                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2953                         "(%d) doesn't match the number hardware can supported "
2954                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2955                 return -EINVAL;
2956         }
2957
2958         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2959                 idx = i / RTE_RETA_GROUP_SIZE;
2960                 shift = i % RTE_RETA_GROUP_SIZE;
2961                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2962                                                 IGB_4_BIT_MASK);
2963                 if (!mask)
2964                         continue;
2965                 if (mask == IGB_4_BIT_MASK)
2966                         r = 0;
2967                 else
2968                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2969                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2970                         if (mask & (0x1 << j))
2971                                 reta |= reta_conf[idx].reta[shift + j] <<
2972                                                         (CHAR_BIT * j);
2973                         else
2974                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2975                 }
2976                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2977         }
2978
2979         return 0;
2980 }
2981
2982 static int
2983 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2984                        struct rte_eth_rss_reta_entry64 *reta_conf,
2985                        uint16_t reta_size)
2986 {
2987         uint8_t i, j, mask;
2988         uint32_t reta;
2989         uint16_t idx, shift;
2990         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2991
2992         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2993                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2994                         "(%d) doesn't match the number hardware can supported "
2995                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2996                 return -EINVAL;
2997         }
2998
2999         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3000                 idx = i / RTE_RETA_GROUP_SIZE;
3001                 shift = i % RTE_RETA_GROUP_SIZE;
3002                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3003                                                 IGB_4_BIT_MASK);
3004                 if (!mask)
3005                         continue;
3006                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3007                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
3008                         if (mask & (0x1 << j))
3009                                 reta_conf[idx].reta[shift + j] =
3010                                         ((reta >> (CHAR_BIT * j)) &
3011                                                 IGB_8_BIT_MASK);
3012                 }
3013         }
3014
3015         return 0;
3016 }
3017
3018 #define MAC_TYPE_FILTER_SUP(type)    do {\
3019         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
3020                 (type) != e1000_82576)\
3021                 return -ENOTSUP;\
3022 } while (0)
3023
3024 static int
3025 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
3026                         struct rte_eth_syn_filter *filter,
3027                         bool add)
3028 {
3029         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3030         uint32_t synqf, rfctl;
3031
3032         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3033                 return -EINVAL;
3034
3035         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3036
3037         if (add) {
3038                 if (synqf & E1000_SYN_FILTER_ENABLE)
3039                         return -EINVAL;
3040
3041                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
3042                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
3043
3044                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3045                 if (filter->hig_pri)
3046                         rfctl |= E1000_RFCTL_SYNQFP;
3047                 else
3048                         rfctl &= ~E1000_RFCTL_SYNQFP;
3049
3050                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
3051         } else {
3052                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
3053                         return -ENOENT;
3054                 synqf = 0;
3055         }
3056
3057         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
3058         E1000_WRITE_FLUSH(hw);
3059         return 0;
3060 }
3061
3062 static int
3063 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
3064                         struct rte_eth_syn_filter *filter)
3065 {
3066         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3067         uint32_t synqf, rfctl;
3068
3069         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3070         if (synqf & E1000_SYN_FILTER_ENABLE) {
3071                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3072                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
3073                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
3074                                 E1000_SYN_FILTER_QUEUE_SHIFT);
3075                 return 0;
3076         }
3077
3078         return -ENOENT;
3079 }
3080
3081 static int
3082 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
3083                         enum rte_filter_op filter_op,
3084                         void *arg)
3085 {
3086         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3087         int ret;
3088
3089         MAC_TYPE_FILTER_SUP(hw->mac.type);
3090
3091         if (filter_op == RTE_ETH_FILTER_NOP)
3092                 return 0;
3093
3094         if (arg == NULL) {
3095                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3096                             filter_op);
3097                 return -EINVAL;
3098         }
3099
3100         switch (filter_op) {
3101         case RTE_ETH_FILTER_ADD:
3102                 ret = eth_igb_syn_filter_set(dev,
3103                                 (struct rte_eth_syn_filter *)arg,
3104                                 TRUE);
3105                 break;
3106         case RTE_ETH_FILTER_DELETE:
3107                 ret = eth_igb_syn_filter_set(dev,
3108                                 (struct rte_eth_syn_filter *)arg,
3109                                 FALSE);
3110                 break;
3111         case RTE_ETH_FILTER_GET:
3112                 ret = eth_igb_syn_filter_get(dev,
3113                                 (struct rte_eth_syn_filter *)arg);
3114                 break;
3115         default:
3116                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
3117                 ret = -EINVAL;
3118                 break;
3119         }
3120
3121         return ret;
3122 }
3123
3124 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
3125         if ((type) != e1000_82580 && (type) != e1000_i350)\
3126                 return -ENOSYS; \
3127 } while (0)
3128
3129 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
3130 static inline int
3131 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
3132                         struct e1000_2tuple_filter_info *filter_info)
3133 {
3134         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3135                 return -EINVAL;
3136         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3137                 return -EINVAL;  /* filter index is out of range. */
3138         if (filter->tcp_flags > TCP_FLAG_ALL)
3139                 return -EINVAL;  /* flags is invalid. */
3140
3141         switch (filter->dst_port_mask) {
3142         case UINT16_MAX:
3143                 filter_info->dst_port_mask = 0;
3144                 filter_info->dst_port = filter->dst_port;
3145                 break;
3146         case 0:
3147                 filter_info->dst_port_mask = 1;
3148                 break;
3149         default:
3150                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3151                 return -EINVAL;
3152         }
3153
3154         switch (filter->proto_mask) {
3155         case UINT8_MAX:
3156                 filter_info->proto_mask = 0;
3157                 filter_info->proto = filter->proto;
3158                 break;
3159         case 0:
3160                 filter_info->proto_mask = 1;
3161                 break;
3162         default:
3163                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3164                 return -EINVAL;
3165         }
3166
3167         filter_info->priority = (uint8_t)filter->priority;
3168         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3169                 filter_info->tcp_flags = filter->tcp_flags;
3170         else
3171                 filter_info->tcp_flags = 0;
3172
3173         return 0;
3174 }
3175
3176 static inline struct e1000_2tuple_filter *
3177 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
3178                         struct e1000_2tuple_filter_info *key)
3179 {
3180         struct e1000_2tuple_filter *it;
3181
3182         TAILQ_FOREACH(it, filter_list, entries) {
3183                 if (memcmp(key, &it->filter_info,
3184                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
3185                         return it;
3186                 }
3187         }
3188         return NULL;
3189 }
3190
3191 /*
3192  * igb_add_2tuple_filter - add a 2tuple filter
3193  *
3194  * @param
3195  * dev: Pointer to struct rte_eth_dev.
3196  * ntuple_filter: ponter to the filter that will be added.
3197  *
3198  * @return
3199  *    - On success, zero.
3200  *    - On failure, a negative value.
3201  */
3202 static int
3203 igb_add_2tuple_filter(struct rte_eth_dev *dev,
3204                         struct rte_eth_ntuple_filter *ntuple_filter)
3205 {
3206         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3207         struct e1000_filter_info *filter_info =
3208                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3209         struct e1000_2tuple_filter *filter;
3210         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
3211         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3212         int i, ret;
3213
3214         filter = rte_zmalloc("e1000_2tuple_filter",
3215                         sizeof(struct e1000_2tuple_filter), 0);
3216         if (filter == NULL)
3217                 return -ENOMEM;
3218
3219         ret = ntuple_filter_to_2tuple(ntuple_filter,
3220                                       &filter->filter_info);
3221         if (ret < 0) {
3222                 rte_free(filter);
3223                 return ret;
3224         }
3225         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3226                                          &filter->filter_info) != NULL) {
3227                 PMD_DRV_LOG(ERR, "filter exists.");
3228                 rte_free(filter);
3229                 return -EEXIST;
3230         }
3231         filter->queue = ntuple_filter->queue;
3232
3233         /*
3234          * look for an unused 2tuple filter index,
3235          * and insert the filter to list.
3236          */
3237         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
3238                 if (!(filter_info->twotuple_mask & (1 << i))) {
3239                         filter_info->twotuple_mask |= 1 << i;
3240                         filter->index = i;
3241                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
3242                                           filter,
3243                                           entries);
3244                         break;
3245                 }
3246         }
3247         if (i >= E1000_MAX_TTQF_FILTERS) {
3248                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
3249                 rte_free(filter);
3250                 return -ENOSYS;
3251         }
3252
3253         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3254         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3255                 imir |= E1000_IMIR_PORT_BP;
3256         else
3257                 imir &= ~E1000_IMIR_PORT_BP;
3258
3259         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3260
3261         ttqf |= E1000_TTQF_QUEUE_ENABLE;
3262         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
3263         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
3264         if (filter->filter_info.proto_mask == 0)
3265                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
3266
3267         /* tcp flags bits setting. */
3268         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3269                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3270                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3271                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3272                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3273                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3274                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3275                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3276                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3277                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3278                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3279                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3280                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3281         } else
3282                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3283         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3284         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
3285         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3286         return 0;
3287 }
3288
3289 /*
3290  * igb_remove_2tuple_filter - remove a 2tuple filter
3291  *
3292  * @param
3293  * dev: Pointer to struct rte_eth_dev.
3294  * ntuple_filter: ponter to the filter that will be removed.
3295  *
3296  * @return
3297  *    - On success, zero.
3298  *    - On failure, a negative value.
3299  */
3300 static int
3301 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
3302                         struct rte_eth_ntuple_filter *ntuple_filter)
3303 {
3304         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3305         struct e1000_filter_info *filter_info =
3306                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3307         struct e1000_2tuple_filter_info filter_2tuple;
3308         struct e1000_2tuple_filter *filter;
3309         int ret;
3310
3311         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
3312         ret = ntuple_filter_to_2tuple(ntuple_filter,
3313                                       &filter_2tuple);
3314         if (ret < 0)
3315                 return ret;
3316
3317         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3318                                          &filter_2tuple);
3319         if (filter == NULL) {
3320                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3321                 return -ENOENT;
3322         }
3323
3324         filter_info->twotuple_mask &= ~(1 << filter->index);
3325         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
3326         rte_free(filter);
3327
3328         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
3329         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3330         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3331         return 0;
3332 }
3333
3334 static inline struct e1000_flex_filter *
3335 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
3336                         struct e1000_flex_filter_info *key)
3337 {
3338         struct e1000_flex_filter *it;
3339
3340         TAILQ_FOREACH(it, filter_list, entries) {
3341                 if (memcmp(key, &it->filter_info,
3342                         sizeof(struct e1000_flex_filter_info)) == 0)
3343                         return it;
3344         }
3345
3346         return NULL;
3347 }
3348
3349 static int
3350 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3351                         struct rte_eth_flex_filter *filter,
3352                         bool add)
3353 {
3354         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3355         struct e1000_filter_info *filter_info =
3356                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3357         struct e1000_flex_filter *flex_filter, *it;
3358         uint32_t wufc, queueing, mask;
3359         uint32_t reg_off;
3360         uint8_t shift, i, j = 0;
3361
3362         flex_filter = rte_zmalloc("e1000_flex_filter",
3363                         sizeof(struct e1000_flex_filter), 0);
3364         if (flex_filter == NULL)
3365                 return -ENOMEM;
3366
3367         flex_filter->filter_info.len = filter->len;
3368         flex_filter->filter_info.priority = filter->priority;
3369         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3370         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3371                 mask = 0;
3372                 /* reverse bits in flex filter's mask*/
3373                 for (shift = 0; shift < CHAR_BIT; shift++) {
3374                         if (filter->mask[i] & (0x01 << shift))
3375                                 mask |= (0x80 >> shift);
3376                 }
3377                 flex_filter->filter_info.mask[i] = mask;
3378         }
3379
3380         wufc = E1000_READ_REG(hw, E1000_WUFC);
3381         if (flex_filter->index < E1000_MAX_FHFT)
3382                 reg_off = E1000_FHFT(flex_filter->index);
3383         else
3384                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3385
3386         if (add) {
3387                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3388                                 &flex_filter->filter_info) != NULL) {
3389                         PMD_DRV_LOG(ERR, "filter exists.");
3390                         rte_free(flex_filter);
3391                         return -EEXIST;
3392                 }
3393                 flex_filter->queue = filter->queue;
3394                 /*
3395                  * look for an unused flex filter index
3396                  * and insert the filter into the list.
3397                  */
3398                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3399                         if (!(filter_info->flex_mask & (1 << i))) {
3400                                 filter_info->flex_mask |= 1 << i;
3401                                 flex_filter->index = i;
3402                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3403                                         flex_filter,
3404                                         entries);
3405                                 break;
3406                         }
3407                 }
3408                 if (i >= E1000_MAX_FLEX_FILTERS) {
3409                         PMD_DRV_LOG(ERR, "flex filters are full.");
3410                         rte_free(flex_filter);
3411                         return -ENOSYS;
3412                 }
3413
3414                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3415                                 (E1000_WUFC_FLX0 << flex_filter->index));
3416                 queueing = filter->len |
3417                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3418                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3419                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3420                                 queueing);
3421                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3422                         E1000_WRITE_REG(hw, reg_off,
3423                                         flex_filter->filter_info.dwords[j]);
3424                         reg_off += sizeof(uint32_t);
3425                         E1000_WRITE_REG(hw, reg_off,
3426                                         flex_filter->filter_info.dwords[++j]);
3427                         reg_off += sizeof(uint32_t);
3428                         E1000_WRITE_REG(hw, reg_off,
3429                                 (uint32_t)flex_filter->filter_info.mask[i]);
3430                         reg_off += sizeof(uint32_t) * 2;
3431                         ++j;
3432                 }
3433         } else {
3434                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3435                                 &flex_filter->filter_info);
3436                 if (it == NULL) {
3437                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3438                         rte_free(flex_filter);
3439                         return -ENOENT;
3440                 }
3441
3442                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3443                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3444                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3445                         (~(E1000_WUFC_FLX0 << it->index)));
3446
3447                 filter_info->flex_mask &= ~(1 << it->index);
3448                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3449                 rte_free(it);
3450                 rte_free(flex_filter);
3451         }
3452
3453         return 0;
3454 }
3455
3456 static int
3457 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3458                         struct rte_eth_flex_filter *filter)
3459 {
3460         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3461         struct e1000_filter_info *filter_info =
3462                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3463         struct e1000_flex_filter flex_filter, *it;
3464         uint32_t wufc, queueing, wufc_en = 0;
3465
3466         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3467         flex_filter.filter_info.len = filter->len;
3468         flex_filter.filter_info.priority = filter->priority;
3469         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3470         memcpy(flex_filter.filter_info.mask, filter->mask,
3471                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3472
3473         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3474                                 &flex_filter.filter_info);
3475         if (it == NULL) {
3476                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3477                 return -ENOENT;
3478         }
3479
3480         wufc = E1000_READ_REG(hw, E1000_WUFC);
3481         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3482
3483         if ((wufc & wufc_en) == wufc_en) {
3484                 uint32_t reg_off = 0;
3485                 if (it->index < E1000_MAX_FHFT)
3486                         reg_off = E1000_FHFT(it->index);
3487                 else
3488                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3489
3490                 queueing = E1000_READ_REG(hw,
3491                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3492                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3493                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3494                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3495                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3496                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3497                 return 0;
3498         }
3499         return -ENOENT;
3500 }
3501
3502 static int
3503 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3504                         enum rte_filter_op filter_op,
3505                         void *arg)
3506 {
3507         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3508         struct rte_eth_flex_filter *filter;
3509         int ret = 0;
3510
3511         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3512
3513         if (filter_op == RTE_ETH_FILTER_NOP)
3514                 return ret;
3515
3516         if (arg == NULL) {
3517                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3518                             filter_op);
3519                 return -EINVAL;
3520         }
3521
3522         filter = (struct rte_eth_flex_filter *)arg;
3523         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3524             || filter->len % sizeof(uint64_t) != 0) {
3525                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3526                 return -EINVAL;
3527         }
3528         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3529                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3530                 return -EINVAL;
3531         }
3532
3533         switch (filter_op) {
3534         case RTE_ETH_FILTER_ADD:
3535                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3536                 break;
3537         case RTE_ETH_FILTER_DELETE:
3538                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3539                 break;
3540         case RTE_ETH_FILTER_GET:
3541                 ret = eth_igb_get_flex_filter(dev, filter);
3542                 break;
3543         default:
3544                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3545                 ret = -EINVAL;
3546                 break;
3547         }
3548
3549         return ret;
3550 }
3551
3552 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3553 static inline int
3554 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3555                         struct e1000_5tuple_filter_info *filter_info)
3556 {
3557         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3558                 return -EINVAL;
3559         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3560                 return -EINVAL;  /* filter index is out of range. */
3561         if (filter->tcp_flags > TCP_FLAG_ALL)
3562                 return -EINVAL;  /* flags is invalid. */
3563
3564         switch (filter->dst_ip_mask) {
3565         case UINT32_MAX:
3566                 filter_info->dst_ip_mask = 0;
3567                 filter_info->dst_ip = filter->dst_ip;
3568                 break;
3569         case 0:
3570                 filter_info->dst_ip_mask = 1;
3571                 break;
3572         default:
3573                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3574                 return -EINVAL;
3575         }
3576
3577         switch (filter->src_ip_mask) {
3578         case UINT32_MAX:
3579                 filter_info->src_ip_mask = 0;
3580                 filter_info->src_ip = filter->src_ip;
3581                 break;
3582         case 0:
3583                 filter_info->src_ip_mask = 1;
3584                 break;
3585         default:
3586                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3587                 return -EINVAL;
3588         }
3589
3590         switch (filter->dst_port_mask) {
3591         case UINT16_MAX:
3592                 filter_info->dst_port_mask = 0;
3593                 filter_info->dst_port = filter->dst_port;
3594                 break;
3595         case 0:
3596                 filter_info->dst_port_mask = 1;
3597                 break;
3598         default:
3599                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3600                 return -EINVAL;
3601         }
3602
3603         switch (filter->src_port_mask) {
3604         case UINT16_MAX:
3605                 filter_info->src_port_mask = 0;
3606                 filter_info->src_port = filter->src_port;
3607                 break;
3608         case 0:
3609                 filter_info->src_port_mask = 1;
3610                 break;
3611         default:
3612                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3613                 return -EINVAL;
3614         }
3615
3616         switch (filter->proto_mask) {
3617         case UINT8_MAX:
3618                 filter_info->proto_mask = 0;
3619                 filter_info->proto = filter->proto;
3620                 break;
3621         case 0:
3622                 filter_info->proto_mask = 1;
3623                 break;
3624         default:
3625                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3626                 return -EINVAL;
3627         }
3628
3629         filter_info->priority = (uint8_t)filter->priority;
3630         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3631                 filter_info->tcp_flags = filter->tcp_flags;
3632         else
3633                 filter_info->tcp_flags = 0;
3634
3635         return 0;
3636 }
3637
3638 static inline struct e1000_5tuple_filter *
3639 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3640                         struct e1000_5tuple_filter_info *key)
3641 {
3642         struct e1000_5tuple_filter *it;
3643
3644         TAILQ_FOREACH(it, filter_list, entries) {
3645                 if (memcmp(key, &it->filter_info,
3646                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3647                         return it;
3648                 }
3649         }
3650         return NULL;
3651 }
3652
3653 /*
3654  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3655  *
3656  * @param
3657  * dev: Pointer to struct rte_eth_dev.
3658  * ntuple_filter: ponter to the filter that will be added.
3659  *
3660  * @return
3661  *    - On success, zero.
3662  *    - On failure, a negative value.
3663  */
3664 static int
3665 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3666                         struct rte_eth_ntuple_filter *ntuple_filter)
3667 {
3668         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3669         struct e1000_filter_info *filter_info =
3670                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3671         struct e1000_5tuple_filter *filter;
3672         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3673         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3674         uint8_t i;
3675         int ret;
3676
3677         filter = rte_zmalloc("e1000_5tuple_filter",
3678                         sizeof(struct e1000_5tuple_filter), 0);
3679         if (filter == NULL)
3680                 return -ENOMEM;
3681
3682         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3683                                             &filter->filter_info);
3684         if (ret < 0) {
3685                 rte_free(filter);
3686                 return ret;
3687         }
3688
3689         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3690                                          &filter->filter_info) != NULL) {
3691                 PMD_DRV_LOG(ERR, "filter exists.");
3692                 rte_free(filter);
3693                 return -EEXIST;
3694         }
3695         filter->queue = ntuple_filter->queue;
3696
3697         /*
3698          * look for an unused 5tuple filter index,
3699          * and insert the filter to list.
3700          */
3701         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3702                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3703                         filter_info->fivetuple_mask |= 1 << i;
3704                         filter->index = i;
3705                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3706                                           filter,
3707                                           entries);
3708                         break;
3709                 }
3710         }
3711         if (i >= E1000_MAX_FTQF_FILTERS) {
3712                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3713                 rte_free(filter);
3714                 return -ENOSYS;
3715         }
3716
3717         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3718         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3719                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3720         if (filter->filter_info.dst_ip_mask == 0)
3721                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3722         if (filter->filter_info.src_port_mask == 0)
3723                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3724         if (filter->filter_info.proto_mask == 0)
3725                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3726         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3727                 E1000_FTQF_QUEUE_MASK;
3728         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3729         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3730         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3731         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3732
3733         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3734         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3735
3736         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3737         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3738                 imir |= E1000_IMIR_PORT_BP;
3739         else
3740                 imir &= ~E1000_IMIR_PORT_BP;
3741         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3742
3743         /* tcp flags bits setting. */
3744         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3745                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3746                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3747                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3748                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3749                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3750                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3751                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3752                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3753                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3754                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3755                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3756                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3757         } else
3758                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3759         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3760         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3761         return 0;
3762 }
3763
3764 /*
3765  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3766  *
3767  * @param
3768  * dev: Pointer to struct rte_eth_dev.
3769  * ntuple_filter: ponter to the filter that will be removed.
3770  *
3771  * @return
3772  *    - On success, zero.
3773  *    - On failure, a negative value.
3774  */
3775 static int
3776 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3777                                 struct rte_eth_ntuple_filter *ntuple_filter)
3778 {
3779         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3780         struct e1000_filter_info *filter_info =
3781                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3782         struct e1000_5tuple_filter_info filter_5tuple;
3783         struct e1000_5tuple_filter *filter;
3784         int ret;
3785
3786         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3787         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3788                                             &filter_5tuple);
3789         if (ret < 0)
3790                 return ret;
3791
3792         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3793                                          &filter_5tuple);
3794         if (filter == NULL) {
3795                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3796                 return -ENOENT;
3797         }
3798
3799         filter_info->fivetuple_mask &= ~(1 << filter->index);
3800         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3801         rte_free(filter);
3802
3803         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3804                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3805         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3806         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3807         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3808         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3809         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3810         return 0;
3811 }
3812
3813 static int
3814 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3815 {
3816         uint32_t rctl;
3817         struct e1000_hw *hw;
3818         struct rte_eth_dev_info dev_info;
3819         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3820                                      VLAN_TAG_SIZE);
3821
3822         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3823
3824 #ifdef RTE_LIBRTE_82571_SUPPORT
3825         /* XXX: not bigger than max_rx_pktlen */
3826         if (hw->mac.type == e1000_82571)
3827                 return -ENOTSUP;
3828 #endif
3829         eth_igb_infos_get(dev, &dev_info);
3830
3831         /* check that mtu is within the allowed range */
3832         if ((mtu < ETHER_MIN_MTU) ||
3833             (frame_size > dev_info.max_rx_pktlen))
3834                 return -EINVAL;
3835
3836         /* refuse mtu that requires the support of scattered packets when this
3837          * feature has not been enabled before. */
3838         if (!dev->data->scattered_rx &&
3839             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3840                 return -EINVAL;
3841
3842         rctl = E1000_READ_REG(hw, E1000_RCTL);
3843
3844         /* switch to jumbo mode if needed */
3845         if (frame_size > ETHER_MAX_LEN) {
3846                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3847                 rctl |= E1000_RCTL_LPE;
3848         } else {
3849                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3850                 rctl &= ~E1000_RCTL_LPE;
3851         }
3852         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3853
3854         /* update max frame size */
3855         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3856
3857         E1000_WRITE_REG(hw, E1000_RLPML,
3858                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3859
3860         return 0;
3861 }
3862
3863 /*
3864  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3865  *
3866  * @param
3867  * dev: Pointer to struct rte_eth_dev.
3868  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3869  * add: if true, add filter, if false, remove filter
3870  *
3871  * @return
3872  *    - On success, zero.
3873  *    - On failure, a negative value.
3874  */
3875 static int
3876 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3877                         struct rte_eth_ntuple_filter *ntuple_filter,
3878                         bool add)
3879 {
3880         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3881         int ret;
3882
3883         switch (ntuple_filter->flags) {
3884         case RTE_5TUPLE_FLAGS:
3885         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3886                 if (hw->mac.type != e1000_82576)
3887                         return -ENOTSUP;
3888                 if (add)
3889                         ret = igb_add_5tuple_filter_82576(dev,
3890                                                           ntuple_filter);
3891                 else
3892                         ret = igb_remove_5tuple_filter_82576(dev,
3893                                                              ntuple_filter);
3894                 break;
3895         case RTE_2TUPLE_FLAGS:
3896         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3897                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3898                         return -ENOTSUP;
3899                 if (add)
3900                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3901                 else
3902                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3903                 break;
3904         default:
3905                 ret = -EINVAL;
3906                 break;
3907         }
3908
3909         return ret;
3910 }
3911
3912 /*
3913  * igb_get_ntuple_filter - get a ntuple filter
3914  *
3915  * @param
3916  * dev: Pointer to struct rte_eth_dev.
3917  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3918  *
3919  * @return
3920  *    - On success, zero.
3921  *    - On failure, a negative value.
3922  */
3923 static int
3924 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3925                         struct rte_eth_ntuple_filter *ntuple_filter)
3926 {
3927         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3928         struct e1000_filter_info *filter_info =
3929                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3930         struct e1000_5tuple_filter_info filter_5tuple;
3931         struct e1000_2tuple_filter_info filter_2tuple;
3932         struct e1000_5tuple_filter *p_5tuple_filter;
3933         struct e1000_2tuple_filter *p_2tuple_filter;
3934         int ret;
3935
3936         switch (ntuple_filter->flags) {
3937         case RTE_5TUPLE_FLAGS:
3938         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3939                 if (hw->mac.type != e1000_82576)
3940                         return -ENOTSUP;
3941                 memset(&filter_5tuple,
3942                         0,
3943                         sizeof(struct e1000_5tuple_filter_info));
3944                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3945                                                     &filter_5tuple);
3946                 if (ret < 0)
3947                         return ret;
3948                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3949                                         &filter_info->fivetuple_list,
3950                                         &filter_5tuple);
3951                 if (p_5tuple_filter == NULL) {
3952                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3953                         return -ENOENT;
3954                 }
3955                 ntuple_filter->queue = p_5tuple_filter->queue;
3956                 break;
3957         case RTE_2TUPLE_FLAGS:
3958         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3959                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3960                         return -ENOTSUP;
3961                 memset(&filter_2tuple,
3962                         0,
3963                         sizeof(struct e1000_2tuple_filter_info));
3964                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3965                 if (ret < 0)
3966                         return ret;
3967                 p_2tuple_filter = igb_2tuple_filter_lookup(
3968                                         &filter_info->twotuple_list,
3969                                         &filter_2tuple);
3970                 if (p_2tuple_filter == NULL) {
3971                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3972                         return -ENOENT;
3973                 }
3974                 ntuple_filter->queue = p_2tuple_filter->queue;
3975                 break;
3976         default:
3977                 ret = -EINVAL;
3978                 break;
3979         }
3980
3981         return 0;
3982 }
3983
3984 /*
3985  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3986  * @dev: pointer to rte_eth_dev structure
3987  * @filter_op:operation will be taken.
3988  * @arg: a pointer to specific structure corresponding to the filter_op
3989  */
3990 static int
3991 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3992                                 enum rte_filter_op filter_op,
3993                                 void *arg)
3994 {
3995         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3996         int ret;
3997
3998         MAC_TYPE_FILTER_SUP(hw->mac.type);
3999
4000         if (filter_op == RTE_ETH_FILTER_NOP)
4001                 return 0;
4002
4003         if (arg == NULL) {
4004                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4005                             filter_op);
4006                 return -EINVAL;
4007         }
4008
4009         switch (filter_op) {
4010         case RTE_ETH_FILTER_ADD:
4011                 ret = igb_add_del_ntuple_filter(dev,
4012                         (struct rte_eth_ntuple_filter *)arg,
4013                         TRUE);
4014                 break;
4015         case RTE_ETH_FILTER_DELETE:
4016                 ret = igb_add_del_ntuple_filter(dev,
4017                         (struct rte_eth_ntuple_filter *)arg,
4018                         FALSE);
4019                 break;
4020         case RTE_ETH_FILTER_GET:
4021                 ret = igb_get_ntuple_filter(dev,
4022                         (struct rte_eth_ntuple_filter *)arg);
4023                 break;
4024         default:
4025                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4026                 ret = -EINVAL;
4027                 break;
4028         }
4029         return ret;
4030 }
4031
4032 static inline int
4033 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
4034                         uint16_t ethertype)
4035 {
4036         int i;
4037
4038         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4039                 if (filter_info->ethertype_filters[i] == ethertype &&
4040                     (filter_info->ethertype_mask & (1 << i)))
4041                         return i;
4042         }
4043         return -1;
4044 }
4045
4046 static inline int
4047 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
4048                         uint16_t ethertype)
4049 {
4050         int i;
4051
4052         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4053                 if (!(filter_info->ethertype_mask & (1 << i))) {
4054                         filter_info->ethertype_mask |= 1 << i;
4055                         filter_info->ethertype_filters[i] = ethertype;
4056                         return i;
4057                 }
4058         }
4059         return -1;
4060 }
4061
4062 static inline int
4063 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
4064                         uint8_t idx)
4065 {
4066         if (idx >= E1000_MAX_ETQF_FILTERS)
4067                 return -1;
4068         filter_info->ethertype_mask &= ~(1 << idx);
4069         filter_info->ethertype_filters[idx] = 0;
4070         return idx;
4071 }
4072
4073
4074 static int
4075 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
4076                         struct rte_eth_ethertype_filter *filter,
4077                         bool add)
4078 {
4079         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4080         struct e1000_filter_info *filter_info =
4081                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4082         uint32_t etqf = 0;
4083         int ret;
4084
4085         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4086                 filter->ether_type == ETHER_TYPE_IPv6) {
4087                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4088                         " ethertype filter.", filter->ether_type);
4089                 return -EINVAL;
4090         }
4091
4092         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4093                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4094                 return -EINVAL;
4095         }
4096         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4097                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4098                 return -EINVAL;
4099         }
4100
4101         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4102         if (ret >= 0 && add) {
4103                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4104                             filter->ether_type);
4105                 return -EEXIST;
4106         }
4107         if (ret < 0 && !add) {
4108                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4109                             filter->ether_type);
4110                 return -ENOENT;
4111         }
4112
4113         if (add) {
4114                 ret = igb_ethertype_filter_insert(filter_info,
4115                         filter->ether_type);
4116                 if (ret < 0) {
4117                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4118                         return -ENOSYS;
4119                 }
4120
4121                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
4122                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
4123                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
4124         } else {
4125                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
4126                 if (ret < 0)
4127                         return -ENOSYS;
4128         }
4129         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
4130         E1000_WRITE_FLUSH(hw);
4131
4132         return 0;
4133 }
4134
4135 static int
4136 igb_get_ethertype_filter(struct rte_eth_dev *dev,
4137                         struct rte_eth_ethertype_filter *filter)
4138 {
4139         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4140         struct e1000_filter_info *filter_info =
4141                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4142         uint32_t etqf;
4143         int ret;
4144
4145         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4146         if (ret < 0) {
4147                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4148                             filter->ether_type);
4149                 return -ENOENT;
4150         }
4151
4152         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
4153         if (etqf & E1000_ETQF_FILTER_ENABLE) {
4154                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
4155                 filter->flags = 0;
4156                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
4157                                 E1000_ETQF_QUEUE_SHIFT;
4158                 return 0;
4159         }
4160
4161         return -ENOENT;
4162 }
4163
4164 /*
4165  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
4166  * @dev: pointer to rte_eth_dev structure
4167  * @filter_op:operation will be taken.
4168  * @arg: a pointer to specific structure corresponding to the filter_op
4169  */
4170 static int
4171 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
4172                                 enum rte_filter_op filter_op,
4173                                 void *arg)
4174 {
4175         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4176         int ret;
4177
4178         MAC_TYPE_FILTER_SUP(hw->mac.type);
4179
4180         if (filter_op == RTE_ETH_FILTER_NOP)
4181                 return 0;
4182
4183         if (arg == NULL) {
4184                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4185                             filter_op);
4186                 return -EINVAL;
4187         }
4188
4189         switch (filter_op) {
4190         case RTE_ETH_FILTER_ADD:
4191                 ret = igb_add_del_ethertype_filter(dev,
4192                         (struct rte_eth_ethertype_filter *)arg,
4193                         TRUE);
4194                 break;
4195         case RTE_ETH_FILTER_DELETE:
4196                 ret = igb_add_del_ethertype_filter(dev,
4197                         (struct rte_eth_ethertype_filter *)arg,
4198                         FALSE);
4199                 break;
4200         case RTE_ETH_FILTER_GET:
4201                 ret = igb_get_ethertype_filter(dev,
4202                         (struct rte_eth_ethertype_filter *)arg);
4203                 break;
4204         default:
4205                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4206                 ret = -EINVAL;
4207                 break;
4208         }
4209         return ret;
4210 }
4211
4212 static int
4213 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
4214                      enum rte_filter_type filter_type,
4215                      enum rte_filter_op filter_op,
4216                      void *arg)
4217 {
4218         int ret = -EINVAL;
4219
4220         switch (filter_type) {
4221         case RTE_ETH_FILTER_NTUPLE:
4222                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
4223                 break;
4224         case RTE_ETH_FILTER_ETHERTYPE:
4225                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
4226                 break;
4227         case RTE_ETH_FILTER_SYN:
4228                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
4229                 break;
4230         case RTE_ETH_FILTER_FLEXIBLE:
4231                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
4232                 break;
4233         default:
4234                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4235                                                         filter_type);
4236                 break;
4237         }
4238
4239         return ret;
4240 }
4241
4242 static int
4243 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
4244                          struct ether_addr *mc_addr_set,
4245                          uint32_t nb_mc_addr)
4246 {
4247         struct e1000_hw *hw;
4248
4249         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4250         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
4251         return 0;
4252 }
4253
4254 static uint64_t
4255 igb_read_systime_cyclecounter(struct rte_eth_dev *dev)
4256 {
4257         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4258         uint64_t systime_cycles;
4259
4260         switch (hw->mac.type) {
4261         case e1000_i210:
4262         case e1000_i211:
4263                 /*
4264                  * Need to read System Time Residue Register to be able
4265                  * to read the other two registers.
4266                  */
4267                 E1000_READ_REG(hw, E1000_SYSTIMR);
4268                 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */
4269                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4270                 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4271                                 * NSEC_PER_SEC;
4272                 break;
4273         case e1000_82580:
4274         case e1000_i350:
4275         case e1000_i354:
4276                 /*
4277                  * Need to read System Time Residue Register to be able
4278                  * to read the other two registers.
4279                  */
4280                 E1000_READ_REG(hw, E1000_SYSTIMR);
4281                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4282                 /* Only the 8 LSB are valid. */
4283                 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH)
4284                                 & 0xff) << 32;
4285                 break;
4286         default:
4287                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4288                 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4289                                 << 32;
4290                 break;
4291         }
4292
4293         return systime_cycles;
4294 }
4295
4296 static uint64_t
4297 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4298 {
4299         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4300         uint64_t rx_tstamp_cycles;
4301
4302         switch (hw->mac.type) {
4303         case e1000_i210:
4304         case e1000_i211:
4305                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4306                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4307                 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4308                                 * NSEC_PER_SEC;
4309                 break;
4310         case e1000_82580:
4311         case e1000_i350:
4312         case e1000_i354:
4313                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4314                 /* Only the 8 LSB are valid. */
4315                 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH)
4316                                 & 0xff) << 32;
4317                 break;
4318         default:
4319                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4320                 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4321                                 << 32;
4322                 break;
4323         }
4324
4325         return rx_tstamp_cycles;
4326 }
4327
4328 static uint64_t
4329 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4330 {
4331         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4332         uint64_t tx_tstamp_cycles;
4333
4334         switch (hw->mac.type) {
4335         case e1000_i210:
4336         case e1000_i211:
4337                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4338                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4339                 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4340                                 * NSEC_PER_SEC;
4341                 break;
4342         case e1000_82580:
4343         case e1000_i350:
4344         case e1000_i354:
4345                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4346                 /* Only the 8 LSB are valid. */
4347                 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH)
4348                                 & 0xff) << 32;
4349                 break;
4350         default:
4351                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4352                 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4353                                 << 32;
4354                 break;
4355         }
4356
4357         return tx_tstamp_cycles;
4358 }
4359
4360 static void
4361 igb_start_timecounters(struct rte_eth_dev *dev)
4362 {
4363         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4364         struct e1000_adapter *adapter =
4365                 (struct e1000_adapter *)dev->data->dev_private;
4366         uint32_t incval = 1;
4367         uint32_t shift = 0;
4368         uint64_t mask = E1000_CYCLECOUNTER_MASK;
4369
4370         switch (hw->mac.type) {
4371         case e1000_82580:
4372         case e1000_i350:
4373         case e1000_i354:
4374                 /* 32 LSB bits + 8 MSB bits = 40 bits */
4375                 mask = (1ULL << 40) - 1;
4376                 /* fall-through */
4377         case e1000_i210:
4378         case e1000_i211:
4379                 /*
4380                  * Start incrementing the register
4381                  * used to timestamp PTP packets.
4382                  */
4383                 E1000_WRITE_REG(hw, E1000_TIMINCA, incval);
4384                 break;
4385         case e1000_82576:
4386                 incval = E1000_INCVALUE_82576;
4387                 shift = IGB_82576_TSYNC_SHIFT;
4388                 E1000_WRITE_REG(hw, E1000_TIMINCA,
4389                                 E1000_INCPERIOD_82576 | incval);
4390                 break;
4391         default:
4392                 /* Not supported */
4393                 return;
4394         }
4395
4396         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
4397         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4398         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4399
4400         adapter->systime_tc.cc_mask = mask;
4401         adapter->systime_tc.cc_shift = shift;
4402         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
4403
4404         adapter->rx_tstamp_tc.cc_mask = mask;
4405         adapter->rx_tstamp_tc.cc_shift = shift;
4406         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4407
4408         adapter->tx_tstamp_tc.cc_mask = mask;
4409         adapter->tx_tstamp_tc.cc_shift = shift;
4410         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4411 }
4412
4413 static int
4414 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
4415 {
4416         struct e1000_adapter *adapter =
4417                         (struct e1000_adapter *)dev->data->dev_private;
4418
4419         adapter->systime_tc.nsec += delta;
4420         adapter->rx_tstamp_tc.nsec += delta;
4421         adapter->tx_tstamp_tc.nsec += delta;
4422
4423         return 0;
4424 }
4425
4426 static int
4427 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
4428 {
4429         uint64_t ns;
4430         struct e1000_adapter *adapter =
4431                         (struct e1000_adapter *)dev->data->dev_private;
4432
4433         ns = rte_timespec_to_ns(ts);
4434
4435         /* Set the timecounters to a new value. */
4436         adapter->systime_tc.nsec = ns;
4437         adapter->rx_tstamp_tc.nsec = ns;
4438         adapter->tx_tstamp_tc.nsec = ns;
4439
4440         return 0;
4441 }
4442
4443 static int
4444 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
4445 {
4446         uint64_t ns, systime_cycles;
4447         struct e1000_adapter *adapter =
4448                         (struct e1000_adapter *)dev->data->dev_private;
4449
4450         systime_cycles = igb_read_systime_cyclecounter(dev);
4451         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
4452         *ts = rte_ns_to_timespec(ns);
4453
4454         return 0;
4455 }
4456
4457 static int
4458 igb_timesync_enable(struct rte_eth_dev *dev)
4459 {
4460         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4461         uint32_t tsync_ctl;
4462         uint32_t tsauxc;
4463
4464         /* Stop the timesync system time. */
4465         E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0);
4466         /* Reset the timesync system time value. */
4467         switch (hw->mac.type) {
4468         case e1000_82580:
4469         case e1000_i350:
4470         case e1000_i354:
4471         case e1000_i210:
4472         case e1000_i211:
4473                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0);
4474                 /* fall-through */
4475         case e1000_82576:
4476                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0);
4477                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0);
4478                 break;
4479         default:
4480                 /* Not supported. */
4481                 return -ENOTSUP;
4482         }
4483
4484         /* Enable system time for it isn't on by default. */
4485         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
4486         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
4487         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
4488
4489         igb_start_timecounters(dev);
4490
4491         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4492         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
4493                         (ETHER_TYPE_1588 |
4494                          E1000_ETQF_FILTER_ENABLE |
4495                          E1000_ETQF_1588));
4496
4497         /* Enable timestamping of received PTP packets. */
4498         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4499         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
4500         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4501
4502         /* Enable Timestamping of transmitted PTP packets. */
4503         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4504         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
4505         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4506
4507         return 0;
4508 }
4509
4510 static int
4511 igb_timesync_disable(struct rte_eth_dev *dev)
4512 {
4513         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4514         uint32_t tsync_ctl;
4515
4516         /* Disable timestamping of transmitted PTP packets. */
4517         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4518         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
4519         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4520
4521         /* Disable timestamping of received PTP packets. */
4522         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4523         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
4524         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4525
4526         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4527         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
4528
4529         /* Stop incrementating the System Time registers. */
4530         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
4531
4532         return 0;
4533 }
4534
4535 static int
4536 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4537                                struct timespec *timestamp,
4538                                uint32_t flags __rte_unused)
4539 {
4540         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4541         struct e1000_adapter *adapter =
4542                         (struct e1000_adapter *)dev->data->dev_private;
4543         uint32_t tsync_rxctl;
4544         uint64_t rx_tstamp_cycles;
4545         uint64_t ns;
4546
4547         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4548         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
4549                 return -EINVAL;
4550
4551         rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev);
4552         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
4553         *timestamp = rte_ns_to_timespec(ns);
4554
4555         return  0;
4556 }
4557
4558 static int
4559 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4560                                struct timespec *timestamp)
4561 {
4562         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4563         struct e1000_adapter *adapter =
4564                         (struct e1000_adapter *)dev->data->dev_private;
4565         uint32_t tsync_txctl;
4566         uint64_t tx_tstamp_cycles;
4567         uint64_t ns;
4568
4569         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4570         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
4571                 return -EINVAL;
4572
4573         tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev);
4574         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
4575         *timestamp = rte_ns_to_timespec(ns);
4576
4577         return  0;
4578 }
4579
4580 static int
4581 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4582 {
4583         int count = 0;
4584         int g_ind = 0;
4585         const struct reg_info *reg_group;
4586
4587         while ((reg_group = igb_regs[g_ind++]))
4588                 count += igb_reg_group_count(reg_group);
4589
4590         return count;
4591 }
4592
4593 static int
4594 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4595 {
4596         int count = 0;
4597         int g_ind = 0;
4598         const struct reg_info *reg_group;
4599
4600         while ((reg_group = igbvf_regs[g_ind++]))
4601                 count += igb_reg_group_count(reg_group);
4602
4603         return count;
4604 }
4605
4606 static int
4607 eth_igb_get_regs(struct rte_eth_dev *dev,
4608         struct rte_dev_reg_info *regs)
4609 {
4610         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4611         uint32_t *data = regs->data;
4612         int g_ind = 0;
4613         int count = 0;
4614         const struct reg_info *reg_group;
4615
4616         /* Support only full register dump */
4617         if ((regs->length == 0) ||
4618             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4619                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4620                         hw->device_id;
4621                 while ((reg_group = igb_regs[g_ind++]))
4622                         count += igb_read_regs_group(dev, &data[count],
4623                                                         reg_group);
4624                 return 0;
4625         }
4626
4627         return -ENOTSUP;
4628 }
4629
4630 static int
4631 igbvf_get_regs(struct rte_eth_dev *dev,
4632         struct rte_dev_reg_info *regs)
4633 {
4634         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4635         uint32_t *data = regs->data;
4636         int g_ind = 0;
4637         int count = 0;
4638         const struct reg_info *reg_group;
4639
4640         /* Support only full register dump */
4641         if ((regs->length == 0) ||
4642             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4643                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4644                         hw->device_id;
4645                 while ((reg_group = igbvf_regs[g_ind++]))
4646                         count += igb_read_regs_group(dev, &data[count],
4647                                                         reg_group);
4648                 return 0;
4649         }
4650
4651         return -ENOTSUP;
4652 }
4653
4654 static int
4655 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4656 {
4657         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4658
4659         /* Return unit is byte count */
4660         return hw->nvm.word_size * 2;
4661 }
4662
4663 static int
4664 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4665         struct rte_dev_eeprom_info *in_eeprom)
4666 {
4667         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4668         struct e1000_nvm_info *nvm = &hw->nvm;
4669         uint16_t *data = in_eeprom->data;
4670         int first, length;
4671
4672         first = in_eeprom->offset >> 1;
4673         length = in_eeprom->length >> 1;
4674         if ((first >= hw->nvm.word_size) ||
4675             ((first + length) >= hw->nvm.word_size))
4676                 return -EINVAL;
4677
4678         in_eeprom->magic = hw->vendor_id |
4679                 ((uint32_t)hw->device_id << 16);
4680
4681         if ((nvm->ops.read) == NULL)
4682                 return -ENOTSUP;
4683
4684         return nvm->ops.read(hw, first, length, data);
4685 }
4686
4687 static int
4688 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4689         struct rte_dev_eeprom_info *in_eeprom)
4690 {
4691         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4692         struct e1000_nvm_info *nvm = &hw->nvm;
4693         uint16_t *data = in_eeprom->data;
4694         int first, length;
4695
4696         first = in_eeprom->offset >> 1;
4697         length = in_eeprom->length >> 1;
4698         if ((first >= hw->nvm.word_size) ||
4699             ((first + length) >= hw->nvm.word_size))
4700                 return -EINVAL;
4701
4702         in_eeprom->magic = (uint32_t)hw->vendor_id |
4703                 ((uint32_t)hw->device_id << 16);
4704
4705         if ((nvm->ops.write) == NULL)
4706                 return -ENOTSUP;
4707         return nvm->ops.write(hw,  first, length, data);
4708 }
4709
4710 static struct rte_driver pmd_igb_drv = {
4711         .type = PMD_PDEV,
4712         .init = rte_igb_pmd_init,
4713 };
4714
4715 static struct rte_driver pmd_igbvf_drv = {
4716         .type = PMD_PDEV,
4717         .init = rte_igbvf_pmd_init,
4718 };
4719
4720 static int
4721 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4722 {
4723         struct e1000_hw *hw =
4724                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4725         uint32_t mask = 1 << queue_id;
4726
4727         E1000_WRITE_REG(hw, E1000_EIMC, mask);
4728         E1000_WRITE_FLUSH(hw);
4729
4730         return 0;
4731 }
4732
4733 static int
4734 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4735 {
4736         struct e1000_hw *hw =
4737                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4738         uint32_t mask = 1 << queue_id;
4739         uint32_t regval;
4740
4741         regval = E1000_READ_REG(hw, E1000_EIMS);
4742         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
4743         E1000_WRITE_FLUSH(hw);
4744
4745         rte_intr_enable(&dev->pci_dev->intr_handle);
4746
4747         return 0;
4748 }
4749
4750 static void
4751 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
4752                    uint8_t index, uint8_t offset)
4753 {
4754         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4755
4756         /* clear bits */
4757         val &= ~((uint32_t)0xFF << offset);
4758
4759         /* write vector and valid bit */
4760         val |= (msix_vector | E1000_IVAR_VALID) << offset;
4761
4762         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
4763 }
4764
4765 static void
4766 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
4767                            uint8_t queue, uint8_t msix_vector)
4768 {
4769         uint32_t tmp = 0;
4770
4771         if (hw->mac.type == e1000_82575) {
4772                 if (direction == 0)
4773                         tmp = E1000_EICR_RX_QUEUE0 << queue;
4774                 else if (direction == 1)
4775                         tmp = E1000_EICR_TX_QUEUE0 << queue;
4776                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
4777         } else if (hw->mac.type == e1000_82576) {
4778                 if ((direction == 0) || (direction == 1))
4779                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
4780                                            ((queue & 0x8) << 1) +
4781                                            8 * direction);
4782         } else if ((hw->mac.type == e1000_82580) ||
4783                         (hw->mac.type == e1000_i350) ||
4784                         (hw->mac.type == e1000_i354) ||
4785                         (hw->mac.type == e1000_i210) ||
4786                         (hw->mac.type == e1000_i211)) {
4787                 if ((direction == 0) || (direction == 1))
4788                         eth_igb_write_ivar(hw, msix_vector,
4789                                            queue >> 1,
4790                                            ((queue & 0x1) << 4) +
4791                                            8 * direction);
4792         }
4793 }
4794
4795 /* Sets up the hardware to generate MSI-X interrupts properly
4796  * @hw
4797  *  board private structure
4798  */
4799 static void
4800 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
4801 {
4802         int queue_id;
4803         uint32_t tmpval, regval, intr_mask;
4804         struct e1000_hw *hw =
4805                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4806         uint32_t vec = E1000_MISC_VEC_ID;
4807         uint32_t base = E1000_MISC_VEC_ID;
4808         uint32_t misc_shift = 0;
4809
4810         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4811
4812         /* won't configure msix register if no mapping is done
4813          * between intr vector and event fd
4814          */
4815         if (!rte_intr_dp_is_en(intr_handle))
4816                 return;
4817
4818         if (rte_intr_allow_others(intr_handle)) {
4819                 vec = base = E1000_RX_VEC_START;
4820                 misc_shift = 1;
4821         }
4822
4823         /* set interrupt vector for other causes */
4824         if (hw->mac.type == e1000_82575) {
4825                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
4826                 /* enable MSI-X PBA support */
4827                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
4828
4829                 /* Auto-Mask interrupts upon ICR read */
4830                 tmpval |= E1000_CTRL_EXT_EIAME;
4831                 tmpval |= E1000_CTRL_EXT_IRCA;
4832
4833                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
4834
4835                 /* enable msix_other interrupt */
4836                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
4837                 regval = E1000_READ_REG(hw, E1000_EIAC);
4838                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
4839                 regval = E1000_READ_REG(hw, E1000_EIAM);
4840                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
4841         } else if ((hw->mac.type == e1000_82576) ||
4842                         (hw->mac.type == e1000_82580) ||
4843                         (hw->mac.type == e1000_i350) ||
4844                         (hw->mac.type == e1000_i354) ||
4845                         (hw->mac.type == e1000_i210) ||
4846                         (hw->mac.type == e1000_i211)) {
4847                 /* turn on MSI-X capability first */
4848                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
4849                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
4850                                         E1000_GPIE_NSICR);
4851                 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
4852                         misc_shift;
4853                 regval = E1000_READ_REG(hw, E1000_EIAC);
4854                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
4855
4856                 /* enable msix_other interrupt */
4857                 regval = E1000_READ_REG(hw, E1000_EIMS);
4858                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
4859                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
4860                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
4861         }
4862
4863         /* use EIAM to auto-mask when MSI-X interrupt
4864          * is asserted, this saves a register write for every interrupt
4865          */
4866         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
4867                 misc_shift;
4868         regval = E1000_READ_REG(hw, E1000_EIAM);
4869         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
4870
4871         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
4872                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
4873                 intr_handle->intr_vec[queue_id] = vec;
4874                 if (vec < base + intr_handle->nb_efd - 1)
4875                         vec++;
4876         }
4877
4878         E1000_WRITE_FLUSH(hw);
4879 }
4880
4881 PMD_REGISTER_DRIVER(pmd_igb_drv);
4882 PMD_REGISTER_DRIVER(pmd_igbvf_drv);